diff --git a/cpp/arcticdb/CMakeLists.txt b/cpp/arcticdb/CMakeLists.txt index 0b343513a93..253f585cda4 100644 --- a/cpp/arcticdb/CMakeLists.txt +++ b/cpp/arcticdb/CMakeLists.txt @@ -247,6 +247,7 @@ set(arcticdb_srcs entity/descriptor_item.hpp entity/field_collection.hpp entity/field_collection_proto.hpp + entity/python_bindings_common.hpp log/log.hpp log/trace.hpp pipeline/column_mapping.hpp @@ -273,6 +274,7 @@ set(arcticdb_srcs python/numpy_buffer_holder.hpp python/python_strings.hpp python/python_handler_data.hpp + python/python_bindings_common.hpp python/python_utils.hpp pipeline/string_reducers.hpp processing/aggregation_utils.hpp @@ -329,6 +331,7 @@ set(arcticdb_srcs storage/s3/s3_client_interface.hpp storage/s3/s3_storage_tool.hpp storage/s3/s3_settings.hpp + storage/python_bindings_common.hpp storage/storage_factory.hpp storage/storage_options.hpp storage/storage.hpp @@ -457,6 +460,7 @@ set(arcticdb_srcs entity/metrics.cpp entity/performance_tracing.cpp entity/protobuf_mappings.cpp + entity/python_bindings_common.cpp entity/types.cpp entity/type_utils.cpp entity/types_proto.cpp @@ -478,6 +482,7 @@ set(arcticdb_srcs pipeline/string_pool_utils.cpp pipeline/value_set.cpp pipeline/write_frame.cpp + python/python_bindings_common.cpp python/normalization_checks.cpp python/python_strings.cpp python/python_utils.cpp @@ -528,6 +533,7 @@ set(arcticdb_srcs storage/s3/s3_storage_tool.cpp storage/s3/s3_client_wrapper.cpp storage/s3/s3_client_wrapper.hpp + storage/python_bindings_common.cpp storage/storage_factory.cpp storage/storage_utils.cpp stream/aggregator.cpp diff --git a/cpp/arcticdb/entity/metrics.hpp b/cpp/arcticdb/entity/metrics.hpp index b400691bde1..1c4b4c1680d 100644 --- a/cpp/arcticdb/entity/metrics.hpp +++ b/cpp/arcticdb/entity/metrics.hpp @@ -32,7 +32,7 @@ constexpr int SUMMARY_AGE_BUCKETS = 5; class MetricsConfig { public: - enum class Model { NO_INIT, PUSH, PULL }; + enum class Model { NO_INIT = 0, PUSH = 1, PULL = 2 }; MetricsConfig() : model_(Model::NO_INIT) {} MetricsConfig( diff --git a/cpp/arcticdb/entity/python_bindings_common.cpp b/cpp/arcticdb/entity/python_bindings_common.cpp new file mode 100644 index 00000000000..344c31f744c --- /dev/null +++ b/cpp/arcticdb/entity/python_bindings_common.cpp @@ -0,0 +1,75 @@ +/* Copyright 2023 Man Group Operations Limited + * + * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software + * will be governed by the Apache License, version 2.0. + */ + +#include +#include +#include +#include + +#include +#include +#include + +namespace py = pybind11; + +namespace arcticdb::entity::apy { + +void register_common_entity_bindings(py::module& m, BindingScope scope) { + bool local_bindings = (scope == BindingScope::LOCAL); + py::class_>(m, "AtomKey", py::module_local(local_bindings)) + .def(py::init()) + .def(py::init()) + .def("change_id", &AtomKey::change_id) + .def_property_readonly("id", &AtomKey::id) + .def_property_readonly("version_id", &AtomKey::version_id) + .def_property_readonly("creation_ts", &AtomKey::creation_ts) + .def_property_readonly("content_hash", &AtomKey::content_hash) + .def_property_readonly("start_index", &AtomKey::start_index) + .def_property_readonly("end_index", &AtomKey::end_index) + .def_property_readonly("type", [](const AtomKey& self) { return self.type(); }) + .def(pybind11::self == pybind11::self) + .def(pybind11::self != pybind11::self) + .def("__repr__", &AtomKey::view_human) + .def(py::self < py::self) + .def(py::pickle( + [](const AtomKey& key) { + constexpr int serialization_version = 0; + return py::make_tuple( + serialization_version, + key.id(), + key.version_id(), + key.creation_ts(), + key.content_hash(), + key.start_index(), + key.end_index(), + key.type() + ); + }, + [](py::tuple t) { + util::check(t.size() >= 7, "Invalid AtomKey pickle object!"); + + AtomKey key( + t[1].cast(), + t[2].cast(), + t[3].cast(), + t[4].cast(), + t[5].cast(), + t[6].cast(), + t[7].cast() + ); + return key; + } + )); + + py::class_(m, "VersionedItem", py::module_local(local_bindings)) + .def_property_readonly("symbol", &VersionedItem::symbol) + .def_property_readonly("timestamp", &VersionedItem::timestamp) + .def_property_readonly("version", &VersionedItem::version); +} + +} // namespace arcticdb::entity::apy diff --git a/cpp/arcticdb/entity/python_bindings_common.hpp b/cpp/arcticdb/entity/python_bindings_common.hpp new file mode 100644 index 00000000000..7a2f46ebc81 --- /dev/null +++ b/cpp/arcticdb/entity/python_bindings_common.hpp @@ -0,0 +1,17 @@ +/* Copyright 2023 Man Group Operations Limited + * + * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software + * will be governed by the Apache License, version 2.0. + */ + +#pragma once + +#include + +namespace arcticdb::entity::apy { + +void register_common_entity_bindings(pybind11::module& m, BindingScope scope); + +} // namespace arcticdb::entity::apy diff --git a/cpp/arcticdb/python/python_bindings_common.cpp b/cpp/arcticdb/python/python_bindings_common.cpp new file mode 100644 index 00000000000..3d9f6130b30 --- /dev/null +++ b/cpp/arcticdb/python/python_bindings_common.cpp @@ -0,0 +1,47 @@ +/* Copyright 2025 Man Group Operations Limited + * + * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software + * will be governed by the Apache License, version 2.0. + */ + +#include +#include +#include +#include + +namespace py = pybind11; + +void register_metrics(py::module&& m, arcticdb::BindingScope scope) { + bool local_bindings = (scope == arcticdb::BindingScope::LOCAL); + auto prometheus = m.def_submodule("prometheus"); + py::class_>( + prometheus, "Instance", py::module_local(local_bindings) + ); + + py::class_>( + prometheus, "MetricsConfig", py::module_local(local_bindings) + ) + .def(py::init< + const std::string&, + const std::string&, + const std::string&, + const std::string&, + const std::string&, + const arcticdb::MetricsConfig::Model>()) + .def_property_readonly("host", [](const arcticdb::MetricsConfig& config) { return config.host; }) + .def_property_readonly("port", [](const arcticdb::MetricsConfig& config) { return config.port; }) + .def_property_readonly("job_name", [](const arcticdb::MetricsConfig& config) { return config.job_name; }) + .def_property_readonly("instance", [](const arcticdb::MetricsConfig& config) { return config.instance; }) + .def_property_readonly( + "prometheus_env", [](const arcticdb::MetricsConfig& config) { return config.prometheus_env; } + ) + .def_property_readonly("model", [](const arcticdb::MetricsConfig& config) { return config.model_; }); + + py::enum_(prometheus, "MetricsConfigModel", py::module_local(local_bindings)) + .value("NO_INIT", arcticdb::MetricsConfig::Model::NO_INIT) + .value("PUSH", arcticdb::MetricsConfig::Model::PUSH) + .value("PULL", arcticdb::MetricsConfig::Model::PULL) + .export_values(); +} diff --git a/cpp/arcticdb/python/python_bindings_common.hpp b/cpp/arcticdb/python/python_bindings_common.hpp new file mode 100644 index 00000000000..5e1166a0f78 --- /dev/null +++ b/cpp/arcticdb/python/python_bindings_common.hpp @@ -0,0 +1,26 @@ +/* Copyright 2025 Man Group Operations Limited + * + * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software + * will be governed by the Apache License, version 2.0. + */ + +#pragma once +#include + +namespace pybind11 { +class module_; +using module = module_; +class tuple; +} // namespace pybind11 + +namespace arcticdb { +enum class BindingScope : uint32_t { LOCAL = 0, GLOBAL = 1 }; + +pybind11::tuple to_tuple(const MetricsConfig& config); +MetricsConfig metrics_config_from_tuple(const pybind11::tuple& t); + +} // namespace arcticdb + +void register_metrics(pybind11::module&& m, arcticdb::BindingScope scope); diff --git a/cpp/arcticdb/python/python_module.cpp b/cpp/arcticdb/python/python_module.cpp index 8866c1f5236..aebd3ec01c1 100644 --- a/cpp/arcticdb/python/python_module.cpp +++ b/cpp/arcticdb/python/python_module.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -291,27 +292,6 @@ void register_instrumentation(py::module&& m) { #endif } -void register_metrics(py::module&& m) { - - auto prometheus = m.def_submodule("prometheus"); - py::class_>(prometheus, "Instance"); - - py::class_>(prometheus, "MetricsConfig") - .def(py::init< - const std::string&, - const std::string&, - const std::string&, - const std::string&, - const std::string&, - const arcticdb::MetricsConfig::Model>()); - - py::enum_(prometheus, "MetricsConfigModel") - .value("NO_INIT", arcticdb::MetricsConfig::Model::NO_INIT) - .value("PUSH", arcticdb::MetricsConfig::Model::PUSH) - .value("PULL", arcticdb::MetricsConfig::Model::PULL) - .export_values(); -} - /// Register handling of non-trivial types. For more information @see arcticdb::TypeHandlerRegistry and /// @see arcticdb::ITypeHandler void register_type_handlers() { @@ -350,36 +330,35 @@ PYBIND11_MODULE(arcticdb_ext, m) { #endif // Set up the global exception handlers first, so module-specific exception handler can override it: auto exceptions = m.def_submodule("exceptions"); - auto base_exception = - py::register_exception(exceptions, "ArcticException", PyExc_RuntimeError); + auto base_exception = py::register_exception(exceptions, "ArcticException", PyExc_RuntimeError); register_error_code_ecosystem(exceptions, base_exception); - arcticdb::async::register_bindings(m); - arcticdb::codec::register_bindings(m); - arcticdb::column_store::register_bindings(m); + async::register_bindings(m); + codec::register_bindings(m); + column_store::register_bindings(m); auto storage_submodule = m.def_submodule("storage", "Segment storage implementation apis"); - auto no_data_found_exception = py::register_exception( + auto no_data_found_exception = py::register_exception( storage_submodule, "NoDataFoundException", base_exception.ptr() ); - arcticdb::storage::apy::register_bindings(storage_submodule, base_exception); + storage::apy::register_bindings(storage_submodule, base_exception); - arcticdb::stream::register_bindings(m); - arcticdb::toolbox::apy::register_bindings(m, base_exception); - arcticdb::util::register_bindings(m); + stream::register_bindings(m); + toolbox::apy::register_bindings(m, base_exception); + util::register_bindings(m); - m.def("get_version_string", &arcticdb::get_arcticdb_version_string); + m.def("get_version_string", &get_arcticdb_version_string); auto version_submodule = m.def_submodule("version_store", "Versioned storage implementation apis"); - arcticdb::version_store::register_bindings(version_submodule, base_exception); - py::register_exception( + version_store::register_bindings(version_submodule, base_exception); + py::register_exception( version_submodule, "NoSuchVersionException", no_data_found_exception.ptr() ); register_configs_map_api(m); register_log(m.def_submodule("log")); register_instrumentation(m.def_submodule("instrumentation")); - register_metrics(m.def_submodule("metrics")); + register_metrics(m.def_submodule("metrics"), BindingScope::GLOBAL); register_type_handlers(); register_termination_handler(); diff --git a/cpp/arcticdb/storage/common.hpp b/cpp/arcticdb/storage/common.hpp index ad62747fb13..94aab419cd5 100644 --- a/cpp/arcticdb/storage/common.hpp +++ b/cpp/arcticdb/storage/common.hpp @@ -72,6 +72,8 @@ inline std::vector stream_to_vector(std::iostream& src) { return v; } +enum class NativeVariantStorageContentType : uint32_t { EMPTY = 0, S3 = 1, GCPXML = 2 }; + class NativeVariantStorage { public: using VariantStorageConfig = std::variant; @@ -101,6 +103,17 @@ class NativeVariantStorage { return std::get(config_); } + NativeVariantStorageContentType setting_type() const { + if (std::holds_alternative(config_)) { + return NativeVariantStorageContentType::EMPTY; + } else if (std::holds_alternative(config_)) { + return NativeVariantStorageContentType::S3; + } else if (std::holds_alternative(config_)) { + return NativeVariantStorageContentType::GCPXML; + } + util::raise_rte("Unknown variant storage type"); + } + private: VariantStorageConfig config_; }; diff --git a/cpp/arcticdb/storage/mock/s3_mock_client.cpp b/cpp/arcticdb/storage/mock/s3_mock_client.cpp index 82e17902aa0..96452e1ce1a 100644 --- a/cpp/arcticdb/storage/mock/s3_mock_client.cpp +++ b/cpp/arcticdb/storage/mock/s3_mock_client.cpp @@ -30,7 +30,9 @@ std::string MockS3Client::get_failure_trigger( ); } -std::optional has_failure_trigger(const std::string& s3_object_name, StorageOperation operation) { +std::optional has_object_failure_trigger( + const std::string& s3_object_name, StorageOperation operation +) { auto failure_string_for_operation = "#Failure_" + operation_to_string(operation) + "_"; auto position = s3_object_name.rfind(failure_string_for_operation); if (position == std::string::npos) @@ -65,12 +67,12 @@ Aws::S3::S3Error create_error( return error; } -const auto not_found_error = create_error(Aws::S3::S3Errors::RESOURCE_NOT_FOUND); -const auto precondition_failed_error = create_error( +const Aws::S3::S3Error not_found_error = create_error(Aws::S3::S3Errors::RESOURCE_NOT_FOUND); +const Aws::S3::S3Error precondition_failed_error = create_error( Aws::S3::S3Errors::UNKNOWN, "PreconditionFailed", "Precondition failed", false, Aws::Http::HttpResponseCode::PRECONDITION_FAILED ); -const auto not_implemented_error = create_error( +const Aws::S3::S3Error not_implemented_error = create_error( Aws::S3::S3Errors::UNKNOWN, "NotImplemented", "A header you provided implies functionality that is not implemented", false ); @@ -78,7 +80,7 @@ const auto not_implemented_error = create_error( S3Result MockS3Client::head_object(const std::string& s3_object_name, const std::string& bucket_name) const { std::scoped_lock lock(mutex_); - auto maybe_error = has_failure_trigger(s3_object_name, StorageOperation::EXISTS); + auto maybe_error = has_object_failure_trigger(s3_object_name, StorageOperation::EXISTS); if (maybe_error.has_value()) { return {*maybe_error}; } @@ -92,7 +94,7 @@ S3Result MockS3Client::head_object(const std::string& s3_object_ S3Result MockS3Client::get_object(const std::string& s3_object_name, const std::string& bucket_name) const { std::scoped_lock lock(mutex_); - auto maybe_error = has_failure_trigger(s3_object_name, StorageOperation::READ); + auto maybe_error = has_object_failure_trigger(s3_object_name, StorageOperation::READ); if (maybe_error.has_value()) { return {*maybe_error}; } @@ -114,7 +116,7 @@ S3Result MockS3Client::put_object( const std::string& s3_object_name, Segment& segment, const std::string& bucket_name, PutHeader header ) { std::scoped_lock lock(mutex_); - auto maybe_error = has_failure_trigger(s3_object_name, StorageOperation::WRITE); + auto maybe_error = has_object_failure_trigger(s3_object_name, StorageOperation::WRITE); if (maybe_error.has_value() && header == PutHeader::IF_NONE_MATCH) { return {not_implemented_error}; @@ -142,7 +144,7 @@ S3Result MockS3Client::delete_objects( ) { std::scoped_lock lock(mutex_); for (auto& s3_object_name : s3_object_names) { - auto maybe_error = has_failure_trigger(s3_object_name, StorageOperation::DELETE); + auto maybe_error = has_object_failure_trigger(s3_object_name, StorageOperation::DELETE); if (maybe_error.has_value()) { return {*maybe_error}; } @@ -150,7 +152,7 @@ S3Result MockS3Client::delete_objects( DeleteObjectsOutput output; for (auto& s3_object_name : s3_object_names) { - auto maybe_error = has_failure_trigger(s3_object_name, StorageOperation::DELETE_LOCAL); + auto maybe_error = has_object_failure_trigger(s3_object_name, StorageOperation::DELETE_LOCAL); if (maybe_error.has_value()) { output.failed_deletes.emplace_back(s3_object_name, "Sample error message"); } else { @@ -169,10 +171,10 @@ folly::Future> MockS3Client::delete_object( const std::string& s3_object_name, const std::string& bucket_name ) { std::scoped_lock lock(mutex_); - if (auto maybe_error = has_failure_trigger(s3_object_name, StorageOperation::DELETE); maybe_error) { + if (auto maybe_error = has_object_failure_trigger(s3_object_name, StorageOperation::DELETE); maybe_error) { S3Result res{*maybe_error}; return folly::makeFuture(res); - } else if (auto maybe_local_error = has_failure_trigger(s3_object_name, StorageOperation::DELETE_LOCAL); + } else if (auto maybe_local_error = has_object_failure_trigger(s3_object_name, StorageOperation::DELETE_LOCAL); maybe_local_error) { S3Result res{*maybe_local_error}; return folly::makeFuture(res); @@ -211,7 +213,7 @@ S3Result MockS3Client::list_objects( it->second.has_value()) { auto s3_object_name = it->first.s3_object_name; - auto maybe_error = has_failure_trigger(s3_object_name, StorageOperation::LIST); + auto maybe_error = has_object_failure_trigger(s3_object_name, StorageOperation::LIST); if (maybe_error.has_value()) return {*maybe_error}; diff --git a/cpp/arcticdb/storage/mock/s3_mock_client.hpp b/cpp/arcticdb/storage/mock/s3_mock_client.hpp index deacb258377..2796729dc5e 100644 --- a/cpp/arcticdb/storage/mock/s3_mock_client.hpp +++ b/cpp/arcticdb/storage/mock/s3_mock_client.hpp @@ -13,6 +13,14 @@ namespace arcticdb::storage::s3 { +extern const Aws::S3::S3Error not_found_error; +extern const Aws::S3::S3Error precondition_failed_error; +extern const Aws::S3::S3Error not_implemented_error; + +std::optional has_object_failure_trigger( + const std::string& s3_object_name, StorageOperation operation +); + struct S3Key { std::string bucket_name; std::string s3_object_name; diff --git a/cpp/arcticdb/storage/python_bindings.cpp b/cpp/arcticdb/storage/python_bindings.cpp index ca242352c54..40d80bfcc0c 100644 --- a/cpp/arcticdb/storage/python_bindings.cpp +++ b/cpp/arcticdb/storage/python_bindings.cpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace py = pybind11; @@ -33,77 +34,6 @@ std::shared_ptr create_library_index(const std::string& environmen return std::make_shared(EnvironmentName{environment_name}, mem_resolver); } -enum class S3SettingsPickleOrder : uint32_t { - TYPE = 0, - AWS_AUTH = 1, - AWS_PROFILE = 2, - USE_INTERNAL_CLIENT_WRAPPER_FOR_TESTING = 3 -}; - -enum class GCPXMLSettingsPickleOrder : uint32_t { - TYPE = 0, - AWS_AUTH = 1, - CA_CERT_PATH = 2, - CA_CERT_DIR = 3, - SSL = 4, - HTTPS = 5, - PREFIX = 6, - ENDPOINT = 7, - SECRET = 8, - ACCESS = 9, - BUCKET = 10, -}; - -s3::GCPXMLSettings gcp_settings(const py::tuple& t) { - util::check(t.size() == 11, "Invalid GCPXMLSettings pickle objects, expected 11 attributes but was {}", t.size()); - return s3::GCPXMLSettings{ - t[static_cast(GCPXMLSettingsPickleOrder::AWS_AUTH)].cast(), - t[static_cast(GCPXMLSettingsPickleOrder::CA_CERT_PATH)].cast(), - t[static_cast(GCPXMLSettingsPickleOrder::CA_CERT_DIR)].cast(), - t[static_cast(GCPXMLSettingsPickleOrder::SSL)].cast(), - t[static_cast(GCPXMLSettingsPickleOrder::HTTPS)].cast(), - t[static_cast(GCPXMLSettingsPickleOrder::PREFIX)].cast(), - t[static_cast(GCPXMLSettingsPickleOrder::ENDPOINT)].cast(), - t[static_cast(GCPXMLSettingsPickleOrder::SECRET)].cast(), - t[static_cast(GCPXMLSettingsPickleOrder::ACCESS)].cast(), - t[static_cast(GCPXMLSettingsPickleOrder::BUCKET)].cast() - }; -} - -s3::S3Settings s3_settings(const py::tuple& t) { - util::check(t.size() == 4, "Invalid S3Settings pickle objects"); - return s3::S3Settings{ - t[static_cast(S3SettingsPickleOrder::AWS_AUTH)].cast(), - t[static_cast(S3SettingsPickleOrder::AWS_PROFILE)].cast(), - t[static_cast(S3SettingsPickleOrder::USE_INTERNAL_CLIENT_WRAPPER_FOR_TESTING)].cast() - }; -} - -py::tuple to_tuple(const s3::GCPXMLSettings& settings) { - return py::make_tuple( - s3::NativeSettingsType::GCPXML, - settings.aws_auth(), - settings.ca_cert_path(), - settings.ca_cert_dir(), - settings.ssl(), - settings.https(), - settings.prefix(), - settings.endpoint(), - settings.secret(), - settings.access(), - settings.bucket() - ); -} - -py::tuple to_tuple(const s3::S3Settings& settings) { - return py::make_tuple( - s3::NativeSettingsType::S3, - settings.aws_auth(), - settings.aws_profile(), - settings.use_internal_client_wrapper_for_testing() - ); -} - void register_bindings(py::module& storage, py::exception& base_exception) { storage.attr("CONFIG_LIBRARY_NAME") = py::str(arcticdb::storage::CONFIG_LIBRARY_NAME); @@ -134,11 +64,6 @@ void register_bindings(py::module& storage, py::exception(storage, "OpenMode") - .value("READ", OpenMode::READ) - .value("WRITE", OpenMode::WRITE) - .value("DELETE", OpenMode::DELETE); - py::enum_(storage, "ModifiableLibraryOption", R"pbdoc( Library options that can be modified after library creation. @@ -168,109 +93,7 @@ void register_bindings(py::module& storage, py::exception(storage, "AWSAuthMethod") - .value("DISABLED", s3::AWSAuthMethod::DISABLED) - .value("DEFAULT_CREDENTIALS_PROVIDER_CHAIN", s3::AWSAuthMethod::DEFAULT_CREDENTIALS_PROVIDER_CHAIN) - .value("STS_PROFILE_CREDENTIALS_PROVIDER", s3::AWSAuthMethod::STS_PROFILE_CREDENTIALS_PROVIDER); - - py::enum_(storage, "NativeSettingsType") - .value("S3", s3::NativeSettingsType::S3) - .value("GCPXML", s3::NativeSettingsType::GCPXML); - - py::class_(storage, "S3Settings") - .def(py::init()) - .def(py::pickle( - [](const s3::S3Settings& settings) { return to_tuple(settings); }, - [](py::tuple t) { return s3_settings(t); } - )) - .def_property_readonly("aws_profile", [](const s3::S3Settings& settings) { return settings.aws_profile(); }) - .def_property_readonly("aws_auth", [](const s3::S3Settings& settings) { return settings.aws_auth(); }) - .def_property_readonly("use_internal_client_wrapper_for_testing", [](const s3::S3Settings& settings) { - return settings.use_internal_client_wrapper_for_testing(); - }); - - py::class_(storage, "GCPXMLSettings") - .def(py::init<>()) - .def(py::pickle( - [](const s3::GCPXMLSettings& settings) { return to_tuple(settings); }, - [](py::tuple t) { return gcp_settings(t); } - )) - .def_property("bucket", &s3::GCPXMLSettings::bucket, &s3::GCPXMLSettings::set_bucket) - .def_property("endpoint", &s3::GCPXMLSettings::endpoint, &s3::GCPXMLSettings::set_endpoint) - .def_property("access", &s3::GCPXMLSettings::access, &s3::GCPXMLSettings::set_access) - .def_property("secret", &s3::GCPXMLSettings::secret, &s3::GCPXMLSettings::set_secret) - .def_property("prefix", &s3::GCPXMLSettings::prefix, &s3::GCPXMLSettings::set_prefix) - .def_property("aws_auth", &s3::GCPXMLSettings::aws_auth, &s3::GCPXMLSettings::set_aws_auth) - .def_property("https", &s3::GCPXMLSettings::https, &s3::GCPXMLSettings::set_https) - .def_property("ssl", &s3::GCPXMLSettings::ssl, &s3::GCPXMLSettings::set_ssl) - .def_property("ca_cert_path", &s3::GCPXMLSettings::ca_cert_path, &s3::GCPXMLSettings::set_cert_path) - .def_property("ca_cert_dir", &s3::GCPXMLSettings::ca_cert_dir, &s3::GCPXMLSettings::set_cert_dir); - - py::class_(storage, "NativeVariantStorage") - .def(py::init<>()) - .def(py::init()) - .def(py::pickle( - [](const NativeVariantStorage& settings) { - return util::variant_match( - settings.variant(), - [](const s3::S3Settings& settings) { return to_tuple(settings); }, - [](const s3::GCPXMLSettings& settings) { return to_tuple(settings); }, - [](const auto&) -> py::tuple { util::raise_rte("Invalid native storage setting type"); } - ); - }, - [](py::tuple t) { - util::check(t.size() >= 1, "Expected at least one attribute in Native Settings pickle"); - auto type = - t[static_cast(S3SettingsPickleOrder::TYPE)].cast(); - switch (type) { - case s3::NativeSettingsType::S3: - return NativeVariantStorage(s3_settings(t)); - case s3::NativeSettingsType::GCPXML: - return NativeVariantStorage(gcp_settings(t)); - } - util::raise_rte("Inaccessible"); - } - )) - .def("update", &NativeVariantStorage::update) - .def("as_s3_settings", &NativeVariantStorage::as_s3_settings) - .def("as_gcpxml_settings", &NativeVariantStorage::as_gcpxml_settings) - .def("__repr__", &NativeVariantStorage::to_string); - - py::implicitly_convertible(); - - storage.def( - "create_mem_config_resolver", - [](const py::object& env_config_map_py) -> std::shared_ptr { - arcticdb::proto::storage::EnvironmentConfigsMap ecm; - pb_from_python(env_config_map_py, ecm); - auto resolver = std::make_shared(); - for (auto& [env, cfg] : ecm.env_by_id()) { - EnvironmentName env_name{env}; - for (auto& [id, variant_storage] : cfg.storage_by_id()) { - resolver->add_storage(env_name, StorageName{id}, variant_storage); - } - for (auto& [id, lib_desc] : cfg.lib_by_path()) { - resolver->add_library(env_name, lib_desc); - } - } - return resolver; - } - ); - - py::class_>(storage, "ConfigResolver"); - - py::class_>(storage, "Library") - .def_property_readonly( - "library_path", [](const Library& library) { return library.library_path().to_delim_path(); } - ) - .def_property_readonly("open_mode", [](const Library& library) { return library.open_mode(); }) - .def_property_readonly("config", [](const Library& library) { - return util::variant_match( - library.config(), - [](const arcticdb::proto::storage::VersionStoreConfig& cfg) { return pb_to_python(cfg); }, - [](const std::monostate&) -> py::object { return py::none{}; } - ); - }); + register_common_storage_bindings(storage, BindingScope::GLOBAL); py::class_(storage, "S3Override") .def(py::init<>()) @@ -409,34 +232,6 @@ void register_bindings(py::module& storage, py::exception>(storage, "LibraryIndex") - .def(py::init<>([](const std::string& environment_name) { - auto resolver = std::make_shared(); - return std::make_unique(EnvironmentName{environment_name}, resolver); - })) - .def_static( - "create_from_resolver", - [](const std::string& environment_name, std::shared_ptr resolver) { - return std::make_shared(EnvironmentName{environment_name}, resolver); - } - ) - .def("list_libraries", - [](LibraryIndex& library_index, std::string_view prefix = "") { - std::vector res; - for (const auto& lp : library_index.list_libraries(prefix)) { - res.emplace_back(lp.to_delim_path()); - } - return res; - }) - .def("get_library", - [](LibraryIndex& library_index, - const std::string& library_path, - OpenMode open_mode = OpenMode::DELETE, - const NativeVariantStorage& native_storage_config = NativeVariantStorage()) { - LibraryPath path = LibraryPath::from_delim_path(library_path); - return library_index.get_library(path, open_mode, UserAuth{}, native_storage_config); - }); } } // namespace arcticdb::storage::apy diff --git a/cpp/arcticdb/storage/python_bindings_common.cpp b/cpp/arcticdb/storage/python_bindings_common.cpp new file mode 100644 index 00000000000..22670cd5b4c --- /dev/null +++ b/cpp/arcticdb/storage/python_bindings_common.cpp @@ -0,0 +1,240 @@ +/* Copyright 2025 Man Group Operations Limited + * + * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software + * will be governed by the Apache License, version 2.0. + */ + +#include +#include +#include +#include +#include + +namespace py = pybind11; + +namespace arcticdb::storage::apy { +using namespace python_util; + +s3::GCPXMLSettings gcp_settings(const py::tuple& t) { + static size_t py_object_size = 11; + util::check( + t.size() >= py_object_size, + "Invalid GCPXMLSettings pickle objects, expected at least {} attributes but was {}", + py_object_size, + t.size() + ); + util::warn( + t.size() > py_object_size, + "GCPXMLSettings py tuple expects {} attributes but has {}. Will continue by ignoring extra attributes.", + py_object_size, + t.size() + ); + return s3::GCPXMLSettings{ + t[static_cast(GCPXMLSettingsPickleOrder::AWS_AUTH)].cast(), + t[static_cast(GCPXMLSettingsPickleOrder::CA_CERT_PATH)].cast(), + t[static_cast(GCPXMLSettingsPickleOrder::CA_CERT_DIR)].cast(), + t[static_cast(GCPXMLSettingsPickleOrder::SSL)].cast(), + t[static_cast(GCPXMLSettingsPickleOrder::HTTPS)].cast(), + t[static_cast(GCPXMLSettingsPickleOrder::PREFIX)].cast(), + t[static_cast(GCPXMLSettingsPickleOrder::ENDPOINT)].cast(), + t[static_cast(GCPXMLSettingsPickleOrder::SECRET)].cast(), + t[static_cast(GCPXMLSettingsPickleOrder::ACCESS)].cast(), + t[static_cast(GCPXMLSettingsPickleOrder::BUCKET)].cast() + }; +} + +s3::S3Settings s3_settings(const py::tuple& t) { + static size_t py_object_size = 4; + util::check(t.size() >= py_object_size, "Invalid S3Settings pickle objects"); + util::warn( + t.size() > py_object_size, + "S3Settings py tuple expects {} attributes but has {}. Will continue by ignoring extra attributes.", + py_object_size, + t.size() + ); + return s3::S3Settings{ + t[static_cast(S3SettingsPickleOrder::AWS_AUTH)].cast(), + t[static_cast(S3SettingsPickleOrder::AWS_PROFILE)].cast(), + t[static_cast(S3SettingsPickleOrder::USE_INTERNAL_CLIENT_WRAPPER_FOR_TESTING)].cast() + }; +} + +py::tuple to_tuple(const s3::GCPXMLSettings& settings) { + return py::make_tuple( + s3::NativeSettingsType::GCPXML, + settings.aws_auth(), + settings.ca_cert_path(), + settings.ca_cert_dir(), + settings.ssl(), + settings.https(), + settings.prefix(), + settings.endpoint(), + settings.secret(), + settings.access(), + settings.bucket() + ); +} + +py::tuple to_tuple(const s3::S3Settings& settings) { + return py::make_tuple( + s3::NativeSettingsType::S3, + settings.aws_auth(), + settings.aws_profile(), + settings.use_internal_client_wrapper_for_testing() + ); +} + +void register_common_storage_bindings(py::module& storage, BindingScope scope) { + bool local_bindings = (scope == BindingScope::LOCAL); + py::enum_( + storage, "NativeVariantStorageContentType", py::module_local(local_bindings) + ) + .value("EMPTY", NativeVariantStorageContentType::EMPTY) + .value("S3", NativeVariantStorageContentType::S3) + .value("GCPXML", NativeVariantStorageContentType::GCPXML); + + py::enum_(storage, "AWSAuthMethod", py::module_local(local_bindings)) + .value("DISABLED", s3::AWSAuthMethod::DISABLED) + .value("DEFAULT_CREDENTIALS_PROVIDER_CHAIN", s3::AWSAuthMethod::DEFAULT_CREDENTIALS_PROVIDER_CHAIN) + .value("STS_PROFILE_CREDENTIALS_PROVIDER", s3::AWSAuthMethod::STS_PROFILE_CREDENTIALS_PROVIDER); + + py::enum_(storage, "NativeSettingsType", py::module_local(local_bindings)) + .value("S3", s3::NativeSettingsType::S3) + .value("GCPXML", s3::NativeSettingsType::GCPXML); + + py::class_(storage, "S3Settings", py::module_local(local_bindings)) + .def(py::init(), + py::arg("aws_auth"), + py::arg("aws_profile"), + py::arg("use_internal_client_wrapper_for_testing")) + .def(py::pickle( + [](const s3::S3Settings& settings) { return to_tuple(settings); }, + [](py::tuple t) { return s3_settings(t); } + )) + .def_property_readonly("aws_profile", [](const s3::S3Settings& settings) { return settings.aws_profile(); }) + .def_property_readonly("aws_auth", [](const s3::S3Settings& settings) { return settings.aws_auth(); }) + .def_property_readonly("use_internal_client_wrapper_for_testing", [](const s3::S3Settings& settings) { + return settings.use_internal_client_wrapper_for_testing(); + }); + + py::class_(storage, "GCPXMLSettings", py::module_local(local_bindings)) + .def(py::init<>()) + .def(py::pickle( + [](const s3::GCPXMLSettings& settings) { return to_tuple(settings); }, + [](py::tuple t) { return gcp_settings(t); } + )) + .def_property("bucket", &s3::GCPXMLSettings::bucket, &s3::GCPXMLSettings::set_bucket) + .def_property("endpoint", &s3::GCPXMLSettings::endpoint, &s3::GCPXMLSettings::set_endpoint) + .def_property("access", &s3::GCPXMLSettings::access, &s3::GCPXMLSettings::set_access) + .def_property("secret", &s3::GCPXMLSettings::secret, &s3::GCPXMLSettings::set_secret) + .def_property("prefix", &s3::GCPXMLSettings::prefix, &s3::GCPXMLSettings::set_prefix) + .def_property("aws_auth", &s3::GCPXMLSettings::aws_auth, &s3::GCPXMLSettings::set_aws_auth) + .def_property("https", &s3::GCPXMLSettings::https, &s3::GCPXMLSettings::set_https) + .def_property("ssl", &s3::GCPXMLSettings::ssl, &s3::GCPXMLSettings::set_ssl) + .def_property("ca_cert_path", &s3::GCPXMLSettings::ca_cert_path, &s3::GCPXMLSettings::set_cert_path) + .def_property("ca_cert_dir", &s3::GCPXMLSettings::ca_cert_dir, &s3::GCPXMLSettings::set_cert_dir); + + py::class_(storage, "NativeVariantStorage", py::module_local(local_bindings)) + .def(py::init<>()) + .def(py::init()) + .def(py::pickle( + [](const NativeVariantStorage& settings) { + return util::variant_match( + settings.variant(), + [](const s3::S3Settings& settings) { return to_tuple(settings); }, + [](const s3::GCPXMLSettings& settings) { return to_tuple(settings); }, + [](const auto&) -> py::tuple { util::raise_rte("Invalid native storage setting type"); } + ); + }, + [](py::tuple t) { + util::check(t.size() >= 1, "Expected at least one attribute in Native Settings pickle"); + auto type = + t[static_cast(S3SettingsPickleOrder::TYPE)].cast(); + switch (type) { + case s3::NativeSettingsType::S3: + return NativeVariantStorage(s3_settings(t)); + case s3::NativeSettingsType::GCPXML: + return NativeVariantStorage(gcp_settings(t)); + } + util::raise_rte("Inaccessible"); + } + )) + .def("update", &NativeVariantStorage::update) + .def("as_s3_settings", &NativeVariantStorage::as_s3_settings) + .def("as_gcpxml_settings", &NativeVariantStorage::as_gcpxml_settings) + .def("__repr__", &NativeVariantStorage::to_string) + .def_property_readonly("setting_type", &NativeVariantStorage::setting_type); + + py::implicitly_convertible(); + + storage.def( + "create_mem_config_resolver", + [](const py::object& env_config_map_py) -> std::shared_ptr { + arcticdb::proto::storage::EnvironmentConfigsMap ecm; + pb_from_python(env_config_map_py, ecm); + auto resolver = std::make_shared(); + for (auto& [env, cfg] : ecm.env_by_id()) { + EnvironmentName env_name{env}; + for (auto& [id, variant_storage] : cfg.storage_by_id()) { + resolver->add_storage(env_name, StorageName{id}, variant_storage); + } + for (auto& [id, lib_desc] : cfg.lib_by_path()) { + resolver->add_library(env_name, lib_desc); + } + } + return resolver; + } + ); + py::class_>( + storage, "ConfigResolver", py::module_local(local_bindings) + ); + + py::class_>(storage, "LibraryIndex", py::module_local(local_bindings)) + .def(py::init<>([](const std::string& environment_name) { + auto resolver = std::make_shared(); + return std::make_unique(EnvironmentName{environment_name}, resolver); + })) + .def_static( + "create_from_resolver", + [](const std::string& environment_name, std::shared_ptr resolver) { + return std::make_shared(EnvironmentName{environment_name}, resolver); + } + ) + .def("list_libraries", + [](LibraryIndex& library_index, std::string_view prefix = "") { + std::vector res; + for (const auto& lp : library_index.list_libraries(prefix)) { + res.emplace_back(lp.to_delim_path()); + } + return res; + }) + .def("get_library", + [](LibraryIndex& library_index, + const std::string& library_path, + OpenMode open_mode = OpenMode::DELETE, + const NativeVariantStorage& native_storage_config = NativeVariantStorage()) { + LibraryPath path = LibraryPath::from_delim_path(library_path); + return library_index.get_library(path, open_mode, UserAuth{}, native_storage_config); + }); + + py::class_>(storage, "Library", py::module_local(local_bindings)) + .def_property_readonly( + "library_path", [](const Library& library) { return library.library_path().to_delim_path(); } + ) + .def_property_readonly("open_mode", [](const Library& library) { return library.open_mode(); }) + .def_property_readonly("config", [](const Library& library) { + return util::variant_match( + library.config(), + [](const arcticdb::proto::storage::VersionStoreConfig& cfg) { return pb_to_python(cfg); }, + [](const std::monostate&) -> py::object { return py::none{}; } + ); + }); + + py::enum_(storage, "OpenMode", py::module_local(local_bindings)) + .value("READ", OpenMode::READ) + .value("WRITE", OpenMode::WRITE) + .value("DELETE", OpenMode::DELETE); +} +} // namespace arcticdb::storage::apy diff --git a/cpp/arcticdb/storage/python_bindings_common.hpp b/cpp/arcticdb/storage/python_bindings_common.hpp new file mode 100644 index 00000000000..a8bebc9289b --- /dev/null +++ b/cpp/arcticdb/storage/python_bindings_common.hpp @@ -0,0 +1,51 @@ +/* Copyright 2025 Man Group Operations Limited + * + * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software + * will be governed by the Apache License, version 2.0. + */ + +#pragma once + +#include +#include +#include + +namespace arcticdb::storage { + +namespace s3 { +class S3Settings; +class GCPXMLSettings; +}; // namespace s3 + +namespace apy { + +enum class S3SettingsPickleOrder : uint32_t { + TYPE = 0, + AWS_AUTH = 1, + AWS_PROFILE = 2, + USE_INTERNAL_CLIENT_WRAPPER_FOR_TESTING = 3 +}; + +enum class GCPXMLSettingsPickleOrder : uint32_t { + TYPE = 0, + AWS_AUTH = 1, + CA_CERT_PATH = 2, + CA_CERT_DIR = 3, + SSL = 4, + HTTPS = 5, + PREFIX = 6, + ENDPOINT = 7, + SECRET = 8, + ACCESS = 9, + BUCKET = 10, +}; + +s3::GCPXMLSettings gcp_settings(const pybind11::tuple& t); +s3::S3Settings s3_settings(const pybind11::tuple& t); +pybind11::tuple to_tuple(const s3::GCPXMLSettings& settings); +pybind11::tuple to_tuple(const s3::S3Settings& settings); +void register_common_storage_bindings(pybind11::module& m, BindingScope scope); +} // namespace apy +} // namespace arcticdb::storage \ No newline at end of file diff --git a/cpp/arcticdb/storage/s3/s3_client_wrapper.cpp b/cpp/arcticdb/storage/s3/s3_client_wrapper.cpp index d2d90a246eb..7edc0f31f8a 100644 --- a/cpp/arcticdb/storage/s3/s3_client_wrapper.cpp +++ b/cpp/arcticdb/storage/s3/s3_client_wrapper.cpp @@ -8,6 +8,7 @@ #include #include +#include #include @@ -17,17 +18,24 @@ using namespace object_store_utils; namespace s3 { -std::optional S3ClientTestWrapper::has_failure_trigger(const std::string& bucket_name) const { - bool static_failures_enabled = ConfigsMap::instance()->get_int("S3ClientTestWrapper.EnableFailures", 0) == 1; +std::optional S3ClientTestWrapper::has_failure_trigger( + const std::string& s3_object_name, const std::string& bucket_name, StorageOperation operation +) const { + if (auto error = has_bucket_failure_trigger(bucket_name)) { + return error; + } + return has_object_failure_trigger(s3_object_name, operation); +} + +std::optional S3ClientTestWrapper::has_bucket_failure_trigger(const std::string& bucket_name) const { // Check if mock failures are enabled - if (!static_failures_enabled) { + if (ConfigsMap::instance()->get_int("S3ClientTestWrapper.EnableFailures", 0) != 1) { return std::nullopt; } // Get target buckets (if not set or "all", affects all buckets) - auto failure_buckets_str = ConfigsMap::instance()->get_string("S3ClientTestWrapper.FailureBucket", "all"); - - if (failure_buckets_str != "all") { + if (auto failure_buckets_str = ConfigsMap::instance()->get_string("S3ClientTestWrapper.FailureBucket", "all"); + failure_buckets_str != "all") { // Split the comma-separated bucket names and check if current bucket is in the list std::istringstream bucket_stream(failure_buckets_str); std::string target_bucket; @@ -68,8 +76,7 @@ std::optional S3ClientTestWrapper::has_failure_trigger(const s S3Result S3ClientTestWrapper::head_object( const std::string& s3_object_name, const std::string& bucket_name ) const { - auto maybe_error = has_failure_trigger(bucket_name); - if (maybe_error.has_value()) { + if (auto maybe_error = has_failure_trigger(s3_object_name, bucket_name, StorageOperation::EXISTS)) { return {*maybe_error}; } @@ -78,8 +85,7 @@ S3Result S3ClientTestWrapper::head_object( S3Result S3ClientTestWrapper::get_object(const std::string& s3_object_name, const std::string& bucket_name) const { - auto maybe_error = has_failure_trigger(bucket_name); - if (maybe_error.has_value()) { + if (auto maybe_error = has_failure_trigger(s3_object_name, bucket_name, StorageOperation::READ)) { return {*maybe_error}; } @@ -89,8 +95,7 @@ S3Result S3ClientTestWrapper::get_object(const std::string& s3_object_n folly::Future> S3ClientTestWrapper::get_object_async( const std::string& s3_object_name, const std::string& bucket_name ) const { - auto maybe_error = has_failure_trigger(bucket_name); - if (maybe_error.has_value()) { + if (auto maybe_error = has_failure_trigger(s3_object_name, bucket_name, StorageOperation::READ)) { return folly::makeFuture>({*maybe_error}); } @@ -100,9 +105,20 @@ folly::Future> S3ClientTestWrapper::get_object_async( S3Result S3ClientTestWrapper::put_object( const std::string& s3_object_name, Segment& segment, const std::string& bucket_name, PutHeader header ) { - auto maybe_error = has_failure_trigger(bucket_name); - if (maybe_error.has_value()) { - return {*maybe_error}; + if (auto maybe_bucket_error = has_bucket_failure_trigger(bucket_name)) { + return {*maybe_bucket_error}; + } + if (auto maybe_object_error = has_object_failure_trigger(s3_object_name, StorageOperation::WRITE)) { + if (header == PutHeader::IF_NONE_MATCH) { + return {not_implemented_error}; + } + return {*maybe_object_error}; + } + + if (header == PutHeader::IF_NONE_MATCH) { + if (auto head_result = actual_client_->head_object(s3_object_name, bucket_name); head_result.is_success()) { + return {precondition_failed_error}; + } } return actual_client_->put_object(s3_object_name, segment, bucket_name, header); @@ -111,22 +127,36 @@ S3Result S3ClientTestWrapper::put_object( S3Result S3ClientTestWrapper::delete_objects( const std::vector& s3_object_names, const std::string& bucket_name ) { - auto maybe_error = has_failure_trigger(bucket_name); - if (maybe_error.has_value()) { - return {*maybe_error}; + if (auto maybe_bucket_error = has_bucket_failure_trigger(bucket_name)) { + return {*maybe_bucket_error}; + } + + for (const auto& s3_object_name : s3_object_names) { + if (auto maybe_object_error = has_object_failure_trigger(s3_object_name, StorageOperation::DELETE)) { + return {*maybe_object_error}; + } } - return actual_client_->delete_objects(s3_object_names, bucket_name); + auto result = actual_client_->delete_objects(s3_object_names, bucket_name); + if (result.is_success()) { + for (const auto& s3_object_name : s3_object_names) { + if (has_object_failure_trigger(s3_object_name, StorageOperation::DELETE_LOCAL)) { + result.get_output().failed_deletes.emplace_back(s3_object_name, "Simulated local delete failure"); + } + } + } + + return result; } folly::Future> S3ClientTestWrapper::delete_object( const std::string& s3_object_name, const std::string& bucket_name ) { - auto maybe_error = has_failure_trigger(bucket_name); - if (maybe_error.has_value()) { - return folly::makeFuture>({*maybe_error}); + if (auto maybe_error = has_failure_trigger(s3_object_name, bucket_name, StorageOperation::DELETE)) { + return folly::makeFuture(S3Result{*maybe_error}); + } else if (auto maybe_object_error = has_object_failure_trigger(s3_object_name, StorageOperation::DELETE_LOCAL)) { + return folly::makeFuture(S3Result{*maybe_object_error}); } - return actual_client_->delete_object(s3_object_name, bucket_name); } @@ -134,12 +164,21 @@ S3Result S3ClientTestWrapper::list_objects( const std::string& name_prefix, const std::string& bucket_name, const std::optional& continuation_token ) const { - auto maybe_error = has_failure_trigger(bucket_name); - if (maybe_error.has_value()) { - return {*maybe_error}; + if (auto maybe_bucket_error = has_bucket_failure_trigger(bucket_name); maybe_bucket_error.has_value()) { + return {*maybe_bucket_error}; + } + + auto result = actual_client_->list_objects(name_prefix, bucket_name, continuation_token); + + if (result.is_success()) { + for (const auto& s3_object_name : result.get_output().s3_object_names) { + if (auto maybe_object_error = has_object_failure_trigger(s3_object_name, StorageOperation::LIST)) { + return {*maybe_object_error}; + } + } } - return actual_client_->list_objects(name_prefix, bucket_name, continuation_token); + return result; } } // namespace s3 diff --git a/cpp/arcticdb/storage/s3/s3_client_wrapper.hpp b/cpp/arcticdb/storage/s3/s3_client_wrapper.hpp index 010396c190a..9d71e945d02 100644 --- a/cpp/arcticdb/storage/s3/s3_client_wrapper.hpp +++ b/cpp/arcticdb/storage/s3/s3_client_wrapper.hpp @@ -53,7 +53,10 @@ class S3ClientTestWrapper : public S3ClientInterface { private: // Returns error if failures are enabled for the given bucket - std::optional has_failure_trigger(const std::string& bucket_name) const; + std::optional has_bucket_failure_trigger(const std::string& bucket_name) const; + std::optional has_failure_trigger( + const std::string& s3_object_name, const std::string& bucket_name, StorageOperation operation + ) const; std::unique_ptr actual_client_; }; diff --git a/cpp/arcticdb/version/python_bindings.cpp b/cpp/arcticdb/version/python_bindings.cpp index 69ed15b8440..595d7dcafbf 100644 --- a/cpp/arcticdb/version/python_bindings.cpp +++ b/cpp/arcticdb/version/python_bindings.cpp @@ -25,6 +25,7 @@ #include #include #include +#include namespace arcticdb::version_store { @@ -144,51 +145,7 @@ void register_bindings(py::module& version, py::exception(version, "StreamDescriptorMismatch", base_exception.ptr()); - py::class_>(version, "AtomKey") - .def(py::init()) - .def(py::init()) - .def("change_id", &AtomKey::change_id) - .def_property_readonly("id", &AtomKey::id) - .def_property_readonly("version_id", &AtomKey::version_id) - .def_property_readonly("creation_ts", &AtomKey::creation_ts) - .def_property_readonly("content_hash", &AtomKey::content_hash) - .def_property_readonly("start_index", &AtomKey::start_index) - .def_property_readonly("end_index", &AtomKey::end_index) - .def_property_readonly("type", [](const AtomKey& self) { return self.type(); }) - .def(pybind11::self == pybind11::self) - .def(pybind11::self != pybind11::self) - .def("__repr__", &AtomKey::view_human) - .def(py::self < py::self) - .def(py::pickle( - [](const AtomKey& key) { - constexpr int serialization_version = 0; - return py::make_tuple( - serialization_version, - key.id(), - key.version_id(), - key.creation_ts(), - key.content_hash(), - key.start_index(), - key.end_index(), - key.type() - ); - }, - [](py::tuple t) { - util::check(t.size() >= 7, "Invalid AtomKey pickle object!"); - - [[maybe_unused]] const int serialization_version = t[0].cast(); - AtomKey key( - t[1].cast(), - t[2].cast(), - t[3].cast(), - t[4].cast(), - t[5].cast(), - t[6].cast(), - t[7].cast() - ); - return key; - } - )); + entity::apy::register_common_entity_bindings(version, arcticdb::BindingScope::GLOBAL); py::class_>(version, "RefKey") .def(py::init()) @@ -368,12 +325,6 @@ void register_bindings(py::module& version, py::exception(version, "VersionedItem") - .def_property_readonly("symbol", &VersionedItem::symbol) - .def_property_readonly("timestamp", &VersionedItem::timestamp) - .def_property_readonly("version", &VersionedItem::version); - py::class_(version, "DescriptorItem") .def_property_readonly("symbol", &DescriptorItem::symbol) .def_property_readonly("version", &DescriptorItem::version) diff --git a/python/arcticdb/_core_api.py b/python/arcticdb/_core_api.py new file mode 100644 index 00000000000..7039c6924aa --- /dev/null +++ b/python/arcticdb/_core_api.py @@ -0,0 +1,15 @@ +""" +Copyright 2023 Man Group Operations Ltd. +NO WARRANTY, EXPRESSED OR IMPLIED. + +Non-public APIs depended by downstream repos +Treat everything here as public APIs!!!! +Any breaking change made here may break downstream repos and requires bumping major version!!!! +""" + +from arcticdb_ext.storage import NativeVariantStorage, NativeVariantStorageContentType, NoDataFoundException, KeyType +from arcticdb_ext.metrics.prometheus import MetricsConfig +from arcticdb_ext.version_store import AtomKey, RefKey +from arcticdb_ext.exceptions import StorageException, ArcticException +from arcticdb_ext import set_config_int +from arcticdb.version_store._store import _env_config_from_lib_config as env_config_from_lib_config diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index 18d4af8444c..ad33423dd82 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -41,7 +41,12 @@ ) from arcticc.pb2.storage_pb2 import EnvironmentConfigsMap from arcticdb.version_store.helper import add_gcp_library_to_env, add_s3_library_to_env -from arcticdb_ext.storage import AWSAuthMethod, NativeVariantStorage, GCPXMLSettings as NativeGCPXMLSettings +from arcticdb_ext.storage import ( + AWSAuthMethod, + NativeVariantStorage, + GCPXMLSettings as NativeGCPXMLSettings, + S3Settings as NativeS3Settings, +) from arcticdb_ext.tools import S3Tool # All storage client libraries to be imported on-demand to speed up start-up of ad-hoc test runs @@ -525,9 +530,19 @@ def real_s3_sts_from_environment_variables( logger.error(f"Error creating access key: {e}") raise e + out.native_config = NativeVariantStorage( + NativeS3Settings( + aws_auth=AWSAuthMethod.STS_PROFILE_CREDENTIALS_PROVIDER, + aws_profile=profile_name, + use_internal_client_wrapper_for_testing=False, + ) + ) out.aws_auth = AWSAuthMethod.STS_PROFILE_CREDENTIALS_PROVIDER out.aws_profile = profile_name real_s3_sts_write_local_credentials(out, config_file_path) + out.default_key = Key( + id="", secret="", user_name="unknown user" + ) # Reset to ensure client can't fallback to default key+secret auth return out diff --git a/python/tests/compat/arcticdb/test_core_api.py b/python/tests/compat/arcticdb/test_core_api.py new file mode 100644 index 00000000000..f382019f1c7 --- /dev/null +++ b/python/tests/compat/arcticdb/test_core_api.py @@ -0,0 +1,121 @@ +from arcticdb._core_api import ( + NativeVariantStorage, + MetricsConfig, + env_config_from_lib_config, + NativeVariantStorageContentType, +) +from arcticdb_ext.storage import AWSAuthMethod, S3Settings as NativeS3Settings, GCPXMLSettings as NativeGCPXMLSettings +from arcticdb_ext.metrics.prometheus import MetricsConfigModel +from arcticdb_ext.storage import create_mem_config_resolver, LibraryIndex, OpenMode + + +def test_native_variant_storage_s3_accessors(): + s3_settings = NativeS3Settings( + aws_auth=AWSAuthMethod.DEFAULT_CREDENTIALS_PROVIDER_CHAIN, + aws_profile="abc", + use_internal_client_wrapper_for_testing=True, + ) + native_setting = NativeVariantStorage(s3_settings) + + assert native_setting.setting_type == NativeVariantStorageContentType.S3 + + retrieved_s3 = native_setting.as_s3_settings() + assert retrieved_s3.aws_auth == s3_settings.aws_auth + assert retrieved_s3.aws_profile == s3_settings.aws_profile + assert retrieved_s3.use_internal_client_wrapper_for_testing == s3_settings.use_internal_client_wrapper_for_testing + + +def test_native_variant_storage_gcp_accessors(): + gcp_settings = NativeGCPXMLSettings() + gcp_settings.bucket = "bucket" + gcp_settings.endpoint = "endpoint" + gcp_settings.access = "access" + gcp_settings.secret = "secret" + gcp_settings.aws_auth = AWSAuthMethod.DEFAULT_CREDENTIALS_PROVIDER_CHAIN + gcp_settings.prefix = "abc" + gcp_settings.https = True + gcp_settings.ssl = False + gcp_settings.ca_cert_path = "ca_cert_path" + gcp_settings.ca_cert_dir = "ca_cert_dir" + native_setting = NativeVariantStorage(gcp_settings) + + assert native_setting.setting_type == NativeVariantStorageContentType.GCPXML + + retrieved_gcp = native_setting.as_gcpxml_settings() + assert retrieved_gcp.aws_auth == gcp_settings.aws_auth + assert retrieved_gcp.ca_cert_path == gcp_settings.ca_cert_path + assert retrieved_gcp.ca_cert_dir == gcp_settings.ca_cert_dir + assert retrieved_gcp.ssl == gcp_settings.ssl + assert retrieved_gcp.https == gcp_settings.https + assert retrieved_gcp.prefix == gcp_settings.prefix + assert retrieved_gcp.endpoint == gcp_settings.endpoint + assert retrieved_gcp.secret == gcp_settings.secret + assert retrieved_gcp.access == gcp_settings.access + assert retrieved_gcp.bucket == gcp_settings.bucket + + +def test_native_variant_storage_empty_accessor(): + native_setting = NativeVariantStorage() + + assert native_setting.setting_type == NativeVariantStorageContentType.EMPTY + + +def test_aws_auth_method_value(): + assert AWSAuthMethod.DISABLED.value == 0 + assert AWSAuthMethod.DEFAULT_CREDENTIALS_PROVIDER_CHAIN.value == 1 + assert AWSAuthMethod.STS_PROFILE_CREDENTIALS_PROVIDER.value == 2 + + +def test_native_variant_storage_setting_type_value(): + assert NativeVariantStorageContentType.EMPTY.value == 0 + assert NativeVariantStorageContentType.S3.value == 1 + assert NativeVariantStorageContentType.GCPXML.value == 2 + + +def test_metrics_config_accessors(): + config_values = { + "host": "host", + "port": "port", + "job": "job", + "instance": "instance", + "prometheus_env": "prometheus_env", + "metrics_config_model": MetricsConfigModel.PULL, + } + prom_conf = MetricsConfig( + "host", + "port", + "job", + "instance", + "prometheus_env", + MetricsConfigModel.PULL, + ) + + assert prom_conf.host == config_values["host"] + assert prom_conf.port == config_values["port"] + assert prom_conf.job_name == config_values["job"] + assert prom_conf.instance == config_values["instance"] + assert prom_conf.prometheus_env == config_values["prometheus_env"] + assert prom_conf.model == config_values["metrics_config_model"] + + +def test_metrics_config_model_value(): + assert MetricsConfigModel.NO_INIT.value == 0 + assert MetricsConfigModel.PUSH.value == 1 + assert MetricsConfigModel.PULL.value == 2 + + +def test_env_config_from_lib_config(lmdb_version_store_v1): + lib_cfg = lmdb_version_store_v1.lib_cfg() + env = lmdb_version_store_v1.env + open_mode = lmdb_version_store_v1.open_mode() + native_cfg = lmdb_version_store_v1.lib_native_cfg() + + envs_cfg = env_config_from_lib_config(lib_cfg, env) + cfg_resolver = create_mem_config_resolver(envs_cfg) + lib_idx = LibraryIndex.create_from_resolver(env, cfg_resolver) + enterprise_open_mode = OpenMode(open_mode.value) + + result = lib_idx.get_library(lib_cfg.lib_desc.name, enterprise_open_mode, native_cfg) + assert result.config == lmdb_version_store_v1._library.config + assert result.open_mode == lmdb_version_store_v1._library.open_mode + assert result.library_path == lmdb_version_store_v1._library.library_path diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 8390dd6a842..8d43b8e4ca2 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -302,6 +302,12 @@ def s3_and_nfs_storage_bucket(test_prefix, request): yield bucket +@pytest.fixture(scope="session") +def wrapped_s3_storage(wrapped_s3_storage_factory): + with wrapped_s3_storage_factory.create_fixture() as f: + yield f + + @pytest.fixture(scope="session") def s3_storage(s3_storage_factory) -> Generator[S3Bucket, None, None]: with s3_storage_factory.create_fixture() as f: @@ -780,6 +786,11 @@ def s3_store_factory(lib_name, s3_storage) -> Generator[Callable[..., NativeVers yield from _store_factory(lib_name, s3_storage) +@pytest.fixture +def wrapped_s3_store_factory(lib_name, wrapped_s3_storage) -> Generator[Callable[..., NativeVersionStore], None, None]: + yield from _store_factory(lib_name, wrapped_s3_storage) + + @pytest.fixture def s3_no_ssl_store_factory(lib_name, s3_no_ssl_storage) -> Generator[Callable[..., NativeVersionStore], None, None]: yield from _store_factory(lib_name, s3_no_ssl_storage) @@ -837,6 +848,11 @@ def in_memory_store_factory(mem_storage, lib_name) -> Generator[Callable[..., Na yield from _store_factory(lib_name, mem_storage) +@pytest.fixture +def wrapped_s3_version_store(wrapped_s3_store_factory): + return wrapped_s3_store_factory() + + # endregion # region ================================ `NativeVersionStore` Fixtures ================================= @pytest.fixture diff --git a/python/tests/integration/arcticdb/test_arctic.py b/python/tests/integration/arcticdb/test_arctic.py index 5243c6e968f..e6cc2f23d2f 100644 --- a/python/tests/integration/arcticdb/test_arctic.py +++ b/python/tests/integration/arcticdb/test_arctic.py @@ -22,7 +22,7 @@ from arcticdb_ext import get_config_int, set_config_int from arcticdb_ext.exceptions import InternalException, SortingException, UserInputException -from arcticdb_ext.storage import NoDataFoundException, KeyType +from arcticdb_ext.storage import NoDataFoundException, KeyType, AWSAuthMethod from arcticdb.exceptions import ArcticDbNotYetImplemented, NoSuchVersionException from arcticdb.adapters.mongo_library_adapter import MongoLibraryAdapter from arcticdb.arctic import Arctic @@ -43,10 +43,10 @@ ) from arcticdb.authorization.permissions import OpenMode from arcticdb.version_store._store import NativeVersionStore - from arcticdb.version_store.library import ArcticInvalidApiUsageException from tests.conftest import Marks from tests.util.marking import marks +from tests.util.storage_test import get_s3_storage_config from ...util.mark import ( AZURE_TESTS_MARK, MONGO_TESTS_MARK, @@ -237,6 +237,10 @@ def test_s3_sts_expiry_check(lib_name, real_s3_sts_storage): @pytest.mark.storage def test_s3_sts_auth_store(real_s3_sts_version_store): lib = real_s3_sts_version_store + assert lib.lib_native_cfg().as_s3_settings().aws_auth == AWSAuthMethod.STS_PROFILE_CREDENTIALS_PROVIDER + storage_cfg = get_s3_storage_config(lib.lib_cfg()) + assert storage_cfg.credential_name == "" # to ensure client can't fallback to default key+secret auth + assert storage_cfg.credential_key == "" df = pd.DataFrame({"a": [1, 2, 3]}) lib.write("sym", df) assert_frame_equal(lib.read("sym").data, df) diff --git a/python/tests/integration/arcticdb/test_s3.py b/python/tests/integration/arcticdb/test_s3.py index 490a79bd954..d3286eb15ff 100644 --- a/python/tests/integration/arcticdb/test_s3.py +++ b/python/tests/integration/arcticdb/test_s3.py @@ -32,8 +32,9 @@ ) -def test_s3_storage_failures(mock_s3_store_with_error_simulation): - lib = mock_s3_store_with_error_simulation +@pytest.mark.parametrize("version_store", ["wrapped_s3_version_store", "mock_s3_store_with_error_simulation"]) +def test_s3_storage_failures(request, version_store): + lib = request.getfixturevalue(version_store) symbol_success = "symbol" symbol_fail_write = "symbol#Failure_Write_99_0" symbol_fail_read = "symbol#Failure_Read_17_0"