Consolidate bindings for Enterprise ABI compatibility

cpp/arcticdb/entity/metrics.hpp

@@ -32,7 +32,7 @@ constexpr int SUMMARY_AGE_BUCKETS = 5;
 
 class MetricsConfig {
   public:
-    enum class Model { NO_INIT, PUSH, PULL };
+    enum class Model { NO_INIT = 0, PUSH = 1, PULL = 2 };
     MetricsConfig() : model_(Model::NO_INIT) {}
 
     MetricsConfig(

cpp/arcticdb/entity/python_bindings_common.cpp

@@ -19,7 +19,8 @@ namespace py = pybind11;
 
 namespace arcticdb::entity::apy {
 
-void register_common_entity_bindings(py::module& m, bool local_bindings) {
+void register_common_entity_bindings(py::module& m, BindingScope scope) {
+    bool local_bindings = (scope == BindingScope::LOCAL);
     py::class_<AtomKey, std::shared_ptr<AtomKey>>(m, "AtomKey", py::module_local(local_bindings))
             .def(py::init())
             .def(py::init<StreamId, VersionId, timestamp, ContentHash, IndexValue, IndexValue, KeyType>())
@@ -52,7 +53,6 @@ void register_common_entity_bindings(py::module& m, bool local_bindings) {
                     [](py::tuple t) {
                         util::check(t.size() >= 7, "Invalid AtomKey pickle object!");
 
-                        [[maybe_unused]] const int serialization_version = t[0].cast<int>();
                         AtomKey key(
                                 t[1].cast<StreamId>(),
                                 t[2].cast<VersionId>(),

cpp/arcticdb/entity/python_bindings_common.hpp

@@ -8,10 +8,10 @@
 
 #pragma once
 
-#include <pybind11/pybind11.h>
+#include <arcticdb/python/python_bindings_common.hpp>
 
 namespace arcticdb::entity::apy {
 
-void register_common_entity_bindings(pybind11::module& m, bool local_bindings);
+void register_common_entity_bindings(pybind11::module& m, BindingScope scope);
 
 } // namespace arcticdb::entity::apy

cpp/arcticdb/python/python_bindings_common.cpp

@@ -1,10 +1,20 @@
+#include <pybind11/pybind11.h>
 #include <arcticdb/python/python_bindings_common.hpp>
 #include <arcticdb/entity/metrics.hpp>
 #include <arcticdb/util/preconditions.hpp>
 
 namespace py = pybind11;
 
-namespace arcticdb::apy {
+namespace arcticdb {
+
+enum class MetricsConfigPickleOrder : uint32_t {
+    HOST = 0,
+    PORT = 1,
+    JOB_NAME = 2,
+    INSTANCE = 3,
+    PROMETHEUS_ENV = 4,
+    MODEL = 5
+};
 
 py::tuple to_tuple(const MetricsConfig& config) {
     return py::make_tuple(
@@ -13,7 +23,19 @@ py::tuple to_tuple(const MetricsConfig& config) {
 }
 
 MetricsConfig metrics_config_from_tuple(const py::tuple& t) {
-    util::check(t.size() == 6, "Invalid MetricsConfig pickle object, expected 6 attributes but was {}", t.size());
+    static size_t py_object_size = 6;
+    util::check(
+            t.size() >= py_object_size,
+            "Invalid MetricsConfig pickle object, expected at least {} attributes but was {}",
+            py_object_size,
+            t.size()
+    );
+    util::warn(
+            t.size() > py_object_size,
+            "MetricsConfig py tuple expects {} attributes but has {}. Will continue by ignoring extra attributes.",
+            py_object_size,
+            t.size()
+    );
     return MetricsConfig{
             t[static_cast<uint32_t>(MetricsConfigPickleOrder::HOST)].cast<std::string>(),
             t[static_cast<uint32_t>(MetricsConfigPickleOrder::PORT)].cast<std::string>(),
@@ -24,10 +46,10 @@ MetricsConfig metrics_config_from_tuple(const py::tuple& t) {
     };
 }
 
-} // namespace arcticdb::apy
-
-void register_metrics(py::module&& m, bool local_bindings) {
+} // namespace arcticdb
 
+void register_metrics(py::module&& m, arcticdb::BindingScope scope) {
+    bool local_bindings = (scope == arcticdb::BindingScope::LOCAL);
     auto prometheus = m.def_submodule("prometheus");
     py::class_<arcticdb::PrometheusInstance, std::shared_ptr<arcticdb::PrometheusInstance>>(
             prometheus, "Instance", py::module_local(local_bindings)
@@ -44,8 +66,8 @@ void register_metrics(py::module&& m, bool local_bindings) {
                     const std::string&,
                     const arcticdb::MetricsConfig::Model>())
             .def(py::pickle(
-                    [](const arcticdb::MetricsConfig& config) { return arcticdb::apy::to_tuple(config); },
-                    [](py::tuple t) { return arcticdb::apy::metrics_config_from_tuple(t); }
+                    [](const arcticdb::MetricsConfig& config) { return arcticdb::to_tuple(config); },
+                    [](py::tuple t) { return arcticdb::metrics_config_from_tuple(t); }
             ));
 
     py::enum_<arcticdb::MetricsConfig::Model>(prometheus, "MetricsConfigModel", py::module_local(local_bindings))
@@ -54,5 +76,5 @@ void register_metrics(py::module&& m, bool local_bindings) {
             .value("PULL", arcticdb::MetricsConfig::Model::PULL)
             .export_values();
 
-    prometheus.def("metrics_config_from_tuple", &arcticdb::apy::metrics_config_from_tuple);
+    prometheus.def("metrics_config_from_tuple", &arcticdb::metrics_config_from_tuple);
 }

cpp/arcticdb/python/python_bindings_common.hpp

@@ -1,21 +1,18 @@
 #pragma once
-#include <pybind11/pybind11.h>
 #include <arcticdb/entity/metrics.hpp>
 
-namespace arcticdb::apy {
+namespace pybind11 {
+class module_;
+using module = module_;
+class tuple;
+} // namespace pybind11
 
-enum class MetricsConfigPickleOrder : uint32_t {
-    HOST = 0,
-    PORT = 1,
-    JOB_NAME = 2,
-    INSTANCE = 3,
-    PROMETHEUS_ENV = 4,
-    MODEL = 5
-};
+namespace arcticdb {
+enum class BindingScope : uint32_t { LOCAL = 0, GLOBAL = 1 };
 
 pybind11::tuple to_tuple(const MetricsConfig& config);
 MetricsConfig metrics_config_from_tuple(const pybind11::tuple& t);
 
-} // namespace arcticdb::apy
+} // namespace arcticdb
 
-void register_metrics(pybind11::module&& m, bool local_bindings);
+void register_metrics(pybind11::module&& m, arcticdb::BindingScope scope);

cpp/arcticdb/python/python_module.cpp

@@ -330,36 +330,35 @@ PYBIND11_MODULE(arcticdb_ext, m) {
 #endif
     // Set up the global exception handlers first, so module-specific exception handler can override it:
     auto exceptions = m.def_submodule("exceptions");
-    auto base_exception =
-            py::register_exception<arcticdb::ArcticException>(exceptions, "ArcticException", PyExc_RuntimeError);
+    auto base_exception = py::register_exception<ArcticException>(exceptions, "ArcticException", PyExc_RuntimeError);
     register_error_code_ecosystem(exceptions, base_exception);
 
-    arcticdb::async::register_bindings(m);
-    arcticdb::codec::register_bindings(m);
-    arcticdb::column_store::register_bindings(m);
+    async::register_bindings(m);
+    codec::register_bindings(m);
+    column_store::register_bindings(m);
 
     auto storage_submodule = m.def_submodule("storage", "Segment storage implementation apis");
-    auto no_data_found_exception = py::register_exception<arcticdb::storage::NoDataFoundException>(
+    auto no_data_found_exception = py::register_exception<storage::NoDataFoundException>(
             storage_submodule, "NoDataFoundException", base_exception.ptr()
     );
-    arcticdb::storage::apy::register_bindings(storage_submodule, base_exception);
+    storage::apy::register_bindings(storage_submodule, base_exception);
 
-    arcticdb::stream::register_bindings(m);
-    arcticdb::toolbox::apy::register_bindings(m, base_exception);
-    arcticdb::util::register_bindings(m);
+    stream::register_bindings(m);
+    toolbox::apy::register_bindings(m, base_exception);
+    util::register_bindings(m);
 
-    m.def("get_version_string", &arcticdb::get_arcticdb_version_string);
+    m.def("get_version_string", &get_arcticdb_version_string);
 
     auto version_submodule = m.def_submodule("version_store", "Versioned storage implementation apis");
-    arcticdb::version_store::register_bindings(version_submodule, base_exception);
-    py::register_exception<arcticdb::NoSuchVersionException>(
+    version_store::register_bindings(version_submodule, base_exception);
+    py::register_exception<NoSuchVersionException>(
             version_submodule, "NoSuchVersionException", no_data_found_exception.ptr()
     );
 
     register_configs_map_api(m);
     register_log(m.def_submodule("log"));
     register_instrumentation(m.def_submodule("instrumentation"));
-    register_metrics(m.def_submodule("metrics"), false);
+    register_metrics(m.def_submodule("metrics"), BindingScope::GLOBAL);
     register_type_handlers();
 
     register_termination_handler();

cpp/arcticdb/storage/python_bindings.cpp

@@ -93,7 +93,7 @@ void register_bindings(py::module& storage, py::exception<arcticdb::ArcticExcept
 
     storage.def("create_library_index", &create_library_index);
 
-    register_common_bindings(storage, false);
+    register_common_storage_bindings(storage, BindingScope::GLOBAL);
 
     py::class_<S3Override>(storage, "S3Override")
             .def(py::init<>())

cpp/arcticdb/storage/python_bindings_common.cpp

@@ -9,7 +9,19 @@ namespace arcticdb::storage::apy {
 using namespace python_util;
 
 s3::GCPXMLSettings gcp_settings(const py::tuple& t) {
-    util::check(t.size() == 11, "Invalid GCPXMLSettings pickle objects, expected 11 attributes but was {}", t.size());
+    static size_t py_object_size = 11;
+    util::check(
+            t.size() >= py_object_size,
+            "Invalid GCPXMLSettings pickle objects, expected at least {} attributes but was {}",
+            py_object_size,
+            t.size()
+    );
+    util::warn(
+            t.size() > py_object_size,
+            "GCPXMLSettings py tuple expects {} attributes but has {}. Will continue by ignoring extra attributes.",
+            py_object_size,
+            t.size()
+    );
     return s3::GCPXMLSettings{
             t[static_cast<uint32_t>(GCPXMLSettingsPickleOrder::AWS_AUTH)].cast<s3::AWSAuthMethod>(),
             t[static_cast<uint32_t>(GCPXMLSettingsPickleOrder::CA_CERT_PATH)].cast<std::string>(),
@@ -25,7 +37,14 @@ s3::GCPXMLSettings gcp_settings(const py::tuple& t) {
 }
 
 s3::S3Settings s3_settings(const py::tuple& t) {
-    util::check(t.size() == 4, "Invalid S3Settings pickle objects");
+    static size_t py_object_size = 4;
+    util::check(t.size() >= py_object_size, "Invalid S3Settings pickle objects");
+    util::warn(
+            t.size() > py_object_size,
+            "S3Settings py tuple expects {} attributes but has {}. Will continue by ignoring extra attributes.",
+            py_object_size,
+            t.size()
+    );
     return s3::S3Settings{
             t[static_cast<uint32_t>(S3SettingsPickleOrder::AWS_AUTH)].cast<s3::AWSAuthMethod>(),
             t[static_cast<uint32_t>(S3SettingsPickleOrder::AWS_PROFILE)].cast<std::string>(),
@@ -58,8 +77,8 @@ py::tuple to_tuple(const s3::S3Settings& settings) {
     );
 }
 
-void register_common_bindings(py::module& storage, bool local_bindings) {
-
+void register_common_storage_bindings(py::module& storage, BindingScope scope) {
+    bool local_bindings = (scope == BindingScope::LOCAL);
     py::enum_<s3::AWSAuthMethod>(storage, "AWSAuthMethod", py::module_local(local_bindings))
             .value("DISABLED", s3::AWSAuthMethod::DISABLED)
             .value("DEFAULT_CREDENTIALS_PROVIDER_CHAIN", s3::AWSAuthMethod::DEFAULT_CREDENTIALS_PROVIDER_CHAIN)

cpp/arcticdb/storage/python_bindings_common.hpp

@@ -4,6 +4,7 @@
 #include <arcticdb/util/error_code.hpp>
 #include <arcticdb/storage/s3/s3_settings.hpp>
 #include <arcticdb/storage/common.hpp>
+#include <arcticdb/python/python_bindings_common.hpp>
 
 namespace arcticdb::storage::apy {
 
@@ -32,7 +33,7 @@ s3::GCPXMLSettings gcp_settings(const pybind11::tuple& t);
 s3::S3Settings s3_settings(const pybind11::tuple& t);
 pybind11::tuple to_tuple(const s3::GCPXMLSettings& settings);
 pybind11::tuple to_tuple(const s3::S3Settings& settings);
-void register_common_bindings(pybind11::module& m, bool local_bindings);
+void register_common_storage_bindings(pybind11::module& m, BindingScope scope);
 NativeVariantStorage reconstruct_native_variant_storage_py_tuple(const pybind11::tuple& t);
 
 } // namespace arcticdb::storage::apy

cpp/arcticdb/version/python_bindings.cpp

@@ -145,7 +145,7 @@ void register_bindings(py::module& version, py::exception<arcticdb::ArcticExcept
 
     py::register_exception<StreamDescriptorMismatch>(version, "StreamDescriptorMismatch", base_exception.ptr());
 
-    entity::apy::register_common_entity_bindings(version, false);
+    entity::apy::register_common_entity_bindings(version, arcticdb::BindingScope::GLOBAL);
 
     py::class_<RefKey, std::shared_ptr<RefKey>>(version, "RefKey")
             .def(py::init())

python/arcticdb/_core_api.py

@@ -0,0 +1,20 @@
+"""
+Copyright 2023 Man Group Operations Ltd.
+NO WARRANTY, EXPRESSED OR IMPLIED.
+
+This module implements a backwards compatible version of msgpack functions.
+"""
+
+# Treat everything here as public APIs!!!! Any change made here may break downstream repos!!!!
+
+from arcticdb_ext.storage import NativeVariantStorage
+from arcticdb_ext.metrics.prometheus import MetricsConfig
+from arcticdb.version_store._store import _env_config_from_lib_config as env_config_from_lib_config
+
+
+def convert_native_variant_storage_to_py_tuple(native_cfg: NativeVariantStorage):
+    return native_cfg.__getstate__()
+
+
+def convert_metrics_config_to_py_tuple(metrics_cfg: MetricsConfig):
+    return metrics_cfg.__getstate__()