endpoints: refactor Locality data-members to be shared from pool

source/common/upstream/BUILD

@@ -295,6 +295,19 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "locality_pool_lib",
+    srcs = ["locality_pool.cc"],
+    hdrs = ["locality_pool.h"],
+    deps = [
+        "//envoy/event:dispatcher_interface",
+        "//envoy/singleton:manager_interface",
+        "//envoy/upstream:locality_lib",
+        "//source/common/shared_pool:shared_pool_lib",
+        "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
+    ],
+)
+
 envoy_cc_library(
     name = "prod_cluster_info_factory_lib",
     srcs = ["prod_cluster_info_factory.cc"],
@@ -437,6 +450,7 @@ envoy_cc_library(
     ],
     deps = [
         ":load_balancer_context_base_lib",
+        ":locality_pool_lib",
         ":resource_manager_lib",
         ":upstream_factory_context_lib",
         "//envoy/event:timer_interface",

source/common/upstream/health_discovery_service.cc

@@ -373,9 +373,12 @@ HdsCluster::HdsCluster(Server::Configuration::ServerFactoryContext& server_conte
       // Initialize an endpoint host object.
       auto address_or_error = Network::Address::resolveProtoAddress(host.endpoint().address());
       THROW_IF_NOT_OK_REF(address_or_error.status());
+      auto const_locality_shared_pool = LocalityPool::getConstLocalitySharedPool(
+          server_context_.singletonManager(), server_context_.mainThreadDispatcher());
       HostSharedPtr endpoint = std::shared_ptr<HostImpl>(THROW_OR_RETURN_VALUE(
           HostImpl::create(info_, "", std::move(address_or_error.value()), nullptr, nullptr, 1,
-                           locality_endpoints.locality(), host.endpoint().health_check_config(), 0,
+                           const_locality_shared_pool->getObject(locality_endpoints.locality()),
+                           host.endpoint().health_check_config(), 0,
                            envoy::config::core::v3::UNKNOWN),
           std::unique_ptr<HostImpl>));
       // Add this host/endpoint pointer to our flat list of endpoints for health checking.
@@ -489,9 +492,12 @@ void HdsCluster::updateHosts(
         auto address_or_error =
             Network::Address::resolveProtoAddress(endpoint.endpoint().address());
         THROW_IF_NOT_OK_REF(address_or_error.status());
+        auto const_locality_shared_pool = LocalityPool::getConstLocalitySharedPool(
+            server_context_.singletonManager(), server_context_.mainThreadDispatcher());
         host = std::shared_ptr<HostImpl>(THROW_OR_RETURN_VALUE(
             HostImpl::create(info_, "", std::move(address_or_error.value()), nullptr, nullptr, 1,
-                             endpoints.locality(), endpoint.endpoint().health_check_config(), 0,
+                             const_locality_shared_pool->getObject(endpoints.locality()),
+                             endpoint.endpoint().health_check_config(), 0,
                              envoy::config::core::v3::UNKNOWN),
             std::unique_ptr<HostImpl>));
 

source/common/upstream/locality_pool.cc

@@ -0,0 +1,20 @@
+#include "source/common/upstream/locality_pool.h"
+
+#include "envoy/config/core/v3/base.pb.h"
+
+namespace Envoy {
+namespace Upstream {
+
+SINGLETON_MANAGER_REGISTRATION(const_locality_shared_pool);
+
+ConstLocalitySharedPoolSharedPtr
+LocalityPool::getConstLocalitySharedPool(Singleton::Manager& manager,
+                                         Event::Dispatcher& dispatcher) {
+  // Creating a pinned localities pool.
+  return manager.getTyped<ConstLocalitySharedPool>(
+      SINGLETON_MANAGER_REGISTERED_NAME(const_locality_shared_pool),
+      [&dispatcher] { return std::make_shared<ConstLocalitySharedPool>(dispatcher); }, true);
+}
+
+} // namespace Upstream
+} // namespace Envoy

source/common/upstream/locality_pool.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <memory>
+
+#include "envoy/config/core/v3/base.pb.h"
+#include "envoy/event/dispatcher.h"
+#include "envoy/singleton/manager.h"
+#include "envoy/upstream/locality.h"
+
+#include "source/common/shared_pool/shared_pool.h"
+
+namespace Envoy {
+namespace Upstream {
+
+using ConstLocalitySharedPool =
+    SharedPool::ObjectSharedPool<const envoy::config::core::v3::Locality, LocalityHash,
+                                 LocalityEqualTo>;
+using ConstLocalitySharedPoolSharedPtr = std::shared_ptr<ConstLocalitySharedPool>;
+
+class LocalityPool {
+public:
+  /**
+   * Returns an ObjectSharedPool to store const Locality
+   * @param manager used to create singleton
+   * @param dispatcher the dispatcher object reference to the thread that created the
+   * ObjectSharedPool
+   */
+  static ConstLocalitySharedPoolSharedPtr getConstLocalitySharedPool(Singleton::Manager& manager,
+                                                                     Event::Dispatcher& dispatcher);
+};
+
+} // namespace Upstream
+} // namespace Envoy

source/common/upstream/upstream_impl.cc

@@ -56,6 +56,7 @@
 #include "source/common/stats/deferred_creation.h"
 #include "source/common/upstream/cluster_factory_impl.h"
 #include "source/common/upstream/health_checker_impl.h"
+#include "source/common/upstream/locality_pool.h"
 #include "source/server/transport_socket_config_impl.h"
 
 #include "absl/container/node_hash_set.h"
@@ -450,7 +451,8 @@ LoadMetricStats::StatMapPtr LoadMetricStatsImpl::latch() {
 absl::StatusOr<std::unique_ptr<HostDescriptionImpl>> HostDescriptionImpl::create(
     ClusterInfoConstSharedPtr cluster, const std::string& hostname,
     Network::Address::InstanceConstSharedPtr dest_address, MetadataConstSharedPtr endpoint_metadata,
-    MetadataConstSharedPtr locality_metadata, const envoy::config::core::v3::Locality& locality,
+    MetadataConstSharedPtr locality_metadata,
+    std::shared_ptr<const envoy::config::core::v3::Locality> locality,
     const envoy::config::endpoint::v3::Endpoint::HealthCheckConfig& health_check_config,
     uint32_t priority, const AddressVector& address_list) {
   absl::Status creation_status = absl::OkStatus();
@@ -464,7 +466,8 @@ absl::StatusOr<std::unique_ptr<HostDescriptionImpl>> HostDescriptionImpl::create
 HostDescriptionImpl::HostDescriptionImpl(
     absl::Status& creation_status, ClusterInfoConstSharedPtr cluster, const std::string& hostname,
     Network::Address::InstanceConstSharedPtr dest_address, MetadataConstSharedPtr endpoint_metadata,
-    MetadataConstSharedPtr locality_metadata, const envoy::config::core::v3::Locality& locality,
+    MetadataConstSharedPtr locality_metadata,
+    std::shared_ptr<const envoy::config::core::v3::Locality> locality,
     const envoy::config::endpoint::v3::Endpoint::HealthCheckConfig& health_check_config,
     uint32_t priority, const AddressVector& address_list)
     : HostDescriptionImplBase(cluster, hostname, dest_address, endpoint_metadata, locality_metadata,
@@ -476,7 +479,8 @@ HostDescriptionImpl::HostDescriptionImpl(
 HostDescriptionImplBase::HostDescriptionImplBase(
     ClusterInfoConstSharedPtr cluster, const std::string& hostname,
     Network::Address::InstanceConstSharedPtr dest_address, MetadataConstSharedPtr endpoint_metadata,
-    MetadataConstSharedPtr locality_metadata, const envoy::config::core::v3::Locality& locality,
+    MetadataConstSharedPtr locality_metadata,
+    std::shared_ptr<const envoy::config::core::v3::Locality> locality,
     const envoy::config::endpoint::v3::Endpoint::HealthCheckConfig& health_check_config,
     uint32_t priority, absl::Status& creation_status)
     : cluster_(cluster), hostname_(hostname),
@@ -486,8 +490,8 @@ HostDescriptionImplBase::HostDescriptionImplBase(
                                               Config::MetadataEnvoyLbKeys::get().CANARY)
                   .bool_value()),
       endpoint_metadata_(endpoint_metadata), locality_metadata_(locality_metadata),
-      locality_(locality),
-      locality_zone_stat_name_(locality.zone(), cluster->statsScope().symbolTable()),
+      locality_(std::move(locality)),
+      locality_zone_stat_name_(locality_->zone(), cluster->statsScope().symbolTable()),
       priority_(priority),
       socket_factory_(resolveTransportSocketFactory(dest_address, endpoint_metadata_.get())) {
   if (health_check_config.port_value() != 0 && dest_address->type() != Network::Address::Type::Ip) {
@@ -677,7 +681,7 @@ absl::StatusOr<std::unique_ptr<HostImpl>> HostImpl::create(
     ClusterInfoConstSharedPtr cluster, const std::string& hostname,
     Network::Address::InstanceConstSharedPtr address, MetadataConstSharedPtr endpoint_metadata,
     MetadataConstSharedPtr locality_metadata, uint32_t initial_weight,
-    const envoy::config::core::v3::Locality& locality,
+    std::shared_ptr<const envoy::config::core::v3::Locality> locality,
     const envoy::config::endpoint::v3::Endpoint::HealthCheckConfig& health_check_config,
     uint32_t priority, const envoy::config::core::v3::HealthStatus health_status,
     const AddressVector& address_list) {
@@ -1540,6 +1544,9 @@ ClusterImplBase::ClusterImplBase(const envoy::config::cluster::v3::Cluster& clus
           cluster_context.serverFactoryContext().clusterManager().localClusterName().value_or("") ==
           cluster.name()),
       const_metadata_shared_pool_(Config::Metadata::getConstMetadataSharedPool(
+          cluster_context.serverFactoryContext().singletonManager(),
+          cluster_context.serverFactoryContext().mainThreadDispatcher())),
+      const_locality_shared_pool_(LocalityPool::getConstLocalitySharedPool(
           cluster_context.serverFactoryContext().singletonManager(),
           cluster_context.serverFactoryContext().mainThreadDispatcher())) {
   auto& server_context = cluster_context.serverFactoryContext();
@@ -2139,10 +2146,12 @@ void PriorityStateManager::registerHostForPriority(
           ? parent_.constMetadataSharedPool()->getObject(locality_lb_endpoint.metadata())
           : nullptr;
   const auto host = std::shared_ptr<HostImpl>(THROW_OR_RETURN_VALUE(
-      HostImpl::create(parent_.info(), hostname, address, endpoint_metadata, locality_metadata,
-                       lb_endpoint.load_balancing_weight().value(), locality_lb_endpoint.locality(),
-                       lb_endpoint.endpoint().health_check_config(),
-                       locality_lb_endpoint.priority(), lb_endpoint.health_status(), address_list),
+      HostImpl::create(
+          parent_.info(), hostname, address, endpoint_metadata, locality_metadata,
+          lb_endpoint.load_balancing_weight().value(),
+          parent_.constLocalitySharedPool()->getObject(locality_lb_endpoint.locality()),
+          lb_endpoint.endpoint().health_check_config(), locality_lb_endpoint.priority(),
+          lb_endpoint.health_status(), address_list),
       std::unique_ptr<HostImpl>));
   registerHostForPriority(host, locality_lb_endpoint);
 }

source/common/upstream/upstream_impl.h

@@ -59,6 +59,7 @@
 #include "source/common/shared_pool/shared_pool.h"
 #include "source/common/stats/isolated_store_impl.h"
 #include "source/common/upstream/load_balancer_context_base.h"
+#include "source/common/upstream/locality_pool.h"
 #include "source/common/upstream/resource_manager_impl.h"
 #include "source/common/upstream/transport_socket_match_impl.h"
 #include "source/common/upstream/upstream_factory_context_impl.h"
@@ -212,7 +213,7 @@ class HostDescriptionImplBase : virtual public HostDescription,
   LoadMetricStats& loadMetricStats() const override { return load_metric_stats_; }
   const std::string& hostnameForHealthChecks() const override { return health_checks_hostname_; }
   const std::string& hostname() const override { return hostname_; }
-  const envoy::config::core::v3::Locality& locality() const override { return locality_; }
+  const envoy::config::core::v3::Locality& locality() const override { return *locality_; }
   const MetadataConstSharedPtr localityMetadata() const override { return locality_metadata_; }
   Stats::StatName localityZoneStatName() const override {
     return locality_zone_stat_name_.statName();
@@ -248,7 +249,7 @@ class HostDescriptionImplBase : virtual public HostDescription,
       ClusterInfoConstSharedPtr cluster, const std::string& hostname,
       Network::Address::InstanceConstSharedPtr dest_address,
       MetadataConstSharedPtr endpoint_metadata, MetadataConstSharedPtr locality_metadata,
-      const envoy::config::core::v3::Locality& locality,
+      std::shared_ptr<const envoy::config::core::v3::Locality> locality,
       const envoy::config::endpoint::v3::Endpoint::HealthCheckConfig& health_check_config,
       uint32_t priority, absl::Status& creation_status);
 
@@ -267,7 +268,7 @@ class HostDescriptionImplBase : virtual public HostDescription,
   mutable absl::Mutex metadata_mutex_;
   MetadataConstSharedPtr endpoint_metadata_ ABSL_GUARDED_BY(metadata_mutex_);
   const MetadataConstSharedPtr locality_metadata_;
-  const envoy::config::core::v3::Locality locality_;
+  const std::shared_ptr<const envoy::config::core::v3::Locality> locality_;
   Stats::StatNameDynamicStorage locality_zone_stat_name_;
   mutable HostStats stats_;
   mutable LoadMetricStatsImpl load_metric_stats_;
@@ -294,7 +295,7 @@ class HostDescriptionImpl : public HostDescriptionImplBase {
   create(ClusterInfoConstSharedPtr cluster, const std::string& hostname,
          Network::Address::InstanceConstSharedPtr dest_address,
          MetadataConstSharedPtr endpoint_metadata, MetadataConstSharedPtr locality_metadata,
-         const envoy::config::core::v3::Locality& locality,
+         std::shared_ptr<const envoy::config::core::v3::Locality> locality,
          const envoy::config::endpoint::v3::Endpoint::HealthCheckConfig& health_check_config,
          uint32_t priority, const AddressVector& address_list = {});
 
@@ -310,7 +311,7 @@ class HostDescriptionImpl : public HostDescriptionImplBase {
       absl::Status& creation_status, ClusterInfoConstSharedPtr cluster, const std::string& hostname,
       Network::Address::InstanceConstSharedPtr dest_address,
       MetadataConstSharedPtr endpoint_metadata, MetadataConstSharedPtr locality_metadata,
-      const envoy::config::core::v3::Locality& locality,
+      std::shared_ptr<const envoy::config::core::v3::Locality> locality,
       const envoy::config::endpoint::v3::Endpoint::HealthCheckConfig& health_check_config,
       uint32_t priority, const AddressVector& address_list = {});
 
@@ -477,7 +478,7 @@ class HostImpl : public HostImplBase, public HostDescriptionImpl {
   create(ClusterInfoConstSharedPtr cluster, const std::string& hostname,
          Network::Address::InstanceConstSharedPtr address, MetadataConstSharedPtr endpoint_metadata,
          MetadataConstSharedPtr locality_metadata, uint32_t initial_weight,
-         const envoy::config::core::v3::Locality& locality,
+         std::shared_ptr<const envoy::config::core::v3::Locality> locality,
          const envoy::config::endpoint::v3::Endpoint::HealthCheckConfig& health_check_config,
          uint32_t priority, const envoy::config::core::v3::HealthStatus health_status,
          const AddressVector& address_list = {});
@@ -486,7 +487,8 @@ class HostImpl : public HostImplBase, public HostDescriptionImpl {
   HostImpl(absl::Status& creation_status, ClusterInfoConstSharedPtr cluster,
            const std::string& hostname, Network::Address::InstanceConstSharedPtr address,
            MetadataConstSharedPtr endpoint_metadata, MetadataConstSharedPtr locality_metadata,
-           uint32_t initial_weight, const envoy::config::core::v3::Locality& locality,
+           uint32_t initial_weight,
+           std::shared_ptr<const envoy::config::core::v3::Locality> locality,
            const envoy::config::endpoint::v3::Endpoint::HealthCheckConfig& health_check_config,
            uint32_t priority, const envoy::config::core::v3::HealthStatus health_status,
            const AddressVector& address_list = {})
@@ -1173,6 +1175,10 @@ class ClusterImplBase : public Cluster, protected Logger::Loggable<Logger::Id::u
     return const_metadata_shared_pool_;
   }
 
+  ConstLocalitySharedPoolSharedPtr constLocalitySharedPool() const {
+    return const_locality_shared_pool_;
+  }
+
   // Upstream::Cluster
   HealthChecker* healthChecker() override { return health_checker_.get(); }
   ClusterInfoConstSharedPtr info() const override { return info_; }
@@ -1251,6 +1257,7 @@ class ClusterImplBase : public Cluster, protected Logger::Loggable<Logger::Id::u
   uint64_t pending_initialize_health_checks_{};
   const bool local_cluster_;
   Config::ConstMetadataSharedPoolSharedPtr const_metadata_shared_pool_;
+  ConstLocalitySharedPoolSharedPtr const_locality_shared_pool_;
   Common::CallbackHandlePtr priority_update_cb_;
   UnitFloat drop_overload_{0};
   std::string drop_category_;

source/extensions/clusters/common/logical_host.cc

@@ -32,8 +32,11 @@ LogicalHost::LogicalHost(
           std::make_shared<const envoy::config::core::v3::Metadata>(lb_endpoint.metadata()),
           std::make_shared<const envoy::config::core::v3::Metadata>(
               locality_lb_endpoint.metadata()),
-          locality_lb_endpoint.locality(), lb_endpoint.endpoint().health_check_config(),
-          locality_lb_endpoint.priority(), creation_status),
+          // TODO(adisuissa): Create through localities shared pool.
+          std::make_shared<const envoy::config::core::v3::Locality>(
+              locality_lb_endpoint.locality()),
+          lb_endpoint.endpoint().health_check_config(), locality_lb_endpoint.priority(),
+          creation_status),
       override_transport_socket_options_(override_transport_socket_options), address_(address),
       address_list_or_null_(makeAddressListOrNull(address, address_list)) {
   health_check_address_ =

source/extensions/clusters/dns/dns_cluster.cc

@@ -321,7 +321,8 @@ DnsClusterImpl::ResolveTarget::createStrictDnsHosts(
         std::make_shared<const envoy::config::core::v3::Metadata>(lb_endpoint_.metadata()),
         std::make_shared<const envoy::config::core::v3::Metadata>(
             locality_lb_endpoints_.metadata()),
-        lb_endpoint_.load_balancing_weight().value(), locality_lb_endpoints_.locality(),
+        lb_endpoint_.load_balancing_weight().value(),
+        parent_.constLocalitySharedPool()->getObject(locality_lb_endpoints_.locality()),
         lb_endpoint_.endpoint().health_check_config(), locality_lb_endpoints_.priority(),
         lb_endpoint_.health_status());
 

source/extensions/clusters/original_dst/original_dst_cluster.cc

@@ -76,7 +76,7 @@ HostSelectionResponse OriginalDstCluster::LoadBalancer::chooseHost(LoadBalancerC
         HostSharedPtr host(std::shared_ptr<HostImpl>(THROW_OR_RETURN_VALUE(
             HostImpl::create(
                 info, info->name() + dst_addr.asString(), std::move(host_ip_port), nullptr, nullptr,
-                1, envoy::config::core::v3::Locality().default_instance(),
+                1, std::make_shared<envoy::config::core::v3::Locality>(),
                 envoy::config::endpoint::v3::Endpoint::HealthCheckConfig().default_instance(), 0,
                 envoy::config::core::v3::UNKNOWN),
             std::unique_ptr<HostImpl>)));

source/extensions/clusters/redis/redis_cluster.h

@@ -166,7 +166,9 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl {
               std::make_shared<envoy::config::core::v3::Metadata>(
                   parent.localityLbEndpoint().metadata()),
               parent.lbEndpoint().load_balancing_weight().value(),
-              parent.localityLbEndpoint().locality(),
+              // TODO(adisuissa): Convert to use a shared pool of localities.
+              std::make_shared<const envoy::config::core::v3::Locality>(
+                  parent.localityLbEndpoint().locality()),
               parent.lbEndpoint().endpoint().health_check_config(),
               parent.localityLbEndpoint().priority(), parent.lbEndpoint().health_status()),
           primary_(primary) {}

source/extensions/clusters/reverse_connection/reverse_connection.cc

@@ -103,7 +103,7 @@ Upstream::HostSelectionResponse RevConCluster::checkAndCreateHost(absl::string_v
   auto host_result = Upstream::HostImpl::create(
       info(), absl::StrCat(info()->name(), static_cast<std::string>(node_id)),
       std::move(host_address), nullptr /* endpoint_metadata */, nullptr /* locality_metadata */,
-      1 /* initial_weight */, envoy::config::core::v3::Locality().default_instance(),
+      1 /* initial_weight */, std::make_shared<const envoy::config::core::v3::Locality>(),
       envoy::config::endpoint::v3::Endpoint::HealthCheckConfig().default_instance(),
       0 /* priority */, envoy::config::core::v3::UNKNOWN);