bpf_metadata: Skip if destination is local host

Original source address need not and can not be used if the destination
is a local pod, or a local host. Add the local host case to bpf_metadata
filter, and add handling of both cases to socket_option_source_address,
as at that time the chosen destination is known.

In principle, we should not need these checks in bpf_metadata any more,
but tests/ currently depends on them.

Signed-off-by: Jarno Rajahalme <[email protected]>

Author: jrajahalme

cilium/bpf_metadata.cc

@@ -276,7 +276,7 @@ uint32_t Config::resolvePolicyId(const Network::Address::Ip* ip) const {
 
   // default destination identity to the world if needed
   if (id == 0) {
-    id = Cilium::ID::WORLD;
+    id = Cilium::ID::World;
     ENVOY_LOG(trace, "bpf_metadata: Identity for IP defaults to WORLD", ip->addressAsString());
   }
 
@@ -368,6 +368,10 @@ const PolicyInstance& Config::getPolicy(const std::string& pod_ip) const {
   return npmap_->getPolicyInstance(pod_ip, allow_egress);
 }
 
+bool Config::exists(const std::string& pod_ip) const {
+  return npmap_ != nullptr && npmap_->exists(pod_ip);
+}
+
 absl::optional<Cilium::BpfMetadata::SocketMetadata>
 Config::extractSocketMetadata(Network::ConnectionSocket& socket) {
   Network::Address::InstanceConstSharedPtr src_address =
@@ -473,7 +477,7 @@ Config::extractSocketMetadata(Network::ConnectionSocket& socket) {
 
     // Resolve source identity for the Ingress address
     source_identity = resolvePolicyId(ingress_ip);
-    if (source_identity == Cilium::ID::WORLD) {
+    if (source_identity == Cilium::ID::World) {
       // No security ID available for the configured source IP
       ENVOY_LOG(warn,
                 "cilium.bpf_metadata (north/south L7 LB): Unknown local Ingress IP source address "
@@ -484,7 +488,8 @@ Config::extractSocketMetadata(Network::ConnectionSocket& socket) {
 
     // Original source address is never used for north/south LB
     src_address = nullptr;
-  } else if (!use_original_source_address_ || (npmap_ != nullptr && npmap_->exists(other_ip))) {
+  } else if (!use_original_source_address_ || destination_identity == Cilium::ID::Host ||
+             (npmap_ != nullptr && npmap_->exists(other_ip))) {
     // Otherwise only use the original source address if permitted and the destination is not
     // in the same node.
     //

cilium/bpf_metadata.h

@@ -68,8 +68,8 @@ struct SocketMetadata : public Logger::Loggable<Logger::Id::filter> {
   std::shared_ptr<Envoy::Cilium::SourceAddressSocketOption> buildSourceAddressSocketOption(
       int linger_time, const std::shared_ptr<CiliumDestinationFilterState>& dest_fs = nullptr) {
     return std::make_shared<Envoy::Cilium::SourceAddressSocketOption>(
-        source_identity_, linger_time, original_source_address_, source_address_ipv4_,
-        source_address_ipv6_, dest_fs);
+        source_identity_, policy_resolver_, linger_time, original_source_address_,
+        source_address_ipv4_, source_address_ipv6_, dest_fs);
   };
 
   // Add ProxyLib L7 protocol as requested application protocol on the socket.
@@ -144,6 +144,7 @@ class Config : public Cilium::PolicyResolver,
   // PolicyResolver
   uint32_t resolvePolicyId(const Network::Address::Ip*) const override;
   const PolicyInstance& getPolicy(const std::string&) const override;
+  bool exists(const std::string&) const override;
 
   virtual absl::optional<SocketMetadata> extractSocketMetadata(Network::ConnectionSocket& socket);
 

cilium/filter_state_cilium_policy.h

@@ -27,6 +27,7 @@ class PolicyResolver {
 
   virtual uint32_t resolvePolicyId(const Network::Address::Ip*) const PURE;
   virtual const PolicyInstance& getPolicy(const std::string&) const PURE;
+  virtual bool exists(const std::string&) const PURE;
 };
 
 // FilterState that holds relevant connection & policy information that can be retrieved
@@ -55,7 +56,7 @@ class CiliumPolicyFilterState : public StreamInfo::FilterState::Object,
     if (resolver) {
       return resolver->resolvePolicyId(ip);
     }
-    return Cilium::ID::WORLD; // default to WORLD policy ID if resolver is no longer available
+    return Cilium::ID::World; // default to WORLD policy ID if resolver is no longer available
   }
 
   const PolicyInstance& getPolicy() const {

cilium/host_map.h

@@ -154,7 +154,7 @@ class PolicyHostMap : public Singleton::Instance,
           return it->second;
         }
       }
-      return ID::UNKNOWN;
+      return ID::Unknown;
     }
 
     uint64_t resolve(absl::uint128 addr6) const {
@@ -164,7 +164,7 @@ class PolicyHostMap : public Singleton::Instance,
           return it->second;
         }
       }
-      return ID::UNKNOWN;
+      return ID::Unknown;
     }
 
     uint64_t resolve(const Network::Address::Ip* addr) const {
@@ -176,7 +176,7 @@ class PolicyHostMap : public Singleton::Instance,
       if (ipv6) {
         return resolve(ipv6->address());
       }
-      return ID::WORLD;
+      return ID::World;
     }
 
   protected:
@@ -195,7 +195,7 @@ class PolicyHostMap : public Singleton::Instance,
 
   uint64_t resolve(const Network::Address::Ip* addr) const {
     const ThreadLocalHostMap* hostmap = getHostMap();
-    return (hostmap != nullptr) ? hostmap->resolve(addr) : ID::UNKNOWN;
+    return (hostmap != nullptr) ? hostmap->resolve(addr) : ID::Unknown;
   }
 
   void logmaps(const std::string& msg) {

cilium/policy_id.h

@@ -6,8 +6,19 @@ namespace Envoy {
 namespace Cilium {
 
 enum ID : uint64_t {
-  UNKNOWN = 0,
-  WORLD = 2,
+  Unknown = 0,
+  Host = 1,
+  World = 2,
+  Unmanaged = 3,
+  Health = 4,
+  Init = 5,
+  RemoteNode = 6,
+  KubeApiServer = 7,
+  Ingress = 8,
+  WorldIPv4 = 9,
+  WorldIPv6 = 10,
+  EncryptedOverlay = 11,
+
   // LocalIdentityFlag is the bit in the numeric identity that identifies
   // a numeric identity to have local scope
   LocalIdentityFlag = 1 << 24,

cilium/socket_option_source_address.cc

@@ -3,6 +3,7 @@
 #include <sys/socket.h>
 
 #include <cstdint>
+#include <memory>
 #include <utility>
 #include <vector>
 
@@ -17,18 +18,20 @@
 
 #include "absl/numeric/int128.h"
 #include "cilium/filter_state_cilium_destination.h"
+#include "cilium/filter_state_cilium_policy.h"
+#include "cilium/policy_id.h"
 
 namespace Envoy {
 namespace Cilium {
 
 SourceAddressSocketOption::SourceAddressSocketOption(
-    uint32_t source_identity, int linger_time,
+    uint32_t source_identity, const std::weak_ptr<PolicyResolver>& policy_resolver, int linger_time,
     Network::Address::InstanceConstSharedPtr original_source_address,
     Network::Address::InstanceConstSharedPtr ipv4_source_address,
     Network::Address::InstanceConstSharedPtr ipv6_source_address,
     std::shared_ptr<CiliumDestinationFilterState> dest_fs)
-    : source_identity_(source_identity), linger_time_(linger_time),
-      original_source_address_(std::move(original_source_address)),
+    : source_identity_(source_identity), policy_resolver_(policy_resolver),
+      linger_time_(linger_time), original_source_address_(std::move(original_source_address)),
       ipv4_source_address_(std::move(ipv4_source_address)),
       ipv6_source_address_(std::move(ipv6_source_address)), dest_fs_(std::move(dest_fs)) {
   ENVOY_LOG(debug,
@@ -71,14 +74,51 @@ bool SourceAddressSocketOption::setOption(
   }
 
   if (source_address->ip() && dest_fs_->getDestinationAddress() &&
-      dest_fs_->getDestinationAddress()->ip() &&
-      source_address->ip()->addressAsString() ==
-          dest_fs_->getDestinationAddress()->ip()->addressAsString()) {
+      dest_fs_->getDestinationAddress()->ip()) {
+    // Skip using original source if hairpinning back to the source, as otherwise Linux would
+    // drop the packet
+    auto destination_ip = dest_fs_->getDestinationAddress()->ip();
+    const auto& destination_ip_str = destination_ip->addressAsString();
+    if (source_address->ip()->addressAsString() == destination_ip_str) {
+      ENVOY_LOG(trace,
+                "Skipping restore of local address on socket: {} - source address is same as "
+                "destination address {}",
+                socket.ioHandle().fdDoNotUse(), destination_ip_str);
+      return true;
+    }
+    // Also skip using original source if destination is a local pod or the local host,
+    // as otherwise there could be 5-tuple collisions, and the local host may not be able to
+    // send replies back to the proxy otherwise.
+    const auto resolver = policy_resolver_.lock();
+    if (!resolver) {
+      // No policy resolver
+      ENVOY_LOG(debug, "No policy resolver");
+      return false;
+    }
+
+    auto destination_identity = resolver->resolvePolicyId(destination_ip);
     ENVOY_LOG(trace,
-              "Skipping restore of local address on socket: {} - source address is same as "
-              "destination address {}",
-              socket.ioHandle().fdDoNotUse(), source_address->ip()->addressAsString());
-    return true;
+              "Socket {} destination address {} has security identity {}",
+              socket.ioHandle().fdDoNotUse(), destination_ip_str, destination_identity);
+
+    if (destination_identity == Cilium::ID::Host) {
+      ENVOY_LOG(
+          trace,
+          "Skipping restore of local address on socket: {} - destination is the local host {}",
+          socket.ioHandle().fdDoNotUse(), destination_ip_str);
+      return true;
+    }
+
+    if (resolver->exists(destination_ip_str)) {
+      ENVOY_LOG(trace,
+                "Skipping restore of local address on socket: {} - destination is a local pod {}",
+                socket.ioHandle().fdDoNotUse(), destination_ip_str);
+      return true;
+    }
+  } else if (source_address->ip()) {
+      ENVOY_LOG(debug,
+                "Destination address filter state for socket {} is not available",
+                socket.ioHandle().fdDoNotUse());
   }
 
   // Note: SO_LINGER option is set on the socket of the upstream connection.

cilium/socket_option_source_address.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <cstdint>
+#include <memory>
 #include <vector>
 
 #include "envoy/config/core/v3/socket_option.pb.h"
@@ -25,7 +26,8 @@ class SourceAddressSocketOption : public Network::Socket::Option,
                                   public Logger::Loggable<Logger::Id::filter> {
 public:
   SourceAddressSocketOption(
-      uint32_t source_identity, int linger_time = -1,
+      uint32_t source_identity, const std::weak_ptr<PolicyResolver>& policy_resolver,
+      int linger_time = -1,
       Network::Address::InstanceConstSharedPtr original_source_address = nullptr,
       Network::Address::InstanceConstSharedPtr ipv4_source_address = nullptr,
       Network::Address::InstanceConstSharedPtr ipv6_source_address = nullptr,
@@ -45,6 +47,11 @@ class SourceAddressSocketOption : public Network::Socket::Option,
   bool isSupported() const override { return true; }
 
   uint32_t source_identity_;
+
+  // need information about the destination policy/identity to decide if original source address can
+  // be used or not.
+  const std::weak_ptr<PolicyResolver> policy_resolver_;
+
   int linger_time_;
 
   Network::Address::InstanceConstSharedPtr original_source_address_;