feat: fall back to contact points on control reconnect (DRIVER-201) (#1065)

The driver reconnects on addresses it resolved once and never re-reads
DNS (#215). Default `fallback-to-original-contact-points` to true: once a
reconnection round has exhausted the live nodes, the retained unresolved
contact points are tried again, so their names are resolved afresh. Init
and fallback build that plan through one path. isControlNode matches the
pending node by reference and re-checks the identified node, because the
endpoint equals() was a blocking lookup per event with unresolved
placeholders in the plan. A new MockResolverIT case covers recovery to
new addresses.

Fixes #215
Refs: #890

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: nikagra

core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java

@@ -701,8 +701,17 @@ public enum DefaultDriverOption implements DriverOption {
   CONTROL_CONNECTION_AGREEMENT_WARN("advanced.control-connection.schema-agreement.warn-on-failure"),
 
   /**
-   * Whether to forcibly add original contact points held by MetadataManager to the reconnection
-   * plan, in case there is no live nodes available according to LBP. Experimental.
+   * Whether to append the original contact points held by MetadataManager to the control
+   * connection's reconnection plan, after the live nodes reported by the load balancing policy.
+   * Defaults to {@code true}.
+   *
+   * <p>This is the driver's DNS re-resolution path. A metadata node discovered from the peers table
+   * holds an address resolved once and never re-resolved; a contact point given as a hostname is
+   * kept unresolved (the default) and looked up again through Netty's resolver on each connect, so
+   * once the live nodes are exhausted the contact points find a cluster that moved to new
+   * addresses, as soon as the JVM's DNS cache ({@code networkaddress.cache.ttl}) has expired. A
+   * resolved contact point ({@code advanced.resolve-contact-points = true}, or a programmatic
+   * resolved address) is appended as it is and never re-resolved.
    *
    * <p>Value-type: boolean
    */

core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java

@@ -369,7 +369,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) {
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_INTERVAL, Duration.ofMillis(200));
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_TIMEOUT, Duration.ofSeconds(10));
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, true);
-    map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, false);
+    map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, true);
     map.put(TypedDriverOption.PREPARE_ON_ALL_NODES, true);
     map.put(TypedDriverOption.REPREPARE_ENABLED, true);
     map.put(TypedDriverOption.REPREPARE_CHECK_SYSTEM_TABLE, false);

core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java

@@ -600,7 +600,13 @@ public String toString() {
   public static final TypedDriverOption<Boolean> CONTROL_CONNECTION_AGREEMENT_WARN =
       new TypedDriverOption<>(
           DefaultDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, GenericType.BOOLEAN);
-  /** Whether to forcibly try original contacts if no live nodes are available */
+  /**
+   * Whether to append the original contact points to the control connection's reconnection plan,
+   * after the live nodes reported by the load balancing policy (defaults to {@code true}).
+   *
+   * <p>A contact point given as a hostname is kept unresolved and looked up again on each connect,
+   * so this is how the driver picks up changed DNS records once the live nodes are exhausted.
+   */
   public static final TypedDriverOption<Boolean> CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS =
       new TypedDriverOption<>(
           DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, GenericType.BOOLEAN);

core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java

@@ -61,7 +61,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Objects;
 import java.util.Queue;
 import java.util.WeakHashMap;
 import java.util.concurrent.CompletableFuture;
@@ -499,6 +498,34 @@ private void connect(
                                           node,
                                           new Exception("Channel closed during endpoint resolve")));
                                   connect(nodes, newErrors, onSuccess, onFailure);
+                                } else if (isUnusableForControl(resolvedNode)) {
+                                  // Events name the identified node, not the placeholder we
+                                  // dialled, so isControlNode() cannot match them without a
+                                  // blocking endpoint lookup. Re-checked here instead.
+                                  controlNodeState = ControlNodeState.NONE;
+                                  LOG.debug(
+                                      "[{}] New channel opened ({}) but {} is ignored, removed "
+                                          + "or forced down, closing and trying next node",
+                                      logPrefix,
+                                      channel,
+                                      resolvedNode);
+                                  // Null out before forceClose() so that onChannelClosed() does not
+                                  // start a redundant reconnection on top of the connect() retry
+                                  // below.
+                                  ControlConnection.this.channel = null;
+                                  channel.forceClose();
+                                  // Recorded like every other drop; on init this list is what the
+                                  // user sees. A reconnection discards it, leaving the debug log.
+                                  List<Entry<Node, Throwable>> newErrors =
+                                      (errors == null) ? new ArrayList<>() : errors;
+                                  newErrors.add(
+                                      new SimpleEntry<>(
+                                          node,
+                                          new Exception(
+                                              "Control node "
+                                                  + resolvedNode
+                                                  + " is ignored, removed or forced down")));
+                                  connect(nodes, newErrors, onSuccess, onFailure);
                                 } else {
                                   controlNodeState = new ControlNodeState(resolvedNode, null);
                                   context
@@ -687,12 +714,32 @@ private boolean isControlNode(Node eventNode) {
           && eventNode.getHostId().equals(state.current.getHostId())) {
         return true;
       }
-      if (state.current == null
-          && state.pending != null
-          && Objects.equals(eventNode.getEndPoint(), state.pending.getEndPoint())) {
-        return true;
-      }
-      return false;
+      // Reference identity, not endpoint equality: with unresolved contact points in the plan,
+      // DefaultEndPoint.equals resolves the unresolved side of a mixed pair -- a blocking lookup on
+      // this admin executor, during the DNS outage the fallback exists for. An event naming the
+      // metadata node for a host still being identified cannot match here at all; connect()
+      // re-checks that node with isUnusableForControl() once the resolve completes.
+      return state.current == null && state.pending != null && eventNode == state.pending;
+    }
+
+    /**
+     * Whether an event has already marked this node unusable for the control connection: ignored by
+     * the load balancing policy, or removed or forced down.
+     *
+     * <p>{@link #connect} runs the same checks on the node it dialled, before adopting the channel;
+     * this one runs a turn later, on the node that channel was identified as.
+     *
+     * <p>Keyed by node instance, so it cannot see a node {@code MetadataManager#registerNode}
+     * minted fresh for an unknown host id: that host is resurrected, not rejected. Detecting it
+     * would need a removal record keyed by host id -- rejecting host ids absent from the metadata
+     * would also reject the new nodes of a cluster that moved, which is what the fallback exists to
+     * recover.
+     */
+    private boolean isUnusableForControl(Node node) {
+      NodeState state = lastNodeState.get(node);
+      return lastNodeDistance.get(node) == NodeDistance.IGNORED
+          || (lastNodeState.containsKey(node)
+              && (state == null /*(removed)*/ || state == NodeState.FORCED_DOWN));
     }
 
     private void onDistanceEvent(DistanceEvent event) {

core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java

@@ -35,10 +35,7 @@
 import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet;
 import edu.umd.cs.findbugs.annotations.NonNull;
 import edu.umd.cs.findbugs.annotations.Nullable;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
@@ -164,9 +161,7 @@ private Queue<Node> newQueryPlan(
       case BEFORE_INIT:
       case DURING_INIT:
         // The contact points are not stored in the metadata yet:
-        List<Node> nodes = new ArrayList<>(context.getMetadataManager().getContactPoints());
-        Collections.shuffle(nodes);
-        return new ConcurrentLinkedQueue<>(nodes);
+        return contactPointPlan();
       case RUNNING:
         LoadBalancingPolicy policy = policiesPerProfile.get(executionProfileName);
         if (policy == null) {
@@ -190,22 +185,32 @@ public Queue<Node> newControlReconnectionQueryPlan() {
 
     // Before RUNNING, newQueryPlan() already built the plan from the contact points, so appending
     // them again would only duplicate every entry. Once RUNNING, the plan comes from the policy and
-    // is an immutable QueryPlan (add()/addAll() throw), so concatenate rather than mutate. The
-    // nodes retained by MetadataManager are appended, not fresh copies: their identity stays stable
-    // across reconnection rounds, and no throwaway node is minted per round.
+    // is an immutable QueryPlan (add()/addAll() throw), so concatenate rather than mutate.
     if (state == State.RUNNING
         && context
             .getConfig()
             .getDefaultProfile()
             .getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) {
-      Object[] contactNodes = context.getMetadataManager().getContactPoints().toArray();
-      ArrayUtils.shuffleHead(contactNodes, contactNodes.length);
-      return new CompositeQueryPlan(regularQueryPlan, new SimpleQueryPlan(contactNodes));
+      return new CompositeQueryPlan(regularQueryPlan, contactPointPlan());
     }
 
     return regularQueryPlan;
   }
 
+  /**
+   * The retained contact points, in random order: the whole plan before the session is initialized,
+   * and what the control connection falls back to once the policy's plan is exhausted. One path for
+   * both, so a fallback round tries exactly what the initial connection tried. The nodes are the
+   * ones {@code MetadataManager} retains, not fresh copies: their identity stays stable across
+   * reconnection rounds, and no throwaway node is minted per round.
+   */
+  @NonNull
+  private Queue<Node> contactPointPlan() {
+    Object[] contactNodes = context.getMetadataManager().getContactPoints().toArray();
+    ArrayUtils.shuffleHead(contactNodes, contactNodes.length);
+    return new SimpleQueryPlan(contactNodes);
+  }
+
   // when it comes in from the outside
   private void onNodeStateEvent(NodeStateEvent event) {
     eventFilter.accept(event);

core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java

@@ -190,6 +190,11 @@ public boolean wasImplicitContactPoint() {
    * they are never added to metadata and never exposed to user-facing APIs (events, {@link
    * com.datastax.oss.driver.api.core.metadata.Metadata#getNodes()}, or {@link
    * com.datastax.oss.driver.api.core.metadata.NodeStateListener} callbacks).
+   *
+   * <p>The metadata node stores {@code nodeInfo.getEndPoint()} as-is and never re-resolves it. A
+   * contact-point hostname is re-read only through the control connection's reconnection fallback
+   * ({@code advanced.control-connection.reconnection.fallback-to-original-contact-points}), which
+   * re-offers the retained, still-unresolved contact points to Netty's resolver on each connect.
    */
   public CompletionStage<Node> registerNode(NodeInfo nodeInfo) {
     Preconditions.checkNotNull(nodeInfo.getHostId(), "Cannot register node without hostId");

core/src/main/resources/reference.conf

@@ -2347,14 +2347,40 @@ datastax-java-driver {
     }
 
     reconnection {
-      # Whether to forcibly add original contact points held by MetadataManager to the reconnection plan,
-      # in case there is no live nodes available according to LBP.
-      # Experimental.
+      # Whether to append the original contact points to the control connection's reconnection
+      # plan, after the live nodes reported by the load balancing policy.
+      #
+      # This is the driver's DNS re-resolution path. A metadata node discovered from the peers table
+      # holds an address resolved once and never re-resolved. A contact point given as a hostname is
+      # kept unresolved (see `advanced.resolve-contact-points`) and looked up again through Netty's
+      # resolver on each connect, so once a reconnection round has exhausted the live nodes, the
+      # contact points find a cluster that moved to new addresses -- as soon as the JVM's DNS cache
+      # (`networkaddress.cache.ttl`) has expired. A resolved contact point
+      # (`resolve-contact-points = true`, or a programmatic resolved address) is appended as it is
+      # and never re-resolved.
+      #
+      # The cost: the contact points cannot be deduplicated against the live nodes at plan time
+      # (hostnames against resolved addresses), so a round that exhausts the live nodes then tries
+      # each contact point once more -- one connect (`advanced.connection.connect-timeout`) plus the
+      # identity read (`advanced.control-connection.timeout`) each, serially, with the control
+      # connection down meanwhile. Neither timeout covers the name lookup: Netty resolves before
+      # it connects, so an unanswered DNS query costs the JVM resolver's own timeout per contact
+      # point. Set this to false when the contact points are IP literals or their records never
+      # change, or to keep reconnection rounds short; the control connection then re-resolves
+      # nothing.
+      #
+      # A contact point is only an address, not a node: whichever node answers there is registered
+      # under its own host id, so a node the driver had removed can come back this way if it is
+      # still listening behind the record (a decommissioned node during a rolling replacement, for
+      # instance). That is inherent to recovering through contact points -- the driver cannot tell
+      # a node that should be gone from a genuinely new one at a moved address, and rejecting host
+      # ids it does not know would defeat the point. It corrects itself on the next node refresh
+      # that reaches a node still in the cluster.
       #
       # Required: yes
       # Modifiable at runtime: yes, the new value will be used for checks issued after the change.
       # Overridable in a profile: no
-      fallback-to-original-contact-points = false
+      fallback-to-original-contact-points = true
     }
   }