DRIVERS-3218 Avoid clearing the connection pool when the server connection rate limiter triggers

source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md

@@ -284,6 +284,15 @@ Endpoint. The pool has the following properties:
 - **Rate-limited:** A Pool MUST limit the number of [Connections](#connection) being
     [established](#establishing-a-connection-internal-implementation) concurrently via the **maxConnecting**
     [pool option](#connection-pool-options).
+- **Backoff-capable** A pool MUST be able to enter backoff mode. A pool will automatically enter backoff mode when a
+    connection checkout fails under conditions that indicate server overload. The rules for entering backoff mode are as
+    follows: - A network error or network timeout during the TCP handshake or the `hello` message for a new connection
+    MUST trigger the backoff state. - Other pending connections MUST not be canceled. - In the case of multiple pending
+    connections, the backoff attempt number MUST only be incremented once. This can be done by recording the state prior
+    to attempting the connection. While the Pool is in backoff, it exhibits the following behaviors: - **maxConnecting**
+    MUST be set to 1. - The Pool MUST wait for the backoff duration before another connection attempt. - A successful
+    heartbeat MUST NOT change the state of the pool. - A failed heartbeat MUST clear the pool. - A subsequent failed
+    connection MUST increase the backoff attempt. - A successful connection MUST return the Pool to ready state.
 
 ```typescript
 interface ConnectionPool {
@@ -314,12 +323,17 @@ interface ConnectionPool {
    *   - "ready":         The healthy state of the pool. It can service checkOut requests and create
    *                      connections in the background. The pool can be set to this state via the
    *                      ready() method.
+   * 
+   *   - "backoff":       The pool is in backoff state. MaxConnecting is set to 1 and the pool backoff period
+   *                      must be observed before attempting another connection.  A subsequent failed connection
+   *                      attempt increases the backoff duration.  The pool can be set to this state via the
+   *                      backoff() method.  
    *
    *   - "closed":        The pool is destroyed. No more Connections may ever be checked out nor any
    *                      created in the background. The pool can be set to this state via the close()
    *                      method. The pool cannot transition to any other state after being closed.
    */
-  state: "paused" | "ready" | "closed";
+  state: "paused" | "ready" | "backoff" | "closed";
 
   // Any of the following connection counts may be computed rather than
   // actually stored on the pool.
@@ -360,6 +374,11 @@ interface ConnectionPool {
    */
   clear(interruptInUseConnections: Optional<Boolean>): void;
 
+  /**
+   * Enter backoff mode or increase backoff amount if already in backoff mode.  Mark the pool as "backoff".
+   */
+  backoff(): void
+
   /**
    *  Mark the pool as "ready", allowing checkOuts to resume and connections to be created in the background.
    *  A pool can only transition from "paused" to "ready". A "closed" pool
@@ -829,6 +848,34 @@ interface PoolClearedEvent {
   interruptInUseConnections: Optional<Boolean>;
 }
 
+/**
+ *  Emitted when a Connection Pool is in backoff
+ */
+interface PoolBackoffEvent {
+  /**
+   *  The ServerAddress of the Endpoint the pool is attempting to connect to.
+   */
+  address: string;
+
+  /**
+   * The backoff attempt number.
+   * 
+   * The incrementing backoff attempt number.  This is included because 
+   * the backoff duration is non-deterministic due to jitter.
+   */
+  attempt: int64;
+
+  /**
+   * The duration the pool will not allow new connection establishments.
+   * 
+   * A driver MAY choose the type idiomatic to the driver.
+   * If the type chosen does not convey units, e.g., `int64`,
+   * then the driver MAY include units in the name, e.g., `durationMS`.
+   */
+  duration: Duration;
+}
+
+
 /**
  *  Emitted when a Connection Pool is closed
  */
@@ -1074,6 +1121,21 @@ placeholders as appropriate:
 
 > Connection pool for {{serverHost}}:{{serverPort}} cleared for serviceId {{serviceId}}
 
+#### Pool Backoff Message
+
+In addition to the common fields defined above, this message MUST contain the following key-value pairs:
+
+| Key        | Suggested Type | Value                        |
+| ---------- | -------------- | ---------------------------- |
+| message    | String         | "Connection pool in backoff" |
+| attempt    | Int            | The backoff attempt number.  |
+| durationMS | Int            | Int32/Int64/Double           |
+
+The unstructured form SHOULD be as follows, using the values defined in the structured format above to fill in
+placeholders as appropriate:
+
+> Connection pool for {{serverHost}}:{{serverPort}} in backoff. Attempt: {{attempt}}. Duration: {{durationMS}} ms
+
 #### Pool Closed Message
 
 In addition to the common fields defined above, this message MUST contain the following key-value pairs:
@@ -1375,6 +1437,8 @@ to close and remove from its pool a [Connection](#connection) which has unread e
 
 ## Changelog
 
+- 2025-XX-YY: Introduce "backoff" state.
+
 - 2025-01-22: Clarify durationMS in logs may be Int32/Int64/Double.
 
 - 2024-11-27: Relaxed the WaitQueue fairness requirement.

source/connection-monitoring-and-pooling/tests/cmap-format/README.md

@@ -74,6 +74,7 @@ Valid Unit Test Operations are the following:
     - `interruptInUseConnections`: Determines whether "in use" connections should be also interrupted
 - `pool.close()`: call `close` on Pool
 - `pool.ready()`: call `ready` on Pool
+- `pool.backoff()`: call `backoff` on Pool
 
 ## Integration Test Format
 

source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-connection-close.yml

@@ -0,0 +1,39 @@
+version: 1
+style: integration
+description: pool enters backoff on connection close
+runOn:
+  - minServerVersion: 4.9.0
+failPoint:
+  configureFailPoint: failCommand
+  mode:
+    times: 1
+  data:
+    failCommands:
+      - isMaster
+      - hello
+    closeConnection: true
+poolOptions:
+  minPoolSize: 0
+operations:
+  - name: ready
+  - name: start
+    target: thread1
+  - name: checkOut
+    thread: thread1
+  - name: waitForEvent
+    event: ConnectionCreated
+    count: 1
+  - name: waitForEvent
+    event: ConnectionCheckOutFailed
+    count: 1
+events:
+  - type: ConnectionCheckOutStarted
+  - type: ConnectionCreated
+  - type: ConnectionClosed
+  - type: ConnectionPoolBackoff
+  - type: ConnectionCheckOutFailed
+ignore:
+  - ConnectionCheckedIn
+  - ConnectionCheckedOut
+  - ConnectionPoolCreated
+  - ConnectionPoolReady

source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.yml

@@ -2,39 +2,37 @@ version: 1
 style: integration
 description: error during minPoolSize population clears pool
 runOn:
-  -
-    # required for appName in fail point
-    minServerVersion: "4.9.0"
+  - minServerVersion: 4.9.0
 failPoint:
   configureFailPoint: failCommand
-  # high amount to ensure not interfered with by monitor checks.
-  mode: { times: 50 }
+  mode: alwaysOn
   data:
-    failCommands: ["isMaster","hello"]
-    closeConnection: true
-    appName: "poolCreateMinSizeErrorTest"
+    failCommands:
+      - isMaster
+      - hello
+    errorCode: 18
+    appName: poolCreateMinSizeErrorTest
 poolOptions:
   minPoolSize: 1
   backgroundThreadIntervalMS: 50
-  appName: "poolCreateMinSizeErrorTest"
+  appName: poolCreateMinSizeErrorTest
 operations:
   - name: ready
   - name: waitForEvent
     event: ConnectionPoolCleared
     count: 1
-  # ensure pool doesn't start making new connections
   - name: wait
     ms: 200
 events:
   - type: ConnectionPoolReady
     address: 42
   - type: ConnectionCreated
     address: 42
-  - type: ConnectionPoolCleared
-    address: 42
   - type: ConnectionClosed
     address: 42
     connectionId: 42
     reason: error
+  - type: ConnectionPoolCleared
+    address: 42
 ignore:
   - ConnectionPoolCreated