From 4e6bfebe799174178cf103c64e08a905fba3ba9f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 28 Oct 2025 07:58:28 -0500 Subject: [PATCH 01/23] DRIVERS-318 Avoid clearing the connection pool when the server connection rate limiter triggers --- .../connection-monitoring-and-pooling.md | 61 ++++++++++++++++++- .../server-discovery-and-monitoring.md | 9 ++- .../server-monitoring.md | 3 +- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md index b6e5ebc81b..2c72191ce7 100644 --- a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md +++ b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md @@ -284,6 +284,15 @@ Endpoint. The pool has the following properties: - **Rate-limited:** A Pool MUST limit the number of [Connections](#connection) being [established](#establishing-a-connection-internal-implementation) concurrently via the **maxConnecting** [pool option](#connection-pool-options). +- **Backoff-capable** A pool MUST be able to enter backoff mode. A pool will automatically enter backoff mode when a + connection checkout fails under conditions that indicate server overload. While the Pool is in backoff, it exhibits + the following behaviors: + - **maxConnecting** is set to 1. + - The Pool waits for the backoff duration before another connection attempt. + - A successful heartbeat does NOT change the state of the pool. + - A failed heartbeat clears the pool. + - A subsequent failed connection will increase the backoff attempt. + - A successful connection will return the Pool to ready state. ```typescript interface ConnectionPool { @@ -314,12 +323,14 @@ interface ConnectionPool { * - "ready": The healthy state of the pool. It can service checkOut requests and create * connections in the background. The pool can be set to this state via the * ready() method. + * + * - "backoff": The pool is in backoff state. XXX * * - "closed": The pool is destroyed. No more Connections may ever be checked out nor any * created in the background. The pool can be set to this state via the close() * method. The pool cannot transition to any other state after being closed. */ - state: "paused" | "ready" | "closed"; + state: "paused" | "ready" | "backoff" | "closed"; // Any of the following connection counts may be computed rather than // actually stored on the pool. @@ -360,6 +371,11 @@ interface ConnectionPool { */ clear(interruptInUseConnections: Optional): void; + /** + * Enter backoff mode or increase backoff amount if already in backoff mode. Mark the pool as "backoff". + */ + backoff(): void + /** * Mark the pool as "ready", allowing checkOuts to resume and connections to be created in the background. * A pool can only transition from "paused" to "ready". A "closed" pool @@ -829,6 +845,34 @@ interface PoolClearedEvent { interruptInUseConnections: Optional; } +/** + * Emitted when a Connection Pool is in backoff + */ +interface PoolBackoffEvent { + /** + * The ServerAddress of the Endpoint the pool is attempting to connect to. + */ + address: string; + + /** + * The backoff attempt number. + * + * The incrementing backoff attempt number. This is included because + * the backoff duration is non-deterministic due to jitter. + */ + attempt: int64; + + /** + * The duration the pool will not allow new connection establishments. + * + * A driver MAY choose the type idiomatic to the driver. + * If the type chosen does not convey units, e.g., `int64`, + * then the driver MAY include units in the name, e.g., `durationMS`. + */ + duration: Duration; +} + + /** * Emitted when a Connection Pool is closed */ @@ -1074,6 +1118,21 @@ placeholders as appropriate: > Connection pool for {{serverHost}}:{{serverPort}} cleared for serviceId {{serviceId}} +#### Pool Backoff Message + +In addition to the common fields defined above, this message MUST contain the following key-value pairs: + +| Key | Suggested Type | Value | +| ---------- | -------------- | ---------------------------- | +| message | String | "Connection pool in backoff" | +| attempt | Int | The backoff attempt number. | +| durationMS | Int | Int32/Int64/Double | + +The unstructured form SHOULD be as follows, using the values defined in the structured format above to fill in +placeholders as appropriate: + +> Connection pool for {{serverHost}}:{{serverPort}} in backoff. Attempt: {{attempt}}. Duration: {{durationMS}} ms + #### Pool Closed Message In addition to the common fields defined above, this message MUST contain the following key-value pairs: diff --git a/source/server-discovery-and-monitoring/server-discovery-and-monitoring.md b/source/server-discovery-and-monitoring/server-discovery-and-monitoring.md index 206e8236c3..4e08a84d6a 100644 --- a/source/server-discovery-and-monitoring/server-discovery-and-monitoring.md +++ b/source/server-discovery-and-monitoring/server-discovery-and-monitoring.md @@ -1056,6 +1056,10 @@ def handleError(error): if isNotWritablePrimary(error): check failing server elif isNetworkError(error) or (not error.completedHandshake and (isNetworkTimeout(error) or isAuthError(error))): + # Ignore network errors and network timeout errors during TLS handshake or "hello" messages. + # These will be handled by the pool backoff. + if error.occurredDuringHello or error.occurredDuringTLSHandshake: + continue if type != LoadBalanced # Mark the server Unknown unknown = new ServerDescription(type=Unknown, error=error) @@ -1169,8 +1173,9 @@ TopologyType is ReplicaSetWithPrimary: referring to the table above we see the s [checkIfHasPrimary](#checkifhasprimary). The result is the TopologyType changes to ReplicaSetNoPrimary. See the test scenario called "Network error writing to primary". -The client MUST close all idle sockets in its connection pool for the server: if one socket is bad, it is likely that -all are. +The clients MUST NOT clear the connection pool when a connection TLS handshake or MongoDB handshake fail with network +errors or timeouts. If the network error or timeout occurs during TCP connection establishment, DNS lookup, or during +the authentication step, then client MUST close all idle sockets in its connection pool for the server. Clients MUST NOT request an immediate check of the server; since application sockets are used frequently, a network error likely means the server has just become unavailable, so an immediate refresh is likely to get a network error, diff --git a/source/server-discovery-and-monitoring/server-monitoring.md b/source/server-discovery-and-monitoring/server-monitoring.md index 5f1be7fa04..4b979fe327 100644 --- a/source/server-discovery-and-monitoring/server-monitoring.md +++ b/source/server-discovery-and-monitoring/server-monitoring.md @@ -482,7 +482,8 @@ When a monitor completes a successful check against a server, it MUST mark the c "ready", and doing so MUST be synchronized with the update to the topology (e.g. by marking the pool as ready in onServerDescriptionChanged). This is required to ensure a server does not get selected while its pool is still paused. See the [Connection Pool](../connection-monitoring-and-pooling/connection-monitoring-and-pooling.md#connection-pool) -definition in the CMAP specification for more details on marking the pool as "ready". +definition in the CMAP specification for more details on marking the pool as "ready". If the server is in "backoff" +state, the monitor MUST NOT mark the connection pool as "ready". ### Error handling From d830b12af54aaa70e2c43492d568187b7ce34fed Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 28 Oct 2025 08:01:57 -0500 Subject: [PATCH 02/23] update changelogs --- .../connection-monitoring-and-pooling.md | 2 ++ .../server-discovery-and-monitoring.md | 2 ++ source/server-discovery-and-monitoring/server-monitoring.md | 2 ++ 3 files changed, 6 insertions(+) diff --git a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md index 2c72191ce7..ff343b6d98 100644 --- a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md +++ b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md @@ -1434,6 +1434,8 @@ to close and remove from its pool a [Connection](#connection) which has unread e ## Changelog +- 2025-XX-YY: Introduce "backoff" state. + - 2025-01-22: Clarify durationMS in logs may be Int32/Int64/Double. - 2024-11-27: Relaxed the WaitQueue fairness requirement. diff --git a/source/server-discovery-and-monitoring/server-discovery-and-monitoring.md b/source/server-discovery-and-monitoring/server-discovery-and-monitoring.md index 4e08a84d6a..395d7bc795 100644 --- a/source/server-discovery-and-monitoring/server-discovery-and-monitoring.md +++ b/source/server-discovery-and-monitoring/server-discovery-and-monitoring.md @@ -2032,6 +2032,8 @@ oversaw the specification process. - 2025-01-22: Add error messages when a new primary is elected or a primary with a stale electionId or setVersion is discovered. +- 2025-XX-YY: Add support for pool backoff state. + ______________________________________________________________________ [^1]: "localThresholdMS" was called "secondaryAcceptableLatencyMS" in the Read Preferences Spec, before it was superseded diff --git a/source/server-discovery-and-monitoring/server-monitoring.md b/source/server-discovery-and-monitoring/server-monitoring.md index 4b979fe327..220052a481 100644 --- a/source/server-discovery-and-monitoring/server-monitoring.md +++ b/source/server-discovery-and-monitoring/server-monitoring.md @@ -972,6 +972,8 @@ outdated or inaccurate. ## Changelog +- 2025-XX-YY: Add support for pool "backoff" state. + - 2024-05-02: Migrated from reStructuredText to Markdown. - 2020-02-20: Extracted server monitoring from SDAM into this new spec. From c6c86265d8f8b291d57109c17e3b21406eea2d9f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 28 Oct 2025 10:11:35 -0500 Subject: [PATCH 03/23] address todo --- .../connection-monitoring-and-pooling.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md index ff343b6d98..e6c5b67d04 100644 --- a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md +++ b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md @@ -324,7 +324,10 @@ interface ConnectionPool { * connections in the background. The pool can be set to this state via the * ready() method. * - * - "backoff": The pool is in backoff state. XXX + * - "backoff": The pool is in backoff state. MaxConnecting is set to 1 and the pool backoff period + * must be observed before attempting another connection. A subsequent failed connection + * attempt increases the backoff duration. The pool can be set to this state via the + * backoff() method. * * - "closed": The pool is destroyed. No more Connections may ever be checked out nor any * created in the background. The pool can be set to this state via the close() From 70de63024b213dd9a80c77de0207065fd8a383a6 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 28 Oct 2025 10:25:17 -0500 Subject: [PATCH 04/23] update unified format and cmap-format --- .../tests/cmap-format/README.md | 1 + source/unified-test-format/schema-1.28.json | 1253 +++++++++++++++++ ...mapEvent-poolBackoffEvent-attempt-type.yml | 13 + .../unified-test-format.md | 12 + 4 files changed, 1279 insertions(+) create mode 100644 source/unified-test-format/schema-1.28.json create mode 100644 source/unified-test-format/tests/invalid/expectedCmapEvent-poolBackoffEvent-attempt-type.yml diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/README.md b/source/connection-monitoring-and-pooling/tests/cmap-format/README.md index ced96961f5..cbfbc95343 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/README.md +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/README.md @@ -74,6 +74,7 @@ Valid Unit Test Operations are the following: - `interruptInUseConnections`: Determines whether "in use" connections should be also interrupted - `pool.close()`: call `close` on Pool - `pool.ready()`: call `ready` on Pool +- `pool.backoff()`: call `backoff` on Pool ## Integration Test Format diff --git a/source/unified-test-format/schema-1.28.json b/source/unified-test-format/schema-1.28.json new file mode 100644 index 0000000000..607d4d54a0 --- /dev/null +++ b/source/unified-test-format/schema-1.28.json @@ -0,0 +1,1253 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema#", + "title": "Unified Test Format", + "type": "object", + "additionalProperties": false, + "required": [ + "description", + "schemaVersion", + "tests" + ], + "properties": { + "description": { + "type": "string" + }, + "schemaVersion": { + "$ref": "#/definitions/version" + }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/runOnRequirement" + } + }, + "createEntities": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/entity" + } + }, + "initialData": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/collectionData" + } + }, + "tests": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/test" + } + }, + "_yamlAnchors": { + "type": "object", + "additionalProperties": true + } + }, + "definitions": { + "version": { + "type": "string", + "pattern": "^[0-9]+(\\.[0-9]+){1,2}$" + }, + "runOnRequirement": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "maxServerVersion": { + "$ref": "#/definitions/version" + }, + "minServerVersion": { + "$ref": "#/definitions/version" + }, + "topologies": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "single", + "replicaset", + "sharded", + "sharded-replicaset", + "load-balanced" + ] + } + }, + "serverless": { + "type": "string", + "enum": [ + "require", + "forbid", + "allow" + ] + }, + "serverParameters": { + "type": "object", + "minProperties": 1 + }, + "auth": { + "type": "boolean" + }, + "authMechanism": { + "type": "string" + }, + "csfle": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "minLibmongocryptVersion": { + "$ref": "#/definitions/version" + } + } + } + ] + } + } + }, + "entity": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "client": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "uriOptions": { + "type": "object" + }, + "useMultipleMongoses": { + "type": "boolean" + }, + "observeEvents": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent", + "poolCreatedEvent", + "poolReadyEvent", + "poolClearedEvent", + "poolBackoffEvent", + "poolClosedEvent", + "connectionCreatedEvent", + "connectionReadyEvent", + "connectionClosedEvent", + "connectionCheckOutStartedEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedOutEvent", + "connectionCheckedInEvent", + "serverDescriptionChangedEvent", + "topologyDescriptionChangedEvent", + "topologyOpeningEvent", + "topologyClosedEvent" + ] + } + }, + "ignoreCommandMonitoringEvents": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "storeEventsAsEntities": { + "deprecated": true, + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/storeEventsAsEntity" + } + }, + "observeLogMessages": { + "type": "object", + "minProperties": 1, + "additionalProperties": false, + "properties": { + "command": { + "$ref": "#/definitions/logSeverityLevel" + }, + "topology": { + "$ref": "#/definitions/logSeverityLevel" + }, + "serverSelection": { + "$ref": "#/definitions/logSeverityLevel" + }, + "connection": { + "$ref": "#/definitions/logSeverityLevel" + } + } + }, + "observeTracingMessages": { + "type": "object", + "additionalProperties": false, + "properties": { + "enableCommandPayload": { + "type": "boolean" + } + } + }, + "serverApi": { + "$ref": "#/definitions/serverApi" + }, + "observeSensitiveCommands": { + "type": "boolean" + }, + "autoEncryptOpts": { + "type": "object", + "additionalProperties": false, + "required": [ + "keyVaultNamespace", + "kmsProviders" + ], + "properties": { + "keyVaultNamespace": { + "type": "string" + }, + "bypassAutoEncryption": { + "type": "boolean" + }, + "kmsProviders": { + "$ref": "#/definitions/kmsProviders" + }, + "schemaMap": { + "type": "object", + "additionalProperties": { + "type": "object" + } + }, + "extraOptions": { + "type": "object" + }, + "encryptedFieldsMap": { + "type": "object", + "additionalProperties": { + "type": "object" + } + }, + "bypassQueryAnalysis": { + "type": "boolean" + }, + "keyExpirationMS": { + "type": "integer" + } + } + }, + "awaitMinPoolSizeMS": { + "type": "integer" + } + } + }, + "clientEncryption": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "clientEncryptionOpts" + ], + "properties": { + "id": { + "type": "string" + }, + "clientEncryptionOpts": { + "$ref": "#/definitions/clientEncryptionOpts" + } + } + }, + "database": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "client", + "databaseName" + ], + "properties": { + "id": { + "type": "string" + }, + "client": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "databaseOptions": { + "$ref": "#/definitions/collectionOrDatabaseOptions" + } + } + }, + "collection": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "database", + "collectionName" + ], + "properties": { + "id": { + "type": "string" + }, + "database": { + "type": "string" + }, + "collectionName": { + "type": "string" + }, + "collectionOptions": { + "$ref": "#/definitions/collectionOrDatabaseOptions" + } + } + }, + "session": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "client" + ], + "properties": { + "id": { + "type": "string" + }, + "client": { + "type": "string" + }, + "sessionOptions": { + "type": "object" + } + } + }, + "bucket": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "database" + ], + "properties": { + "id": { + "type": "string" + }, + "database": { + "type": "string" + }, + "bucketOptions": { + "type": "object" + } + } + }, + "thread": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + } + } + } + } + }, + "logComponent": { + "type": "string", + "enum": [ + "command", + "topology", + "serverSelection", + "connection" + ] + }, + "spanComponent": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "attributes" + ], + "properties": { + "name": { + "type": "string" + }, + "attributes": { + "type": "object" + }, + "nested": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/spanComponent" + } + } + } + }, + "logSeverityLevel": { + "type": "string", + "enum": [ + "emergency", + "alert", + "critical", + "error", + "warning", + "notice", + "info", + "debug", + "trace" + ] + }, + "clientEncryptionOpts": { + "type": "object", + "additionalProperties": false, + "required": [ + "keyVaultClient", + "keyVaultNamespace", + "kmsProviders" + ], + "properties": { + "keyVaultClient": { + "type": "string" + }, + "keyVaultNamespace": { + "type": "string" + }, + "kmsProviders": { + "$ref": "#/definitions/kmsProviders" + }, + "keyExpirationMS": { + "type": "integer" + } + } + }, + "kmsProviders": { + "$defs": { + "stringOrPlaceholder": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "$$placeholder" + ], + "properties": { + "$$placeholder": {} + } + } + ] + } + }, + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^aws(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "accessKeyId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "secretAccessKey": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "sessionToken": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^azure(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "tenantId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "clientId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "clientSecret": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "identityPlatformEndpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^gcp(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "email": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "privateKey": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "endpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^kmip(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "endpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^local(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "key": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + } + } + }, + "storeEventsAsEntity": { + "deprecated": true, + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "events" + ], + "properties": { + "id": { + "type": "string" + }, + "events": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "PoolCreatedEvent", + "PoolReadyEvent", + "PoolClearedEvent", + "PoolBackoffEvent", + "PoolClosedEvent", + "ConnectionCreatedEvent", + "ConnectionReadyEvent", + "ConnectionClosedEvent", + "ConnectionCheckOutStartedEvent", + "ConnectionCheckOutFailedEvent", + "ConnectionCheckedOutEvent", + "ConnectionCheckedInEvent", + "CommandStartedEvent", + "CommandSucceededEvent", + "CommandFailedEvent", + "ServerDescriptionChangedEvent", + "TopologyDescriptionChangedEvent" + ] + } + } + } + }, + "collectionData": { + "type": "object", + "additionalProperties": false, + "required": [ + "collectionName", + "databaseName", + "documents" + ], + "properties": { + "collectionName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "createOptions": { + "type": "object", + "properties": { + "writeConcern": false + } + }, + "documents": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "expectedEventsForClient": { + "type": "object", + "additionalProperties": false, + "required": [ + "client", + "events" + ], + "properties": { + "client": { + "type": "string" + }, + "eventType": { + "type": "string", + "enum": [ + "command", + "cmap", + "sdam" + ] + }, + "events": { + "type": "array" + }, + "ignoreExtraEvents": { + "type": "boolean" + } + }, + "oneOf": [ + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "command" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCommandEvent" + } + } + } + }, + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "cmap" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCmapEvent" + } + } + } + }, + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "sdam" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedSdamEvent" + } + } + } + }, + { + "additionalProperties": false, + "properties": { + "client": { + "type": "string" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCommandEvent" + } + }, + "ignoreExtraEvents": { + "type": "boolean" + } + } + } + ] + }, + "expectedCommandEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "commandStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { + "type": "object" + }, + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" + } + } + }, + "commandSucceededEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reply": { + "type": "object" + }, + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" + } + } + }, + "commandFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" + } + } + } + } + }, + "expectedCmapEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "poolCreatedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "poolReadyEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "poolClearedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "hasServiceId": { + "type": "boolean" + }, + "interruptInUseConnections": { + "type": "boolean" + } + } + }, + "poolBackoffEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "attempt": { + "type": "integer" + } + } + }, + "poolClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCreatedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionReadyEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + } + } + }, + "connectionCheckOutStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCheckOutFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + } + } + }, + "connectionCheckedOutEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCheckedInEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + } + } + }, + "expectedSdamEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "serverDescriptionChangedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "previousDescription": { + "$ref": "#/definitions/serverDescription" + }, + "newDescription": { + "$ref": "#/definitions/serverDescription" + } + } + }, + "topologyDescriptionChangedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "previousDescription": { + "$ref": "#/definitions/topologyDescription" + }, + "newDescription": { + "$ref": "#/definitions/topologyDescription" + } + } + }, + "serverHeartbeatStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" + } + } + }, + "serverHeartbeatSucceededEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" + } + } + }, + "serverHeartbeatFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" + } + } + }, + "topologyOpeningEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "topologyClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + } + } + }, + "serverDescription": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "Standalone", + "Mongos", + "PossiblePrimary", + "RSPrimary", + "RSSecondary", + "RSOther", + "RSArbiter", + "RSGhost", + "LoadBalancer", + "Unknown" + ] + } + } + }, + "topologyDescription": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "Single", + "Unknown", + "ReplicaSetNoPrimary", + "ReplicaSetWithPrimary", + "Sharded", + "LoadBalanced" + ] + } + } + }, + "expectedLogMessagesForClient": { + "type": "object", + "additionalProperties": false, + "required": [ + "client", + "messages" + ], + "properties": { + "client": { + "type": "string" + }, + "messages": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedLogMessage" + } + }, + "ignoreExtraMessages": { + "type": "boolean" + }, + "ignoreMessages": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedLogMessage" + } + } + } + }, + "expectTracingMessagesForClient": { + "additionalProperties": false, + "type": "object", + "required": [ + "client", + "spans" + ], + "properties": { + "client": { + "type": "string" + }, + "ignoreExtraSpans": { + "type": "boolean" + }, + "spans": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/spanComponent" + } + } + } + }, + "expectedLogMessage": { + "type": "object", + "additionalProperties": false, + "required": [ + "level", + "component", + "data" + ], + "properties": { + "level": { + "$ref": "#/definitions/logSeverityLevel" + }, + "component": { + "$ref": "#/definitions/logComponent" + }, + "data": { + "type": "object" + }, + "failureIsRedacted": { + "type": "boolean" + } + } + }, + "collectionOrDatabaseOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "readConcern": { + "type": "object" + }, + "readPreference": { + "type": "object" + }, + "writeConcern": { + "type": "object" + }, + "timeoutMS": { + "type": "integer" + } + } + }, + "serverApi": { + "type": "object", + "additionalProperties": false, + "required": [ + "version" + ], + "properties": { + "version": { + "type": "string" + }, + "strict": { + "type": "boolean" + }, + "deprecationErrors": { + "type": "boolean" + } + } + }, + "operation": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "object" + ], + "properties": { + "name": { + "type": "string" + }, + "object": { + "type": "string" + }, + "arguments": { + "type": "object" + }, + "ignoreResultAndError": { + "type": "boolean" + }, + "expectError": { + "$ref": "#/definitions/expectedError" + }, + "expectResult": {}, + "saveResultAsEntity": { + "type": "string" + } + }, + "allOf": [ + { + "not": { + "required": [ + "expectError", + "expectResult" + ] + } + }, + { + "not": { + "required": [ + "expectError", + "saveResultAsEntity" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "expectResult" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "expectError" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "saveResultAsEntity" + ] + } + } + ] + }, + "expectedError": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "isError": { + "type": "boolean", + "const": true + }, + "isClientError": { + "type": "boolean" + }, + "isTimeoutError": { + "type": "boolean" + }, + "errorContains": { + "type": "string" + }, + "errorCode": { + "type": "integer" + }, + "errorCodeName": { + "type": "string" + }, + "errorLabelsContain": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "errorLabelsOmit": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "writeErrors": { + "type": "object" + }, + "writeConcernErrors": { + "type": "array", + "items": { + "type": "object" + } + }, + "errorResponse": { + "type": "object" + }, + "expectResult": {} + } + }, + "test": { + "type": "object", + "additionalProperties": false, + "required": [ + "description", + "operations" + ], + "properties": { + "description": { + "type": "string" + }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/runOnRequirement" + } + }, + "skipReason": { + "type": "string" + }, + "operations": { + "type": "array", + "items": { + "$ref": "#/definitions/operation" + } + }, + "expectEvents": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/expectedEventsForClient" + } + }, + "expectLogMessages": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/expectedLogMessagesForClient" + } + }, + "expectTracingMessages": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/expectTracingMessagesForClient" + } + }, + "outcome": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/collectionData" + } + } + } + } + } +} diff --git a/source/unified-test-format/tests/invalid/expectedCmapEvent-poolBackoffEvent-attempt-type.yml b/source/unified-test-format/tests/invalid/expectedCmapEvent-poolBackoffEvent-attempt-type.yml new file mode 100644 index 0000000000..e095db151a --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedCmapEvent-poolBackoffEvent-attempt-type.yml @@ -0,0 +1,13 @@ +description: expectedCmapEvent-poolClearedEvent-hasServiceId-type + +schemaVersion: '1.27' + +tests: + - description: foo + operations: [] + expectEvents: + - client: client0 + eventType: cmap + events: + - poolBackoffEvent: + attempt: foo diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 59700ff510..24eb02b457 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -478,6 +478,7 @@ The structure of this object is as follows: - [poolCreatedEvent](#expectedEvent_poolCreatedEvent) - [poolReadyEvent](#expectedEvent_poolReadyEvent) - [poolClearedEvent](#expectedEvent_poolClearedEvent) + - [poolBackoffEvent](#expectedEvent_poolBackoffEvent) - [poolClosedEvent](#expectedEvent_poolClosedEvent) - [connectionCreatedEvent](#expectedEvent_connectionCreatedEvent) - [connectionReadyEvent](#expectedEvent_connectionReadyEvent) @@ -1082,6 +1083,15 @@ The structure of this object is as follows: - `interruptInUseConnections`: Optional boolean. If specified, test runners MUST assert that the field is set and matches this value. + + +- `poolBackoffEvent`: Optional object. Assertions for one or more + [PoolBackoffEvent](../connection-monitoring-and-pooling/connection-monitoring-and-pooling.md#events) fields. + + The structure of this object is as follows: + + - `attempt`: Optional integer. If specified, test runners MUST assert that the field is set and matches this value. + - `poolClosedEvent`: Optional object. If present, this object MUST be an empty document as all fields in this event are @@ -3459,6 +3469,8 @@ other specs *and* collating spec changes developed in parallel or during the sam ## Changelog +- 2025-XX-YY: **Scheam version 1.28.** Add `poolBackoffEvent`. + - 2025-09-17: **Schema version 1.27.** Add `observeTracingMessages` configuration for clients and `expectTracingMessages` for test expectations. This allows From 3d27dd4495603b0cb0d16ec49824ed1d0e965299 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 28 Oct 2025 10:35:48 -0500 Subject: [PATCH 05/23] update schemas --- source/unified-test-format/schema-latest.json | 11 +++++++++ ...apEvent-poolBackoffEvent-attempt-type.json | 23 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 source/unified-test-format/tests/invalid/expectedCmapEvent-poolBackoffEvent-attempt-type.json diff --git a/source/unified-test-format/schema-latest.json b/source/unified-test-format/schema-latest.json index d195ba5af0..607d4d54a0 100644 --- a/source/unified-test-format/schema-latest.json +++ b/source/unified-test-format/schema-latest.json @@ -149,6 +149,7 @@ "poolCreatedEvent", "poolReadyEvent", "poolClearedEvent", + "poolBackoffEvent", "poolClosedEvent", "connectionCreatedEvent", "connectionReadyEvent", @@ -553,6 +554,7 @@ "PoolCreatedEvent", "PoolReadyEvent", "PoolClearedEvent", + "PoolBackoffEvent", "PoolClosedEvent", "ConnectionCreatedEvent", "ConnectionReadyEvent", @@ -790,6 +792,15 @@ } } }, + "poolBackoffEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "attempt": { + "type": "integer" + } + } + }, "poolClosedEvent": { "type": "object", "additionalProperties": false, diff --git a/source/unified-test-format/tests/invalid/expectedCmapEvent-poolBackoffEvent-attempt-type.json b/source/unified-test-format/tests/invalid/expectedCmapEvent-poolBackoffEvent-attempt-type.json new file mode 100644 index 0000000000..bd76644f2e --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedCmapEvent-poolBackoffEvent-attempt-type.json @@ -0,0 +1,23 @@ +{ + "description": "expectedCmapEvent-poolClearedEvent-hasServiceId-type", + "schemaVersion": "1.27", + "tests": [ + { + "description": "foo", + "operations": [], + "expectEvents": [ + { + "client": "client0", + "eventType": "cmap", + "events": [ + { + "poolBackoffEvent": { + "attempt": "foo" + } + } + ] + } + ] + } + ] +} From b72e044735c800a6ad65ef9ce244c1bc54f31b45 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Oct 2025 12:40:11 -0500 Subject: [PATCH 06/23] address review --- source/server-discovery-and-monitoring/server-monitoring.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/server-discovery-and-monitoring/server-monitoring.md b/source/server-discovery-and-monitoring/server-monitoring.md index 220052a481..58fa43c7ba 100644 --- a/source/server-discovery-and-monitoring/server-monitoring.md +++ b/source/server-discovery-and-monitoring/server-monitoring.md @@ -482,8 +482,8 @@ When a monitor completes a successful check against a server, it MUST mark the c "ready", and doing so MUST be synchronized with the update to the topology (e.g. by marking the pool as ready in onServerDescriptionChanged). This is required to ensure a server does not get selected while its pool is still paused. See the [Connection Pool](../connection-monitoring-and-pooling/connection-monitoring-and-pooling.md#connection-pool) -definition in the CMAP specification for more details on marking the pool as "ready". If the server is in "backoff" -state, the monitor MUST NOT mark the connection pool as "ready". +definition in the CMAP specification for more details on marking the pool as "ready". If the pool is in "backoff" state, +the monitor MUST NOT mark the connection pool as "ready". ### Error handling From 20cc9614aaf8c179a06162457399083da4faaa90 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Oct 2025 12:52:33 -0500 Subject: [PATCH 07/23] clarify backoff logic --- .../connection-monitoring-and-pooling.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md index e6c5b67d04..7515325c3e 100644 --- a/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md +++ b/source/connection-monitoring-and-pooling/connection-monitoring-and-pooling.md @@ -285,14 +285,14 @@ Endpoint. The pool has the following properties: [established](#establishing-a-connection-internal-implementation) concurrently via the **maxConnecting** [pool option](#connection-pool-options). - **Backoff-capable** A pool MUST be able to enter backoff mode. A pool will automatically enter backoff mode when a - connection checkout fails under conditions that indicate server overload. While the Pool is in backoff, it exhibits - the following behaviors: - - **maxConnecting** is set to 1. - - The Pool waits for the backoff duration before another connection attempt. - - A successful heartbeat does NOT change the state of the pool. - - A failed heartbeat clears the pool. - - A subsequent failed connection will increase the backoff attempt. - - A successful connection will return the Pool to ready state. + connection checkout fails under conditions that indicate server overload. The rules for entering backoff mode are as + follows: - A network error or network timeout during the TCP handshake or the `hello` message for a new connection + MUST trigger the backoff state. - Other pending connections MUST not be canceled. - In the case of multiple pending + connections, the backoff attempt number MUST only be incremented once. This can be done by recording the state prior + to attempting the connection. While the Pool is in backoff, it exhibits the following behaviors: - **maxConnecting** + MUST be set to 1. - The Pool MUST wait for the backoff duration before another connection attempt. - A successful + heartbeat MUST NOT change the state of the pool. - A failed heartbeat MUST clear the pool. - A subsequent failed + connection MUST increase the backoff attempt. - A successful connection MUST return the Pool to ready state. ```typescript interface ConnectionPool { From dc6797b715f6dc66906629d1488d16bda12c55cd Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Oct 2025 21:32:12 -0500 Subject: [PATCH 08/23] add tests --- .../pool-backoff-connection-close.json | 72 +++ .../pool-backoff-connection-close.yml | 39 ++ .../pool-create-min-size-error.json | 14 +- .../pool-create-min-size-error.yml | 22 +- .../tests/logging/connection-logging.json | 362 ++++++++++- .../tests/logging/connection-logging.yml | 565 ++++++++++++++---- .../tests/sdam-error-handling.json | 44 +- .../tests/sdam-error-handling.yml | 226 +++---- .../unified/backoff-heartbeat-failure.json | 172 ++++++ .../unified/backoff-heartbeat-failure.yml | 94 +++ .../unified/backoff-heartbeat-success.json | 180 ++++++ .../unified/backoff-heartbeat-success.yml | 100 ++++ .../unified/backoff-network-error-fail.json | 302 ++++++++++ .../unified/backoff-network-error-fail.yml | 181 ++++++ .../tests/unified/backoff-network-error.json | 204 +++++++ .../tests/unified/backoff-network-error.yml | 113 ++++ .../backoff-network-timeout-error.json | 201 +++++++ .../unified/backoff-network-timeout-error.yml | 112 ++++ 18 files changed, 2741 insertions(+), 262 deletions(-) create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-connection-close.json create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-connection-close.yml create mode 100644 source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json create mode 100644 source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml create mode 100644 source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json create mode 100644 source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml create mode 100644 source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json create mode 100644 source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml create mode 100644 source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json create mode 100644 source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml create mode 100644 source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json create mode 100644 source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-connection-close.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-connection-close.json new file mode 100644 index 0000000000..571cd9f769 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-connection-close.json @@ -0,0 +1,72 @@ +{ + "version": 1, + "style": "integration", + "description": "pool enters backoff on connection close", + "runOn": [ + { + "minServerVersion": "4.9.0" + } + ], + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "closeConnection": true + } + }, + "poolOptions": { + "minPoolSize": 0 + }, + "operations": [ + { + "name": "ready" + }, + { + "name": "start", + "target": "thread1" + }, + { + "name": "checkOut", + "thread": "thread1" + }, + { + "name": "waitForEvent", + "event": "ConnectionCreated", + "count": 1 + }, + { + "name": "waitForEvent", + "event": "ConnectionCheckOutFailed", + "count": 1 + } + ], + "events": [ + { + "type": "ConnectionCheckOutStarted" + }, + { + "type": "ConnectionCreated" + }, + { + "type": "ConnectionClosed" + }, + { + "type": "ConnectionPoolBackoff" + }, + { + "type": "ConnectionCheckOutFailed" + } + ], + "ignore": [ + "ConnectionCheckedIn", + "ConnectionCheckedOut", + "ConnectionPoolCreated", + "ConnectionPoolReady" + ] +} diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-connection-close.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-connection-close.yml new file mode 100644 index 0000000000..20a086007e --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-connection-close.yml @@ -0,0 +1,39 @@ +version: 1 +style: integration +description: pool enters backoff on connection close +runOn: + - minServerVersion: 4.9.0 +failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - isMaster + - hello + closeConnection: true +poolOptions: + minPoolSize: 0 +operations: + - name: ready + - name: start + target: thread1 + - name: checkOut + thread: thread1 + - name: waitForEvent + event: ConnectionCreated + count: 1 + - name: waitForEvent + event: ConnectionCheckOutFailed + count: 1 +events: + - type: ConnectionCheckOutStarted + - type: ConnectionCreated + - type: ConnectionClosed + - type: ConnectionPoolBackoff + - type: ConnectionCheckOutFailed +ignore: + - ConnectionCheckedIn + - ConnectionCheckedOut + - ConnectionPoolCreated + - ConnectionPoolReady diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.json index 509b2a2356..da9357b963 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.json +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.json @@ -9,15 +9,13 @@ ], "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 50 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "isMaster", "hello" ], - "closeConnection": true, + "errorCode": 18, "appName": "poolCreateMinSizeErrorTest" } }, @@ -49,15 +47,15 @@ "type": "ConnectionCreated", "address": 42 }, - { - "type": "ConnectionPoolCleared", - "address": 42 - }, { "type": "ConnectionClosed", "address": 42, "connectionId": 42, "reason": "error" + }, + { + "type": "ConnectionPoolCleared", + "address": 42 } ], "ignore": [ diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.yml index f43c4ee154..b658193710 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.yml +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.yml @@ -2,27 +2,25 @@ version: 1 style: integration description: error during minPoolSize population clears pool runOn: - - - # required for appName in fail point - minServerVersion: "4.9.0" + - minServerVersion: 4.9.0 failPoint: configureFailPoint: failCommand - # high amount to ensure not interfered with by monitor checks. - mode: { times: 50 } + mode: alwaysOn data: - failCommands: ["isMaster","hello"] - closeConnection: true - appName: "poolCreateMinSizeErrorTest" + failCommands: + - isMaster + - hello + errorCode: 18 + appName: poolCreateMinSizeErrorTest poolOptions: minPoolSize: 1 backgroundThreadIntervalMS: 50 - appName: "poolCreateMinSizeErrorTest" + appName: poolCreateMinSizeErrorTest operations: - name: ready - name: waitForEvent event: ConnectionPoolCleared count: 1 - # ensure pool doesn't start making new connections - name: wait ms: 200 events: @@ -30,11 +28,11 @@ events: address: 42 - type: ConnectionCreated address: 42 - - type: ConnectionPoolCleared - address: 42 - type: ConnectionClosed address: 42 connectionId: 42 reason: error + - type: ConnectionPoolCleared + address: 42 ignore: - ConnectionPoolCreated diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json index 72103b3cab..207618b4a6 100644 --- a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json +++ b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json @@ -355,7 +355,7 @@ "failCommands": [ "saslContinue" ], - "closeConnection": true, + "errorCode": 18, "appName": "clientAppName" } } @@ -368,7 +368,7 @@ "filter": {} }, "expectError": { - "isClientError": true + "isError": true } } ], @@ -446,6 +446,59 @@ } } }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection closed", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + }, + "reason": "An error occurred while using the connection", + "error": { + "$$exists": true + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection checkout failed", + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + }, + "reason": "An error occurred while trying to establish a new connection", + "error": { + "$$exists": true + }, + "durationMS": { + "$$type": [ + "double", + "int", + "long" + ] + } + } + }, { "level": "debug", "component": "connection", @@ -461,6 +514,167 @@ ] } } + } + ] + } + ] + }, + { + "description": "Connection enters backoff on closed connection", + "runOnRequirements": [ + { + "auth": true, + "minServerVersion": "4.4" + } + ], + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "uriOptions": { + "retryReads": true, + "appname": "clientAppName", + "heartbeatFrequencyMS": 5000 + }, + "observeEvents": [ + "serverHeartbeatSucceededEvent" + ], + "observeLogMessages": { + "connection": "debug" + } + } + }, + { + "database": { + "id": "database0", + "client": "client", + "databaseName": "ci-tests" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverHeartbeatSucceededEvent": {} + }, + "count": 1 + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "closeConnection": true, + "appName": "clientAppName" + } + } + } + }, + { + "object": "database0", + "name": "runCommand", + "arguments": { + "command": { + "find": "test" + }, + "commandName": "find" + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection pool created", + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection pool ready", + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection checkout started", + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection created", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } }, { "level": "debug", @@ -488,6 +702,32 @@ } } }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection pool backoff", + "serverHost": { + "$$type": "string" + }, + "durationMS": { + "$$type": "int" + }, + "attempt": { + "$$type": "int" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + }, + "reason": "Connection pool is in backoff", + "error": { + "$$exists": true + } + } + }, { "level": "debug", "component": "connection", @@ -514,6 +754,124 @@ ] } } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection checkout started", + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection created", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, +{ + "level": "debug", + "component": "connection", + "data": { + "message": "Connection ready", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + }, + "durationMS": { + "$$type": [ + "double", + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection checked out", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + }, + "durationMS": { + "$$type": [ + "double", + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection checked in", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } } ] } diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml index 49868a062e..fc351e47ad 100644 --- a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml +++ b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml @@ -1,224 +1,545 @@ -description: "connection-logging" - +description: connection-logging schemaVersion: "1.13" - runOnRequirements: - topologies: - - single # The number of log messages is different for each topology since there is a connection pool per host. - + - single createEntities: - client: - id: &failPointClient failPointClient - + id: failPointClient tests: - - description: "Create a client, run a command, and close the client" + - description: Create a client, run a command, and close the client operations: - name: createEntities object: testRunner arguments: entities: - client: - id: &client client + id: client observeLogMessages: connection: debug - name: listDatabases - object: *client + object: client arguments: filter: {} - name: close - object: *client + object: client expectLogMessages: - - client: *client + - client: client messages: - level: debug component: connection data: - message: "Connection pool created" - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - + message: Connection pool created + serverHost: + $$type: string + serverPort: + $$type: + - int + - long - level: debug component: connection data: - message: "Connection pool ready" - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - + message: Connection pool ready + serverHost: + $$type: string + serverPort: + $$type: + - int + - long - level: debug component: connection data: - message: "Connection checkout started" - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - + message: Connection checkout started + serverHost: + $$type: string + serverPort: + $$type: + - int + - long - level: debug component: connection data: - message: "Connection created" - driverConnectionId: { $$type: [int, long] } - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - + message: Connection created + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long - level: debug component: connection data: - message: "Connection ready" - driverConnectionId: { $$type: [int, long] } - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - durationMS: { $$type: [double, int, long] } - + message: Connection ready + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + durationMS: + $$type: + - double + - int + - long - level: debug component: connection data: - message: "Connection checked out" - driverConnectionId: { $$type: [int, long] } - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - durationMS: { $$type: [double, int, long] } - + message: Connection checked out + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + durationMS: + $$type: + - double + - int + - long - level: debug component: connection data: - message: "Connection checked in" - driverConnectionId: { $$type: [int, long] } - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - - # The next three expected logs are for ending a session. + message: Connection checked in + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long - level: debug component: connection data: - message: "Connection checkout started" - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - + message: Connection checkout started + serverHost: + $$type: string + serverPort: + $$type: + - int + - long - level: debug component: connection data: - message: "Connection checked out" - driverConnectionId: { $$type: [int, long] } - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - durationMS: { $$type: [double, int, long] } - + message: Connection checked out + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + durationMS: + $$type: + - double + - int + - long - level: debug component: connection data: - message: "Connection checked in" - driverConnectionId: { $$type: [int, long] } - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - + message: Connection checked in + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long - level: debug component: connection data: - message: "Connection closed" - driverConnectionId: { $$type: [int, long] } - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - reason: "Connection pool was closed" - + message: Connection closed + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + reason: Connection pool was closed - level: debug component: connection data: - message: "Connection pool closed" - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - - # This test exists to provide coverage of checkout failed and pool cleared events. - - description: "Connection checkout fails due to error establishing connection" + message: Connection pool closed + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - description: Connection checkout fails due to error establishing connection runOnRequirements: - auth: true - minServerVersion: "4.0" # failCommand was added to mongod in 4.0 + minServerVersion: "4.0" operations: - name: createEntities object: testRunner arguments: entities: - client: - id: &client client + id: client uriOptions: retryReads: false - appname: &clientAppName clientAppName - # use a high heartbeatFrequencyMS to avoid a successful monitor check marking the pool as - # ready (and emitting another event) during the course of test execution. + appname: clientAppName heartbeatFrequencyMS: 10000 observeLogMessages: connection: debug - name: failPoint object: testRunner arguments: - client: *failPointClient + client: failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 1 } + mode: + times: 1 data: - failCommands: ["saslContinue"] - closeConnection: true - appName: *clientAppName + failCommands: + - saslContinue + errorCode: 18 + appName: clientAppName - name: listDatabases - object: *client + object: client arguments: filter: {} expectError: - isClientError: true - + isError: true + expectLogMessages: + - client: client + messages: + - level: debug + component: connection + data: + message: Connection pool created + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection pool ready + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection checkout started + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection created + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection closed + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + reason: An error occurred while using the connection + error: + $$exists: true + - level: debug + component: connection + data: + message: Connection checkout failed + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + reason: An error occurred while trying to establish a new connection + error: + $$exists: true + durationMS: + $$type: + - double + - int + - long + - level: debug + component: connection + data: + message: Connection pool cleared + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - description: Connection enters backoff on closed connection + runOnRequirements: + - auth: true + minServerVersion: "4.4" + operations: + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: client + uriOptions: + retryReads: true + appname: clientAppName + heartbeatFrequencyMS: 5000 + observeEvents: + - serverHeartbeatSucceededEvent + observeLogMessages: + connection: debug + - database: + id: database0 + client: client + databaseName: ci-tests + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + serverHeartbeatSucceededEvent: {} + count: 1 + - name: failPoint + object: testRunner + arguments: + client: failPointClient + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - isMaster + - hello + closeConnection: true + appName: clientAppName + - object: database0 + name: runCommand + arguments: + command: + find: test + commandName: find expectLogMessages: - - client: *client + - client: client messages: - level: debug component: connection data: - message: "Connection pool created" - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - + message: Connection pool created + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection pool ready + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection checkout started + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection created + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection closed + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + reason: An error occurred while using the connection + error: + $$exists: true + - level: debug + component: connection + data: + message: Connection pool backoff + serverHost: + $$type: string + durationMS: + $$type: int + attempt: + $$type: int + serverPort: + $$type: + - int + - long + reason: Connection pool is in backoff + error: + $$exists: true - level: debug component: connection data: - message: "Connection pool ready" - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - + message: Connection checkout failed + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + reason: An error occurred while trying to establish a new connection + error: + $$exists: true + durationMS: + $$type: + - double + - int + - long - level: debug component: connection data: - message: "Connection checkout started" - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - + message: Connection checkout started + serverHost: + $$type: string + serverPort: + $$type: + - int + - long - level: debug component: connection data: - message: "Connection created" - driverConnectionId: { $$type: [int, long] } - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - + message: Connection created + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long - level: debug component: connection data: - message: "Connection pool cleared" - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - + message: Connection ready + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + durationMS: + $$type: + - double + - int + - long - level: debug component: connection data: - message: "Connection closed" - driverConnectionId: { $$type: [int, long] } - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - reason: "An error occurred while using the connection" - error: { $$exists: true } - + message: Connection checked out + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + durationMS: + $$type: + - double + - int + - long - level: debug component: connection data: - message: "Connection checkout failed" - serverHost: { $$type: string } - serverPort: { $$type: [int, long] } - reason: "An error occurred while trying to establish a new connection" - error: { $$exists: true } - durationMS: { $$type: [double, int, long] } + message: Connection checked in + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long diff --git a/source/load-balancers/tests/sdam-error-handling.json b/source/load-balancers/tests/sdam-error-handling.json index 47323fae4f..5ff0fd73a9 100644 --- a/source/load-balancers/tests/sdam-error-handling.json +++ b/source/load-balancers/tests/sdam-error-handling.json @@ -41,7 +41,8 @@ "connectionCheckOutFailedEvent", "connectionCheckedInEvent", "connectionClosedEvent", - "poolClearedEvent" + "poolClearedEvent", + "poolBackoffEvent" ] } }, @@ -260,7 +261,7 @@ ] }, { - "description": "errors during the initial connection hello are ignored", + "description": "errors during the initial connection hello trigger backoff", "runOnRequirements": [ { "minServerVersion": "4.4.7" @@ -295,9 +296,17 @@ "document": { "x": 1 } - }, - "expectError": { - "isClientError": true + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "singleClient", + "event": { + "poolBackoffEvent": {} + }, + "count": 1 } } ], @@ -314,10 +323,27 @@ "reason": "error" } }, + { + "poolBackoffEvent": { + "attempt": 1 + } + }, { "connectionCheckOutFailedEvent": { "reason": "connectionError" } + }, + { + "connectionCreatedEvent": {} + }, + { + "connectionReadyEvent": {} + }, + { + "connectionCheckedOutEvent": {} + }, + { + "connectionCheckedInEvent": {} } ] } @@ -372,9 +398,6 @@ { "connectionCreatedEvent": {} }, - { - "poolClearedEvent": {} - }, { "connectionClosedEvent": { "reason": "error" @@ -384,6 +407,9 @@ "connectionCheckOutFailedEvent": { "reason": "connectionError" } + }, + { + "poolClearedEvent": {} } ] } @@ -511,4 +537,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/source/load-balancers/tests/sdam-error-handling.yml b/source/load-balancers/tests/sdam-error-handling.yml index b81d811dc8..1195b2969f 100644 --- a/source/load-balancers/tests/sdam-error-handling.yml +++ b/source/load-balancers/tests/sdam-error-handling.yml @@ -1,12 +1,10 @@ description: state change errors are correctly handled - -schemaVersion: '1.4' - +schemaVersion: "1.4" runOnRequirements: - - topologies: [ load-balanced ] - + - topologies: + - load-balanced _yamlAnchors: - observedEvents: &observedEvents + observedEvents: - connectionCreatedEvent - connectionReadyEvent - connectionCheckedOutEvent @@ -14,163 +12,178 @@ _yamlAnchors: - connectionCheckedInEvent - connectionClosedEvent - poolClearedEvent - createEntities: - client: - id: &failPointClient failPointClient + id: failPointClient useMultipleMongoses: false - client: - id: &singleClient singleClient + id: singleClient useMultipleMongoses: false uriOptions: - appname: &singleClientAppName lbSDAMErrorTestClient + appname: lbSDAMErrorTestClient retryWrites: false - observeEvents: *observedEvents + observeEvents: + - connectionCreatedEvent + - connectionReadyEvent + - connectionCheckedOutEvent + - connectionCheckOutFailedEvent + - connectionCheckedInEvent + - connectionClosedEvent + - poolClearedEvent + - poolBackoffEvent - database: - id: &singleDB singleDB - client: *singleClient - databaseName: &singleDBName singleDB + id: singleDB + client: singleClient + databaseName: singleDB - collection: - id: &singleColl singleColl - database: *singleDB - collectionName: &singleCollName singleColl + id: singleColl + database: singleDB + collectionName: singleColl - client: - id: &multiClient multiClient + id: multiClient useMultipleMongoses: true uriOptions: retryWrites: false - observeEvents: *observedEvents + observeEvents: + - connectionCreatedEvent + - connectionReadyEvent + - connectionCheckedOutEvent + - connectionCheckOutFailedEvent + - connectionCheckedInEvent + - connectionClosedEvent + - poolClearedEvent - database: - id: &multiDB multiDB - client: *multiClient - databaseName: &multiDBName multiDB + id: multiDB + client: multiClient + databaseName: multiDB - collection: - id: &multiColl multiColl - database: *multiDB - collectionName: &multiCollName multiColl - + id: multiColl + database: multiDB + collectionName: multiColl initialData: - - collectionName: *singleCollName - databaseName: *singleDBName + - collectionName: singleColl + databaseName: singleDB documents: - _id: 1 - _id: 2 - _id: 3 - - collectionName: *multiCollName - databaseName: *multiDBName + - collectionName: multiColl + databaseName: multiDB documents: - _id: 1 - _id: 2 - _id: 3 - tests: - description: only connections for a specific serviceId are closed when pools are cleared runOnRequirements: - # This test assumes that two sequential connections receive different serviceIDs. - # Sequential connections to a serverless instance may receive the same serviceID. - - serverless: forbid + - serverless: forbid operations: - # Create two cursors to force two connections. - name: createFindCursor - object: *multiColl + object: multiColl arguments: filter: {} batchSize: 2 - saveResultAsEntity: &cursor0 cursor0 + saveResultAsEntity: cursor0 - name: createFindCursor - object: *multiColl + object: multiColl arguments: filter: {} batchSize: 2 - saveResultAsEntity: &cursor1 cursor1 - # Close both cursors to return the connections to the pool. + saveResultAsEntity: cursor1 - name: close - object: *cursor0 + object: cursor0 - name: close - object: *cursor1 - # Fail an operation with a state change error. + object: cursor1 - name: failPoint object: testRunner arguments: - client: *multiClient + client: multiClient failPoint: configureFailPoint: failCommand - mode: { times: 1 } + mode: + times: 1 data: - failCommands: [insert] - errorCode: &errorCode 11600 # InterruptedAtShutdown + failCommands: + - insert + errorCode: 11600 - name: insertOne - object: *multiColl + object: multiColl arguments: - document: { x: 1 } + document: + x: 1 expectError: - errorCode: *errorCode - # Do another operation to ensure the relevant connection has been closed. + errorCode: 11600 - name: insertOne - object: *multiColl + object: multiColl arguments: - document: { x: 1 } + document: + x: 1 expectEvents: - - client: *multiClient + - client: multiClient eventType: cmap events: - # Create cursors. - connectionCreatedEvent: {} - connectionReadyEvent: {} - connectionCheckedOutEvent: {} - connectionCreatedEvent: {} - connectionReadyEvent: {} - connectionCheckedOutEvent: {} - # Close cursors. - connectionCheckedInEvent: {} - connectionCheckedInEvent: {} - # Set failpoint. - connectionCheckedOutEvent: {} - connectionCheckedInEvent: {} - # First insertOne. - connectionCheckedOutEvent: {} - poolClearedEvent: {} - connectionCheckedInEvent: {} - connectionClosedEvent: reason: stale - # Second insertOne. - connectionCheckedOutEvent: {} - connectionCheckedInEvent: {} - - # This test uses singleClient to ensure that connection attempts are routed - # to the same mongos on which the failpoint is set. - - description: errors during the initial connection hello are ignored + - description: errors during the initial connection hello trigger backoff runOnRequirements: - # Require SERVER-49336 for failCommand + appName on the initial handshake. - - minServerVersion: '4.4.7' + - minServerVersion: 4.4.7 operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 1 } + mode: + times: 1 data: - failCommands: [isMaster, hello] + failCommands: + - isMaster + - hello closeConnection: true - appName: *singleClientAppName + appName: lbSDAMErrorTestClient - name: insertOne - object: *singleColl + object: singleColl arguments: - document: { x: 1 } - expectError: - isClientError: true + document: + x: 1 + - name: waitForEvent + object: testRunner + arguments: + client: singleClient + event: + poolBackoffEvent: {} + count: 1 expectEvents: - - client: *singleClient + - client: singleClient eventType: cmap events: - connectionCreatedEvent: {} - connectionClosedEvent: reason: error + - poolBackoffEvent: + attempt: 1 - connectionCheckOutFailedEvent: reason: connectionError - + - connectionCreatedEvent: {} + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} - description: errors during authentication are processed runOnRequirements: - auth: true @@ -178,96 +191,91 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 1 } + mode: + times: 1 data: - failCommands: [saslContinue] + failCommands: + - saslContinue closeConnection: true - appName: *singleClientAppName + appName: lbSDAMErrorTestClient - name: insertOne - object: *singleColl + object: singleColl arguments: - document: { x: 1 } + document: + x: 1 expectError: isClientError: true expectEvents: - - client: *singleClient + - client: singleClient eventType: cmap events: - connectionCreatedEvent: {} - - poolClearedEvent: {} - connectionClosedEvent: reason: error - connectionCheckOutFailedEvent: reason: connectionError - + - poolClearedEvent: {} - description: stale errors are ignored operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 2 } + mode: + times: 2 data: - failCommands: [getMore] + failCommands: + - getMore closeConnection: true - # Force two connections to be checked out from the pool. - name: createFindCursor - object: *singleColl + object: singleColl arguments: filter: {} batchSize: 2 - saveResultAsEntity: &cursor0 cursor0 + saveResultAsEntity: cursor0 - name: createFindCursor - object: *singleColl + object: singleColl arguments: filter: {} batchSize: 2 - saveResultAsEntity: &cursor1 cursor1 - # Iterate cursor0 three times to force a network error. + saveResultAsEntity: cursor1 - name: iterateUntilDocumentOrError - object: *cursor0 + object: cursor0 - name: iterateUntilDocumentOrError - object: *cursor0 + object: cursor0 - name: iterateUntilDocumentOrError - object: *cursor0 + object: cursor0 expectError: isClientError: true - name: close - object: *cursor0 - # Iterate cursor1 three times to force a network error. + object: cursor0 - name: iterateUntilDocumentOrError - object: *cursor1 + object: cursor1 - name: iterateUntilDocumentOrError - object: *cursor1 + object: cursor1 - name: iterateUntilDocumentOrError - object: *cursor1 + object: cursor1 expectError: isClientError: true - name: close - object: *cursor1 + object: cursor1 expectEvents: - - client: *singleClient + - client: singleClient eventType: cmap events: - # Events for creating both cursors. - connectionCreatedEvent: {} - connectionReadyEvent: {} - connectionCheckedOutEvent: {} - connectionCreatedEvent: {} - connectionReadyEvent: {} - connectionCheckedOutEvent: {} - # Events for iterating and closing the first cursor. The failed - # getMore should cause a poolClearedEvent to be published. - poolClearedEvent: {} - connectionCheckedInEvent: {} - connectionClosedEvent: {} - # Events for iterating and closing the second cursor. The failed - # getMore should not clear the pool because the connection's - # generation number is stale. - connectionCheckedInEvent: {} - connectionClosedEvent: {} diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json new file mode 100644 index 0000000000..effcf69b48 --- /dev/null +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json @@ -0,0 +1,172 @@ +{ + "description": "heartbeat-failure-clears-backoff-pool", + "schemaVersion": "1.4", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "serverless": "forbid", + "topologies": [ + "single", + "replicaset", + "sharded" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "setupClient", + "useMultipleMongoses": false + } + } + ], + "initialData": [ + { + "collectionName": "heartbeat-backoff-error", + "databaseName": "sdam-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "tests": [ + { + "description": "A heartbeat failure during backoff should clear the pool", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "poolBackoffEvent", + "poolClearedEvent", + "serverHeartbeatFailedEvent", + "serverHeartbeatSucceededEvent" + ], + "uriOptions": { + "retryWrites": false, + "heartbeatFrequencyMS": 10000, + "serverMonitoringMode": "poll", + "appname": "heartbeatBackoffFailTest" + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "sdam-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "heartbeat-backoff-error" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverHeartbeatSucceededEvent": {} + }, + "count": 1 + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "appName": "heartbeatBackoffFailTest", + "closeConnection": true + } + } + } + }, + { + "name": "insertMany", + "object": "collection", + "arguments": { + "documents": [ + { + "_id": 3 + }, + { + "_id": 4 + } + ] + }, + "expectError": { + "isError": true + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverHeartbeatFailedEvent": {} + }, + "count": 1 + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "cmap", + "events": [ + { + "poolBackoffEvent": {} + }, + { + "poolBackoffEvent": {} + }, + { + "poolBackoffEvent": {} + }, + { + "poolBackoffEvent": {} + }, + { + "poolBackoffEvent": {} + }, + { + "poolBackoffEvent": {} + }, + { + "poolClearedEvent": {} + } + ] + } + ] + } + ] +} diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml new file mode 100644 index 0000000000..0049189d69 --- /dev/null +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml @@ -0,0 +1,94 @@ +description: heartbeat-failure-clears-backoff-pool +schemaVersion: "1.4" +runOnRequirements: + - minServerVersion: "4.4" + serverless: forbid + topologies: + - single + - replicaset + - sharded +createEntities: + - client: + id: setupClient + useMultipleMongoses: false +initialData: + - collectionName: heartbeat-backoff-error + databaseName: sdam-tests + documents: + - _id: 1 + - _id: 2 +tests: + - description: A heartbeat failure during backoff should clear the pool + operations: + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: client + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - poolBackoffEvent + - poolClearedEvent + - serverHeartbeatFailedEvent + - serverHeartbeatSucceededEvent + uriOptions: + retryWrites: false + heartbeatFrequencyMS: 10000 + serverMonitoringMode: poll + appname: heartbeatBackoffFailTest + - database: + id: database + client: client + databaseName: sdam-tests + - collection: + id: collection + database: database + collectionName: heartbeat-backoff-error + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + serverHeartbeatSucceededEvent: {} + count: 1 + - name: failPoint + object: testRunner + arguments: + client: setupClient + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - isMaster + - hello + appName: heartbeatBackoffFailTest + closeConnection: true + - name: insertMany + object: collection + arguments: + documents: + - _id: 3 + - _id: 4 + expectError: + isError: true + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + serverHeartbeatFailedEvent: {} + count: 1 + expectEvents: + - client: client + eventType: cmap + events: + - poolBackoffEvent: {} + - poolBackoffEvent: {} + - poolBackoffEvent: {} + - poolBackoffEvent: {} + - poolBackoffEvent: {} + - poolBackoffEvent: {} + - poolClearedEvent: {} diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json new file mode 100644 index 0000000000..bd5b95124b --- /dev/null +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json @@ -0,0 +1,180 @@ +{ + "description": "heartbeat-success-backoff", + "schemaVersion": "1.4", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "serverless": "forbid", + "topologies": [ + "single", + "replicaset", + "sharded" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "setupClient", + "useMultipleMongoses": false + } + } + ], + "initialData": [ + { + "collectionName": "heartbeat-backoff-success", + "databaseName": "sdam-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "tests": [ + { + "description": "A heartbeat success during backoff not mark the pool as ready", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "poolBackoffEvent", + "poolClearedEvent", + "serverHeartbeatFailedEvent", + "serverHeartbeatSucceededEvent" + ], + "uriOptions": { + "retryWrites": false, + "heartbeatFrequencyMS": 5000, + "serverMonitoringMode": "poll", + "appname": "heartbeatBackoffSuccessTest" + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "sdam-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "heartbeat-backoff-success" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverHeartbeatSucceededEvent": {} + }, + "count": 1 + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "appName": "heartbeatBackoffSuccessTest", + "closeConnection": true + } + } + } + }, + { + "name": "insertMany", + "object": "collection", + "arguments": { + "documents": [ + { + "_id": 3 + }, + { + "_id": 4 + } + ] + }, + "expectError": { + "isError": true + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "off" + } + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverHeartbeatSucceededEvent": {} + }, + "count": 1 + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "cmap", + "events": [ + { + "poolBackoffEvent": {} + }, + { + "poolBackoffEvent": {} + }, + { + "poolBackoffEvent": {} + }, + { + "poolBackoffEvent": {} + }, + { + "poolBackoffEvent": {} + }, + { + "poolBackoffEvent": {} + } + ] + } + ] + } + ] +} diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml new file mode 100644 index 0000000000..cb94fd4ebc --- /dev/null +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml @@ -0,0 +1,100 @@ +description: heartbeat-success-backoff +schemaVersion: "1.4" +runOnRequirements: + - minServerVersion: "4.4" + serverless: forbid + topologies: + - single + - replicaset + - sharded +createEntities: + - client: + id: setupClient + useMultipleMongoses: false +initialData: + - collectionName: heartbeat-backoff-success + databaseName: sdam-tests + documents: + - _id: 1 + - _id: 2 +tests: + - description: A heartbeat success during backoff not mark the pool as ready + operations: + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: client + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - poolBackoffEvent + - poolClearedEvent + - serverHeartbeatFailedEvent + - serverHeartbeatSucceededEvent + uriOptions: + retryWrites: false + heartbeatFrequencyMS: 5000 + serverMonitoringMode: poll + appname: heartbeatBackoffSuccessTest + - database: + id: database + client: client + databaseName: sdam-tests + - collection: + id: collection + database: database + collectionName: heartbeat-backoff-success + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + serverHeartbeatSucceededEvent: {} + count: 1 + - name: failPoint + object: testRunner + arguments: + client: setupClient + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - isMaster + - hello + appName: heartbeatBackoffSuccessTest + closeConnection: true + - name: insertMany + object: collection + arguments: + documents: + - _id: 3 + - _id: 4 + expectError: + isError: true + - name: failPoint + object: testRunner + arguments: + client: setupClient + failPoint: + configureFailPoint: failCommand + mode: off + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + serverHeartbeatSucceededEvent: {} + count: 1 + expectEvents: + - client: client + eventType: cmap + events: + - poolBackoffEvent: {} + - poolBackoffEvent: {} + - poolBackoffEvent: {} + - poolBackoffEvent: {} + - poolBackoffEvent: {} + - poolBackoffEvent: {} diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json new file mode 100644 index 0000000000..e6c89724de --- /dev/null +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json @@ -0,0 +1,302 @@ +{ + "description": "backoff-network-error-fail", + "schemaVersion": "1.4", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "serverless": "forbid", + "topologies": [ + "single", + "replicaset", + "sharded" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "setupClient", + "useMultipleMongoses": false + } + } + ], + "initialData": [ + { + "collectionName": "backoff-network-error-fail", + "databaseName": "sdam-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "tests": [ + { + "description": "Backoff and fail after network connection errors during connection establishment", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "serverHeartbeatSucceededEvent", + "commandStartedEvent", + "poolBackoffEvent", + "poolReadyEvent", + "poolClearedEvent" + ], + "uriOptions": { + "retryWrites": false, + "heartbeatFrequencyMS": 10000, + "serverMonitoringMode": "poll", + "appname": "backoffNetworkErrorFailTest" + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "sdam-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "backoff-network-error-fail" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverHeartbeatSucceededEvent": {} + }, + "count": 1 + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "appName": "backoffNetworkErrorFailTest", + "closeConnection": true + } + } + } + }, + { + "name": "insertMany", + "object": "collection", + "arguments": { + "documents": [ + { + "_id": 3 + }, + { + "_id": 4 + } + ] + }, + "expectError": { + "isError": true + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolBackoffEvent": {} + }, + "count": 5 + } + } + ] + }, + { + "description": "Backoff and clear the pool after network failures followed by server error", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "serverHeartbeatSucceededEvent", + "commandStartedEvent", + "poolBackoffEvent", + "poolReadyEvent", + "poolClearedEvent" + ], + "uriOptions": { + "retryWrites": false, + "heartbeatFrequencyMS": 10000, + "serverMonitoringMode": "poll", + "appname": "backoffNetworkErrorFailClearTest" + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "sdam-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "backoff-network-error-fail-clear" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverHeartbeatSucceededEvent": {} + }, + "count": 1 + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "appName": "backoffNetworkErrorFailClearTest", + "closeConnection": true + } + } + } + }, + { + "name": "insertMany", + "object": "collection", + "arguments": { + "documents": [ + { + "_id": 3 + }, + { + "_id": 4 + } + ] + }, + "expectError": { + "isError": true + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolBackoffEvent": {} + }, + "count": 5 + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "off" + } + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "hello", + "isMaster" + ], + "appName": "backoffNetworkErrorFailClearTest", + "errorCode": 1 + } + } + } + }, + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "_id": 3 + } + }, + "expectError": { + "isError": true + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolClearedEvent": {} + }, + "count": 1 + } + } + ] + } + ] +} diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml new file mode 100644 index 0000000000..9b62d93510 --- /dev/null +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml @@ -0,0 +1,181 @@ +description: backoff-network-error-fail +schemaVersion: "1.4" +runOnRequirements: + - minServerVersion: "4.4" + serverless: forbid + topologies: + - single + - replicaset + - sharded +createEntities: + - client: + id: setupClient + useMultipleMongoses: false +initialData: + - collectionName: backoff-network-error-fail + databaseName: sdam-tests + documents: + - _id: 1 + - _id: 2 +tests: + - description: Backoff and fail after network connection errors during connection establishment + operations: + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: client + useMultipleMongoses: false + observeEvents: + - serverHeartbeatSucceededEvent + - commandStartedEvent + - poolBackoffEvent + - poolReadyEvent + - poolClearedEvent + uriOptions: + retryWrites: false + heartbeatFrequencyMS: 10000 + serverMonitoringMode: poll + appname: backoffNetworkErrorFailTest + - database: + id: database + client: client + databaseName: sdam-tests + - collection: + id: collection + database: database + collectionName: backoff-network-error-fail + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + serverHeartbeatSucceededEvent: {} + count: 1 + - name: failPoint + object: testRunner + arguments: + client: setupClient + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - isMaster + - hello + appName: backoffNetworkErrorFailTest + closeConnection: true + - name: insertMany + object: collection + arguments: + documents: + - _id: 3 + - _id: 4 + expectError: + isError: true + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + poolBackoffEvent: {} + count: 5 + - description: Backoff and clear the pool after network failures followed by server error + operations: + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: client + useMultipleMongoses: false + observeEvents: + - serverHeartbeatSucceededEvent + - commandStartedEvent + - poolBackoffEvent + - poolReadyEvent + - poolClearedEvent + uriOptions: + retryWrites: false + heartbeatFrequencyMS: 10000 + serverMonitoringMode: poll + appname: backoffNetworkErrorFailClearTest + - database: + id: database + client: client + databaseName: sdam-tests + - collection: + id: collection + database: database + collectionName: backoff-network-error-fail-clear + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + serverHeartbeatSucceededEvent: {} + count: 1 + - name: failPoint + object: testRunner + arguments: + client: setupClient + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - isMaster + - hello + appName: backoffNetworkErrorFailClearTest + closeConnection: true + - name: insertMany + object: collection + arguments: + documents: + - _id: 3 + - _id: 4 + expectError: + isError: true + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + poolBackoffEvent: {} + count: 5 + - name: failPoint + object: testRunner + arguments: + client: setupClient + failPoint: + configureFailPoint: failCommand + mode: off + - name: failPoint + object: testRunner + arguments: + client: setupClient + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - hello + - isMaster + appName: backoffNetworkErrorFailClearTest + errorCode: 1 + - name: insertOne + object: collection + arguments: + document: + _id: 3 + expectError: + isError: true + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + poolClearedEvent: {} + count: 1 diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json new file mode 100644 index 0000000000..e6b71d4e13 --- /dev/null +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json @@ -0,0 +1,204 @@ +{ + "description": "backoff-network-error", + "schemaVersion": "1.4", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "serverless": "forbid", + "topologies": [ + "single", + "replicaset", + "sharded" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "setupClient", + "useMultipleMongoses": false + } + } + ], + "initialData": [ + { + "collectionName": "backoff-network-error", + "databaseName": "sdam-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "tests": [ + { + "description": "Backoff and retry after network connection error during connection establishment", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "serverHeartbeatSucceededEvent", + "commandStartedEvent", + "poolBackoffEvent", + "poolReadyEvent", + "poolClearedEvent" + ], + "uriOptions": { + "retryWrites": false, + "appname": "backoffNetworkErrorTest", + "heartbeatFrequencyMS": 10000, + "serverMonitoringMode": "poll", + "connectTimeoutMS": 250, + "socketTimeoutMS": 250 + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "sdam-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "backoff-network-error" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverHeartbeatSucceededEvent": {} + }, + "count": 1 + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "closeConnection": true, + "appName": "backoffNetworkErrorTest" + } + } + } + }, + { + "name": "insertMany", + "object": "collection", + "arguments": { + "documents": [ + { + "_id": 3 + }, + { + "_id": 4 + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolBackoffEvent": { + "attempt": 1 + } + }, + "count": 1 + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolBackoffEvent": { + "attempt": 2 + } + }, + "count": 1 + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "command", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "backoff-network-error", + "documents": [ + { + "_id": 3 + }, + { + "_id": 4 + } + ] + }, + "commandName": "insert", + "databaseName": "sdam-tests" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "backoff-network-error", + "databaseName": "sdam-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + }, + { + "_id": 3 + }, + { + "_id": 4 + } + ] + } + ] + } + ] +} diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml new file mode 100644 index 0000000000..c4c90bba54 --- /dev/null +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml @@ -0,0 +1,113 @@ +description: backoff-network-error +schemaVersion: "1.4" +runOnRequirements: + - minServerVersion: "4.4" + serverless: forbid + topologies: + - single + - replicaset + - sharded +createEntities: + - client: + id: setupClient + useMultipleMongoses: false +initialData: + - collectionName: backoff-network-error + databaseName: sdam-tests + documents: + - _id: 1 + - _id: 2 +tests: + - description: Backoff and retry after network connection error during connection establishment + operations: + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: client + useMultipleMongoses: false + observeEvents: + - serverHeartbeatSucceededEvent + - commandStartedEvent + - poolBackoffEvent + - poolReadyEvent + - poolClearedEvent + uriOptions: + retryWrites: false + appname: backoffNetworkErrorTest + heartbeatFrequencyMS: 10000 + serverMonitoringMode: poll + connectTimeoutMS: 250 + socketTimeoutMS: 250 + - database: + id: database + client: client + databaseName: sdam-tests + - collection: + id: collection + database: database + collectionName: backoff-network-error + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + serverHeartbeatSucceededEvent: {} + count: 1 + - name: failPoint + object: testRunner + arguments: + client: setupClient + failPoint: + configureFailPoint: failCommand + mode: + times: 2 + data: + failCommands: + - isMaster + - hello + closeConnection: true + appName: backoffNetworkErrorTest + - name: insertMany + object: collection + arguments: + documents: + - _id: 3 + - _id: 4 + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + poolBackoffEvent: + attempt: 1 + count: 1 + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + poolBackoffEvent: + attempt: 2 + count: 1 + expectEvents: + - client: client + eventType: command + events: + - commandStartedEvent: + command: + insert: backoff-network-error + documents: + - _id: 3 + - _id: 4 + commandName: insert + databaseName: sdam-tests + outcome: + - collectionName: backoff-network-error + databaseName: sdam-tests + documents: + - _id: 1 + - _id: 2 + - _id: 3 + - _id: 4 diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json new file mode 100644 index 0000000000..96a35143bc --- /dev/null +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json @@ -0,0 +1,201 @@ +{ + "description": "backoff-network-timeout-error", + "schemaVersion": "1.4", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "backoff": true, + "serverless": "forbid", + "topologies": [ + "single", + "replicaset", + "sharded" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "setupClient", + "useMultipleMongoses": false + } + } + ], + "initialData": [ + { + "collectionName": "backoff-network-timeout-error", + "databaseName": "sdam-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "tests": [ + { + "description": "Backoff and retry after network timeout error during connection establishment", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "serverHeartbeatSucceededEvent", + "commandStartedEvent", + "poolBackoffEvent", + "poolReadyEvent", + "poolClearedEvent" + ], + "uriOptions": { + "retryWrites": false, + "heartbeatFrequencyMS": 10000, + "appname": "backoffNetworkTimeoutErrorTest", + "serverMonitoringMode": "poll", + "connectTimeoutMS": 250, + "socketTimeoutMS": 250 + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "sdam-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "backoff-network-timeout-error" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverHeartbeatSucceededEvent": {} + }, + "count": 1 + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "setupClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "closeConnection": true, + "appName": "backoffNetworkTimeoutErrorTest" + } + } + } + }, + { + "name": "insertMany", + "object": "collection", + "arguments": { + "documents": [ + { + "_id": 3 + }, + { + "_id": 4 + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolBackoffEvent": {} + }, + "count": 2 + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolReadyEvent": {} + }, + "count": 1 + } + } + ], + "expectEvents": [ + { + "client": "client", + "eventType": "command", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "backoff-network-timeout-error", + "documents": [ + { + "_id": 3 + }, + { + "_id": 4 + } + ] + }, + "commandName": "insert", + "databaseName": "sdam-tests" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "backoff-network-timeout-error", + "databaseName": "sdam-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + }, + { + "_id": 3 + }, + { + "_id": 4 + } + ] + } + ] + } + ] +} diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml new file mode 100644 index 0000000000..c67868f38d --- /dev/null +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml @@ -0,0 +1,112 @@ +description: backoff-network-timeout-error +schemaVersion: "1.4" +runOnRequirements: + - minServerVersion: "4.4" + backoff: true + serverless: forbid + topologies: + - single + - replicaset + - sharded +createEntities: + - client: + id: setupClient + useMultipleMongoses: false +initialData: + - collectionName: backoff-network-timeout-error + databaseName: sdam-tests + documents: + - _id: 1 + - _id: 2 +tests: + - description: Backoff and retry after network timeout error during connection establishment + operations: + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: client + useMultipleMongoses: false + observeEvents: + - serverHeartbeatSucceededEvent + - commandStartedEvent + - poolBackoffEvent + - poolReadyEvent + - poolClearedEvent + uriOptions: + retryWrites: false + heartbeatFrequencyMS: 10000 + appname: backoffNetworkTimeoutErrorTest + serverMonitoringMode: poll + connectTimeoutMS: 250 + socketTimeoutMS: 250 + - database: + id: database + client: client + databaseName: sdam-tests + - collection: + id: collection + database: database + collectionName: backoff-network-timeout-error + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + serverHeartbeatSucceededEvent: {} + count: 1 + - name: failPoint + object: testRunner + arguments: + client: setupClient + failPoint: + configureFailPoint: failCommand + mode: + times: 2 + data: + failCommands: + - isMaster + - hello + closeConnection: true + appName: backoffNetworkTimeoutErrorTest + - name: insertMany + object: collection + arguments: + documents: + - _id: 3 + - _id: 4 + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + poolBackoffEvent: {} + count: 2 + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + poolReadyEvent: {} + count: 1 + expectEvents: + - client: client + eventType: command + events: + - commandStartedEvent: + command: + insert: backoff-network-timeout-error + documents: + - _id: 3 + - _id: 4 + commandName: insert + databaseName: sdam-tests + outcome: + - collectionName: backoff-network-timeout-error + databaseName: sdam-tests + documents: + - _id: 1 + - _id: 2 + - _id: 3 + - _id: 4 From e72968a1e545e90807dada50f5b4457c896490e7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Oct 2025 21:33:45 -0500 Subject: [PATCH 09/23] fix test --- .../tests/unified/backoff-network-timeout-error.json | 1 - .../tests/unified/backoff-network-timeout-error.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json index 96a35143bc..cdb67a7194 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json @@ -4,7 +4,6 @@ "runOnRequirements": [ { "minServerVersion": "4.4", - "backoff": true, "serverless": "forbid", "topologies": [ "single", diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml index c67868f38d..5a4bcdbffc 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml @@ -2,7 +2,6 @@ description: backoff-network-timeout-error schemaVersion: "1.4" runOnRequirements: - minServerVersion: "4.4" - backoff: true serverless: forbid topologies: - single From 91cd80f0c1f8cc0eca8d74d2ff89ac6f06ab2739 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Oct 2025 21:41:27 -0500 Subject: [PATCH 10/23] formatting --- .../tests/logging/connection-logging.json | 6 +++--- source/load-balancers/tests/sdam-error-handling.json | 2 +- .../tests/unified/backoff-network-error.json | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json index 207618b4a6..303ac80784 100644 --- a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json +++ b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json @@ -711,10 +711,10 @@ "$$type": "string" }, "durationMS": { - "$$type": "int" + "$$type": "int" }, "attempt": { - "$$type": "int" + "$$type": "int" }, "serverPort": { "$$type": [ @@ -793,7 +793,7 @@ } } }, -{ + { "level": "debug", "component": "connection", "data": { diff --git a/source/load-balancers/tests/sdam-error-handling.json b/source/load-balancers/tests/sdam-error-handling.json index 5ff0fd73a9..45b969d8d4 100644 --- a/source/load-balancers/tests/sdam-error-handling.json +++ b/source/load-balancers/tests/sdam-error-handling.json @@ -537,4 +537,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json index e6b71d4e13..4378c019b4 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json @@ -150,7 +150,7 @@ "attempt": 2 } }, - "count": 1 + "count": 1 } } ], From b0561bd5dcc2b7c714dbe8ff927c77fe73ffac22 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Oct 2025 21:49:39 -0500 Subject: [PATCH 11/23] update schema version --- .../tests/unified/backoff-heartbeat-failure.json | 2 +- .../tests/unified/backoff-heartbeat-failure.yml | 2 +- .../tests/unified/backoff-heartbeat-success.json | 2 +- .../tests/unified/backoff-heartbeat-success.yml | 2 +- .../tests/unified/backoff-network-error-fail.json | 2 +- .../tests/unified/backoff-network-error-fail.yml | 2 +- .../tests/unified/backoff-network-error.json | 4 ++-- .../tests/unified/backoff-network-error.yml | 2 +- .../tests/unified/backoff-network-timeout-error.json | 2 +- .../tests/unified/backoff-network-timeout-error.yml | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json index effcf69b48..2d4e0471f9 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json @@ -1,6 +1,6 @@ { "description": "heartbeat-failure-clears-backoff-pool", - "schemaVersion": "1.4", + "schemaVersion": "1.28", "runOnRequirements": [ { "minServerVersion": "4.4", diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml index 0049189d69..28128b806d 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml @@ -1,5 +1,5 @@ description: heartbeat-failure-clears-backoff-pool -schemaVersion: "1.4" +schemaVersion: "1.28" runOnRequirements: - minServerVersion: "4.4" serverless: forbid diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json index bd5b95124b..4f790007b5 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json @@ -1,6 +1,6 @@ { "description": "heartbeat-success-backoff", - "schemaVersion": "1.4", + "schemaVersion": "1.28", "runOnRequirements": [ { "minServerVersion": "4.4", diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml index cb94fd4ebc..c180eff3b9 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml @@ -1,5 +1,5 @@ description: heartbeat-success-backoff -schemaVersion: "1.4" +schemaVersion: "1.28" runOnRequirements: - minServerVersion: "4.4" serverless: forbid diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json index e6c89724de..c128198415 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json @@ -1,6 +1,6 @@ { "description": "backoff-network-error-fail", - "schemaVersion": "1.4", + "schemaVersion": "1.28", "runOnRequirements": [ { "minServerVersion": "4.4", diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml index 9b62d93510..277958e5b6 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml @@ -1,5 +1,5 @@ description: backoff-network-error-fail -schemaVersion: "1.4" +schemaVersion: "1.28" runOnRequirements: - minServerVersion: "4.4" serverless: forbid diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json index 4378c019b4..a3f62fd674 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json @@ -1,6 +1,6 @@ { "description": "backoff-network-error", - "schemaVersion": "1.4", + "schemaVersion": "1.28", "runOnRequirements": [ { "minServerVersion": "4.4", @@ -150,7 +150,7 @@ "attempt": 2 } }, - "count": 1 + "count": 1 } } ], diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml index c4c90bba54..6adae838cd 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml @@ -1,5 +1,5 @@ description: backoff-network-error -schemaVersion: "1.4" +schemaVersion: "1.28" runOnRequirements: - minServerVersion: "4.4" serverless: forbid diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json index cdb67a7194..87c50bc2f2 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json @@ -1,6 +1,6 @@ { "description": "backoff-network-timeout-error", - "schemaVersion": "1.4", + "schemaVersion": "1.28", "runOnRequirements": [ { "minServerVersion": "4.4", diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml index 5a4bcdbffc..c475cbac4c 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml @@ -1,5 +1,5 @@ description: backoff-network-timeout-error -schemaVersion: "1.4" +schemaVersion: "1.28" runOnRequirements: - minServerVersion: "4.4" serverless: forbid From 5b37566ca9b15720ed2aa0f843cd3fadad8907ae Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Oct 2025 21:53:33 -0500 Subject: [PATCH 12/23] formatting --- .../tests/unified/backoff-network-error.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json index a3f62fd674..b3b230ca6d 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json @@ -150,7 +150,7 @@ "attempt": 2 } }, - "count": 1 + "count": 1 } } ], From a56e7ec30827f593e92e3e78444beeffcced6684 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Oct 2025 22:00:27 -0500 Subject: [PATCH 13/23] update tests --- .../tests/logging/connection-logging.json | 1 - .../tests/logging/connection-logging.yml | 3 +-- source/load-balancers/tests/sdam-error-handling.json | 2 +- source/load-balancers/tests/sdam-error-handling.yml | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json index 303ac80784..f9f34ae95b 100644 --- a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json +++ b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json @@ -523,7 +523,6 @@ "description": "Connection enters backoff on closed connection", "runOnRequirements": [ { - "auth": true, "minServerVersion": "4.4" } ], diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml index fc351e47ad..985fe514bb 100644 --- a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml +++ b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml @@ -320,8 +320,7 @@ tests: - long - description: Connection enters backoff on closed connection runOnRequirements: - - auth: true - minServerVersion: "4.4" + - minServerVersion: "4.4" operations: - name: createEntities object: testRunner diff --git a/source/load-balancers/tests/sdam-error-handling.json b/source/load-balancers/tests/sdam-error-handling.json index 45b969d8d4..7654eff6fb 100644 --- a/source/load-balancers/tests/sdam-error-handling.json +++ b/source/load-balancers/tests/sdam-error-handling.json @@ -1,6 +1,6 @@ { "description": "state change errors are correctly handled", - "schemaVersion": "1.4", + "schemaVersion": "1.28", "runOnRequirements": [ { "topologies": [ diff --git a/source/load-balancers/tests/sdam-error-handling.yml b/source/load-balancers/tests/sdam-error-handling.yml index 1195b2969f..2b2e2ea54d 100644 --- a/source/load-balancers/tests/sdam-error-handling.yml +++ b/source/load-balancers/tests/sdam-error-handling.yml @@ -1,5 +1,5 @@ description: state change errors are correctly handled -schemaVersion: "1.4" +schemaVersion: "1.28" runOnRequirements: - topologies: - load-balanced From 4c0df36a68e1caaf4c64dc6e13c627508e664c31 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 31 Oct 2025 13:02:01 -0500 Subject: [PATCH 14/23] update tests --- .../unified/backoff-heartbeat-failure.json | 38 +++--------- .../unified/backoff-heartbeat-failure.yml | 18 +++--- .../unified/backoff-heartbeat-success.json | 37 ++++------- .../unified/backoff-heartbeat-success.yml | 17 +++--- .../unified/backoff-network-error-fail.json | 21 ++++++- .../unified/backoff-network-error-fail.yml | 12 +++- .../tests/unified/backoff-network-error.json | 61 +------------------ .../tests/unified/backoff-network-error.yml | 30 +-------- .../backoff-network-timeout-error.json | 52 ++-------------- .../unified/backoff-network-timeout-error.yml | 26 ++------ 10 files changed, 78 insertions(+), 234 deletions(-) diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json index 2d4e0471f9..b2a98fa4f6 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json @@ -132,39 +132,21 @@ "arguments": { "client": "client", "event": { - "serverHeartbeatFailedEvent": {} + "poolBackoffEvent": {} }, "count": 1 } - } - ], - "expectEvents": [ + }, { - "client": "client", - "eventType": "cmap", - "events": [ - { - "poolBackoffEvent": {} - }, - { - "poolBackoffEvent": {} - }, - { - "poolBackoffEvent": {} - }, - { - "poolBackoffEvent": {} - }, - { - "poolBackoffEvent": {} - }, - { - "poolBackoffEvent": {} + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverHeartbeatFailedEvent": {} }, - { - "poolClearedEvent": {} - } - ] + "count": 1 + } } ] } diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml index 28128b806d..55823b6908 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml @@ -74,6 +74,13 @@ tests: - _id: 4 expectError: isError: true + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + poolBackoffEvent: {} + count: 1 - name: waitForEvent object: testRunner arguments: @@ -81,14 +88,3 @@ tests: event: serverHeartbeatFailedEvent: {} count: 1 - expectEvents: - - client: client - eventType: cmap - events: - - poolBackoffEvent: {} - - poolBackoffEvent: {} - - poolBackoffEvent: {} - - poolBackoffEvent: {} - - poolBackoffEvent: {} - - poolBackoffEvent: {} - - poolClearedEvent: {} diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json index 4f790007b5..ef07ab5d90 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json @@ -126,6 +126,17 @@ "isError": true } }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolBackoffEvent": {} + }, + "count": 1 + } + }, { "name": "failPoint", "object": "testRunner", @@ -148,32 +159,6 @@ "count": 1 } } - ], - "expectEvents": [ - { - "client": "client", - "eventType": "cmap", - "events": [ - { - "poolBackoffEvent": {} - }, - { - "poolBackoffEvent": {} - }, - { - "poolBackoffEvent": {} - }, - { - "poolBackoffEvent": {} - }, - { - "poolBackoffEvent": {} - }, - { - "poolBackoffEvent": {} - } - ] - } ] } ] diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml index c180eff3b9..6c89736935 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml @@ -74,6 +74,13 @@ tests: - _id: 4 expectError: isError: true + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + poolBackoffEvent: {} + count: 1 - name: failPoint object: testRunner arguments: @@ -88,13 +95,3 @@ tests: event: serverHeartbeatSucceededEvent: {} count: 1 - expectEvents: - - client: client - eventType: cmap - events: - - poolBackoffEvent: {} - - poolBackoffEvent: {} - - poolBackoffEvent: {} - - poolBackoffEvent: {} - - poolBackoffEvent: {} - - poolBackoffEvent: {} diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json index c128198415..43d84ad8f7 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json @@ -126,6 +126,23 @@ "isError": true } }, + { + "name": "insertMany", + "object": "collection", + "arguments": { + "documents": [ + { + "_id": 3 + }, + { + "_id": 4 + } + ] + }, + "expectError": { + "isError": true + } + }, { "name": "waitForEvent", "object": "testRunner", @@ -134,7 +151,7 @@ "event": { "poolBackoffEvent": {} }, - "count": 5 + "count": 1 } } ] @@ -238,7 +255,7 @@ "event": { "poolBackoffEvent": {} }, - "count": 5 + "count": 1 } }, { diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml index 277958e5b6..e5e953af95 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml @@ -74,13 +74,21 @@ tests: - _id: 4 expectError: isError: true + - name: insertMany + object: collection + arguments: + documents: + - _id: 3 + - _id: 4 + expectError: + isError: true - name: waitForEvent object: testRunner arguments: client: client event: poolBackoffEvent: {} - count: 5 + count: 1 - description: Backoff and clear the pool after network failures followed by server error operations: - name: createEntities @@ -143,7 +151,7 @@ tests: client: client event: poolBackoffEvent: {} - count: 5 + count: 1 - name: failPoint object: testRunner arguments: diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json index b3b230ca6d..6da5973eaa 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json @@ -125,6 +125,9 @@ "_id": 4 } ] + }, + "expectError": { + "isError": true } }, { @@ -139,64 +142,6 @@ }, "count": 1 } - }, - { - "name": "waitForEvent", - "object": "testRunner", - "arguments": { - "client": "client", - "event": { - "poolBackoffEvent": { - "attempt": 2 - } - }, - "count": 1 - } - } - ], - "expectEvents": [ - { - "client": "client", - "eventType": "command", - "events": [ - { - "commandStartedEvent": { - "command": { - "insert": "backoff-network-error", - "documents": [ - { - "_id": 3 - }, - { - "_id": 4 - } - ] - }, - "commandName": "insert", - "databaseName": "sdam-tests" - } - } - ] - } - ], - "outcome": [ - { - "collectionName": "backoff-network-error", - "databaseName": "sdam-tests", - "documents": [ - { - "_id": 1 - }, - { - "_id": 2 - }, - { - "_id": 3 - }, - { - "_id": 4 - } - ] } ] } diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml index 6adae838cd..9c00d2729a 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml @@ -75,6 +75,8 @@ tests: documents: - _id: 3 - _id: 4 + expectError: + isError: true - name: waitForEvent object: testRunner arguments: @@ -83,31 +85,3 @@ tests: poolBackoffEvent: attempt: 1 count: 1 - - name: waitForEvent - object: testRunner - arguments: - client: client - event: - poolBackoffEvent: - attempt: 2 - count: 1 - expectEvents: - - client: client - eventType: command - events: - - commandStartedEvent: - command: - insert: backoff-network-error - documents: - - _id: 3 - - _id: 4 - commandName: insert - databaseName: sdam-tests - outcome: - - collectionName: backoff-network-error - databaseName: sdam-tests - documents: - - _id: 1 - - _id: 2 - - _id: 3 - - _id: 4 diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json index 87c50bc2f2..ee9daf224c 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json @@ -100,7 +100,7 @@ "failPoint": { "configureFailPoint": "failCommand", "mode": { - "times": 2 + "times": 1 }, "data": { "failCommands": [ @@ -125,6 +125,9 @@ "_id": 4 } ] + }, + "expectError": { + "isError": true } }, { @@ -135,7 +138,7 @@ "event": { "poolBackoffEvent": {} }, - "count": 2 + "count": 1 } }, { @@ -149,51 +152,6 @@ "count": 1 } } - ], - "expectEvents": [ - { - "client": "client", - "eventType": "command", - "events": [ - { - "commandStartedEvent": { - "command": { - "insert": "backoff-network-timeout-error", - "documents": [ - { - "_id": 3 - }, - { - "_id": 4 - } - ] - }, - "commandName": "insert", - "databaseName": "sdam-tests" - } - } - ] - } - ], - "outcome": [ - { - "collectionName": "backoff-network-timeout-error", - "databaseName": "sdam-tests", - "documents": [ - { - "_id": 1 - }, - { - "_id": 2 - }, - { - "_id": 3 - }, - { - "_id": 4 - } - ] - } ] } ] diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml index c475cbac4c..df4c58c763 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml @@ -62,7 +62,7 @@ tests: failPoint: configureFailPoint: failCommand mode: - times: 2 + times: 1 data: failCommands: - isMaster @@ -75,13 +75,15 @@ tests: documents: - _id: 3 - _id: 4 + expectError: + isError: true - name: waitForEvent object: testRunner arguments: client: client event: poolBackoffEvent: {} - count: 2 + count: 1 - name: waitForEvent object: testRunner arguments: @@ -89,23 +91,3 @@ tests: event: poolReadyEvent: {} count: 1 - expectEvents: - - client: client - eventType: command - events: - - commandStartedEvent: - command: - insert: backoff-network-timeout-error - documents: - - _id: 3 - - _id: 4 - commandName: insert - databaseName: sdam-tests - outcome: - - collectionName: backoff-network-timeout-error - databaseName: sdam-tests - documents: - - _id: 1 - - _id: 2 - - _id: 3 - - _id: 4 From 39acfd252032b29b68f0b25cff391857f7bc16e0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 31 Oct 2025 16:04:02 -0500 Subject: [PATCH 15/23] address review --- .../tests/unified/backoff-heartbeat-failure.json | 7 ++++++- .../tests/unified/backoff-heartbeat-failure.yml | 4 ++++ .../tests/unified/backoff-heartbeat-success.json | 7 ++++++- .../tests/unified/backoff-heartbeat-success.yml | 4 ++++ .../tests/unified/backoff-network-error-fail.json | 13 +++++++++++-- .../tests/unified/backoff-network-error-fail.yml | 7 +++++++ .../tests/unified/backoff-network-error.json | 7 ++++++- .../tests/unified/backoff-network-error.yml | 4 ++++ .../unified/backoff-network-timeout-error.json | 7 ++++++- .../tests/unified/backoff-network-timeout-error.yml | 4 ++++ source/unified-test-format/schema-1.28.json | 3 +++ source/unified-test-format/schema-latest.json | 3 +++ source/unified-test-format/unified-test-format.md | 3 +++ 13 files changed, 67 insertions(+), 6 deletions(-) diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json index b2a98fa4f6..8c6e225916 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json @@ -5,6 +5,7 @@ { "minServerVersion": "4.4", "serverless": "forbid", + "poolBackoff": true, "topologies": [ "single", "replicaset", @@ -123,7 +124,11 @@ ] }, "expectError": { - "isError": true + "isError": true, + "errorLabelsContain": [ + "SystemOverloadedError", + "RetryableError" + ] } }, { diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml index 55823b6908..e908f0a7a1 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml @@ -3,6 +3,7 @@ schemaVersion: "1.28" runOnRequirements: - minServerVersion: "4.4" serverless: forbid + poolBackoff: true topologies: - single - replicaset @@ -74,6 +75,9 @@ tests: - _id: 4 expectError: isError: true + errorLabelsContain: + - SystemOverloadedError + - RetryableError - name: waitForEvent object: testRunner arguments: diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json index ef07ab5d90..beeed33fb0 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json @@ -5,6 +5,7 @@ { "minServerVersion": "4.4", "serverless": "forbid", + "poolBackoff": true, "topologies": [ "single", "replicaset", @@ -123,7 +124,11 @@ ] }, "expectError": { - "isError": true + "isError": true, + "errorLabelsContain": [ + "SystemOverloadedError", + "RetryableError" + ] } }, { diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml index 6c89736935..dbe7715b65 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml @@ -3,6 +3,7 @@ schemaVersion: "1.28" runOnRequirements: - minServerVersion: "4.4" serverless: forbid + poolBackoff: true topologies: - single - replicaset @@ -74,6 +75,9 @@ tests: - _id: 4 expectError: isError: true + errorLabelsContain: + - SystemOverloadedError + - RetryableError - name: waitForEvent object: testRunner arguments: diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json index 43d84ad8f7..c0ae2adc9c 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json @@ -5,6 +5,7 @@ { "minServerVersion": "4.4", "serverless": "forbid", + "poolBackoff": true, "topologies": [ "single", "replicaset", @@ -123,7 +124,11 @@ ] }, "expectError": { - "isError": true + "isError": true, + "errorLabelsContain": [ + "SystemOverloadedError", + "RetryableError" + ] } }, { @@ -244,7 +249,11 @@ ] }, "expectError": { - "isError": true + "isError": true, + "errorLabelsContain": [ + "SystemOverloadedError", + "RetryableError" + ] } }, { diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml index e5e953af95..7fbe8cb551 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml @@ -3,6 +3,7 @@ schemaVersion: "1.28" runOnRequirements: - minServerVersion: "4.4" serverless: forbid + poolBackoff: true topologies: - single - replicaset @@ -74,6 +75,9 @@ tests: - _id: 4 expectError: isError: true + errorLabelsContain: + - SystemOverloadedError + - RetryableError - name: insertMany object: collection arguments: @@ -145,6 +149,9 @@ tests: - _id: 4 expectError: isError: true + errorLabelsContain: + - SystemOverloadedError + - RetryableError - name: waitForEvent object: testRunner arguments: diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json index 6da5973eaa..25edcfaf93 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json @@ -5,6 +5,7 @@ { "minServerVersion": "4.4", "serverless": "forbid", + "poolBackoff": true, "topologies": [ "single", "replicaset", @@ -127,7 +128,11 @@ ] }, "expectError": { - "isError": true + "isError": true, + "errorLabelsContain": [ + "SystemOverloadedError", + "RetryableError" + ] } }, { diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml index 9c00d2729a..a357864ff0 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml @@ -3,6 +3,7 @@ schemaVersion: "1.28" runOnRequirements: - minServerVersion: "4.4" serverless: forbid + poolBackoff: true topologies: - single - replicaset @@ -77,6 +78,9 @@ tests: - _id: 4 expectError: isError: true + errorLabelsContain: + - SystemOverloadedError + - RetryableError - name: waitForEvent object: testRunner arguments: diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json index ee9daf224c..2cad0f685c 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json @@ -5,6 +5,7 @@ { "minServerVersion": "4.4", "serverless": "forbid", + "poolBackoff": true, "topologies": [ "single", "replicaset", @@ -127,7 +128,11 @@ ] }, "expectError": { - "isError": true + "isError": true, + "errorLabelsContain": [ + "SystemOverloadedError", + "RetryableError" + ] } }, { diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml index df4c58c763..18826072ea 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml @@ -3,6 +3,7 @@ schemaVersion: "1.28" runOnRequirements: - minServerVersion: "4.4" serverless: forbid + poolBackoff: true topologies: - single - replicaset @@ -77,6 +78,9 @@ tests: - _id: 4 expectError: isError: true + errorLabelsContain: + - SystemOverloadedError + - RetryableError - name: waitForEvent object: testRunner arguments: diff --git a/source/unified-test-format/schema-1.28.json b/source/unified-test-format/schema-1.28.json index 607d4d54a0..f133df24af 100644 --- a/source/unified-test-format/schema-1.28.json +++ b/source/unified-test-format/schema-1.28.json @@ -112,6 +112,9 @@ } } ] + }, + "poolBackoff": { + "type": "boolean" } } }, diff --git a/source/unified-test-format/schema-latest.json b/source/unified-test-format/schema-latest.json index 607d4d54a0..f133df24af 100644 --- a/source/unified-test-format/schema-latest.json +++ b/source/unified-test-format/schema-latest.json @@ -112,6 +112,9 @@ } } ] + }, + "poolBackoff": { + "type": "boolean" } } }, diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 5d2112fb1f..f153e4103b 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -382,6 +382,9 @@ The structure of this object is as follows: tests. If this field is omitted, there is no lower bound on the required libmongocrypt version. The format of this string is defined in [Version String](#version-string). +- `poolBackoff`: Optional boolean. If it is true, tests MUST only run if the driver supports backoff state in connection + pools. If it is false, tests MUST only run if the driver does not support backoff state in connection pools. + Test runners MAY evaluate these conditions in any order. For example, it may be more efficient to evaluate `serverless` or `auth` before communicating with a server to check its version. From fdb46c610d39563464033893e6a2b8df8da9b18d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 31 Oct 2025 16:11:05 -0500 Subject: [PATCH 16/23] add unified schema tests --- .../runOnRequirement-poolBackoff-type.json | 15 +++++++++++++++ .../invalid/runOnRequirement-poolBackoff-type.yml | 10 ++++++++++ 2 files changed, 25 insertions(+) create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-poolBackoff-type.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-poolBackoff-type.yml diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-poolBackoff-type.json b/source/unified-test-format/tests/invalid/runOnRequirement-poolBackoff-type.json new file mode 100644 index 0000000000..9c107e55f3 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-poolBackoff-type.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-auth-type", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "poolBackoff": "foo" + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-poolBackoff-type.yml b/source/unified-test-format/tests/invalid/runOnRequirement-poolBackoff-type.yml new file mode 100644 index 0000000000..da96fe23a8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-poolBackoff-type.yml @@ -0,0 +1,10 @@ +description: runOnRequirement-auth-type + +schemaVersion: '1.3' + +runOnRequirements: + - poolBackoff: foo + +tests: + - description: foo + operations: [] From 8b0d35da65e110c2bb1db62b94fe78acbd9d9506 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 3 Nov 2025 08:10:56 -0600 Subject: [PATCH 17/23] update tests --- .../tests/cmap-format/README.md | 2 + ...koff-interrupting-pending-connections.json | 76 +++ ...ckoff-interrupting-pending-connections.yml | 42 ++ ...ol-create-min-size-error-with-backoff.json | 65 +++ ...ool-create-min-size-error-with-backoff.yml | 39 ++ .../pool-create-min-size-error.json | 17 +- .../pool-create-min-size-error.yml | 10 +- .../tests/logging/connection-logging.json | 3 +- .../tests/logging/connection-logging.yml | 1 + .../sdam-error-handling-pool-backoff.json | 541 ++++++++++++++++++ .../sdam-error-handling-pool-backoff.yml | 282 +++++++++ .../tests/sdam-error-handling.json | 47 +- .../tests/sdam-error-handling.yml | 23 +- .../unified/backoff-heartbeat-failure.json | 4 +- .../unified/backoff-heartbeat-failure.yml | 3 +- .../unified/backoff-heartbeat-success.json | 170 ------ .../unified/backoff-heartbeat-success.yml | 101 ---- .../unified/backoff-network-error-fail.json | 8 +- .../unified/backoff-network-error-fail.yml | 6 +- .../tests/unified/backoff-network-error.json | 34 ++ .../tests/unified/backoff-network-error.yml | 19 + .../backoff-network-timeout-error.json | 31 +- .../unified/backoff-network-timeout-error.yml | 19 +- 23 files changed, 1195 insertions(+), 348 deletions(-) create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-interrupting-pending-connections.json create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-interrupting-pending-connections.yml create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error-with-backoff.json create mode 100644 source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error-with-backoff.yml create mode 100644 source/load-balancers/tests/sdam-error-handling-pool-backoff.json create mode 100644 source/load-balancers/tests/sdam-error-handling-pool-backoff.yml delete mode 100644 source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json delete mode 100644 source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/README.md b/source/connection-monitoring-and-pooling/tests/cmap-format/README.md index cbfbc95343..740e8b4e5d 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/README.md +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/README.md @@ -89,6 +89,8 @@ The integration test format is identical to the unit test format with the additi - `maxServerVersion` (optional): The maximum server version (inclusive) against which the tests can be run successfully. If this field is omitted, it should be assumed that there is no upper bound on the required server version. + - `poolBackoff` (optional): If it is true, tests MUST only run if the driver supports backoff state in connection + pools. If it is false, tests MUST only run if the driver does not support backoff state in connection pools. - `failPoint`: optional, a document containing a `configureFailPoint` command to run against the endpoint being used for the test. - `poolOptions.appName` (optional): appName attribute to be set in connections, which will be affected by the fail diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-interrupting-pending-connections.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-interrupting-pending-connections.json new file mode 100644 index 0000000000..808f51838c --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-interrupting-pending-connections.json @@ -0,0 +1,76 @@ +{ + "version": 1, + "style": "integration", + "description": "backoff closes pending connections", + "runOn": [ + { + "minServerVersion": "4.9.0", + "poolBackoff": true + } + ], + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "closeConnection": false, + "blockConnection": true, + "blockTimeMS": 10000 + } + }, + "poolOptions": { + "minPoolSize": 0 + }, + "operations": [ + { + "name": "ready" + }, + { + "name": "start", + "target": "thread1" + }, + { + "name": "checkOut", + "thread": "thread1" + }, + { + "name": "waitForEvent", + "event": "ConnectionCreated", + "count": 1 + }, + { + "name": "backoff" + }, + { + "name": "waitForEvent", + "event": "ConnectionCheckOutFailed", + "count": 1 + } + ], + "events": [ + { + "type": "ConnectionCheckOutStarted" + }, + { + "type": "ConnectionCreated" + }, + { + "type": "ConnectionPoolBackoff" + }, + { + "type": "ConnectionClosed" + }, + { + "type": "ConnectionCheckOutFailed" + } + ], + "ignore": [ + "ConnectionCheckedIn", + "ConnectionCheckedOut", + "ConnectionPoolCreated", + "ConnectionPoolReady" + ] +} diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-interrupting-pending-connections.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-interrupting-pending-connections.yml new file mode 100644 index 0000000000..e35b8cc3e7 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-backoff-interrupting-pending-connections.yml @@ -0,0 +1,42 @@ +version: 1 +style: integration +description: backoff closes pending connections +runOn: + - minServerVersion: 4.9.0 + poolBackoff: true +failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - isMaster + - hello + closeConnection: false + blockConnection: true + blockTimeMS: 10000 +poolOptions: + minPoolSize: 0 +operations: + - name: ready + - name: start + target: thread1 + - name: checkOut + thread: thread1 + - name: waitForEvent + event: ConnectionCreated + count: 1 + - name: backoff + - name: waitForEvent + event: ConnectionCheckOutFailed + count: 1 +events: + - type: ConnectionCheckOutStarted + - type: ConnectionCreated + - type: ConnectionPoolBackoff + - type: ConnectionClosed + - type: ConnectionCheckOutFailed +ignore: + - ConnectionCheckedIn + - ConnectionCheckedOut + - ConnectionPoolCreated + - ConnectionPoolReady diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error-with-backoff.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error-with-backoff.json new file mode 100644 index 0000000000..a9debb8d38 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error-with-backoff.json @@ -0,0 +1,65 @@ +{ + "version": 1, + "style": "integration", + "description": "error during minPoolSize population clears pool", + "runOn": [ + { + "minServerVersion": "4.9.0", + "poolBackoff": true + } + ], + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "errorCode": 18, + "appName": "poolCreateMinSizeErrorTest" + } + }, + "poolOptions": { + "minPoolSize": 1, + "backgroundThreadIntervalMS": 50, + "appName": "poolCreateMinSizeErrorTest" + }, + "operations": [ + { + "name": "ready" + }, + { + "name": "waitForEvent", + "event": "ConnectionPoolCleared", + "count": 1 + }, + { + "name": "wait", + "ms": 200 + } + ], + "events": [ + { + "type": "ConnectionPoolReady", + "address": 42 + }, + { + "type": "ConnectionCreated", + "address": 42 + }, + { + "type": "ConnectionClosed", + "address": 42, + "connectionId": 42, + "reason": "error" + }, + { + "type": "ConnectionPoolCleared", + "address": 42 + } + ], + "ignore": [ + "ConnectionPoolCreated" + ] +} diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error-with-backoff.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error-with-backoff.yml new file mode 100644 index 0000000000..547f07bfaf --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error-with-backoff.yml @@ -0,0 +1,39 @@ +version: 1 +style: integration +description: error during minPoolSize population clears pool +runOn: + - minServerVersion: 4.9.0 + poolBackoff: true +failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - isMaster + - hello + errorCode: 18 + appName: poolCreateMinSizeErrorTest +poolOptions: + minPoolSize: 1 + backgroundThreadIntervalMS: 50 + appName: poolCreateMinSizeErrorTest +operations: + - name: ready + - name: waitForEvent + event: ConnectionPoolCleared + count: 1 + - name: wait + ms: 200 +events: + - type: ConnectionPoolReady + address: 42 + - type: ConnectionCreated + address: 42 + - type: ConnectionClosed + address: 42 + connectionId: 42 + reason: error + - type: ConnectionPoolCleared + address: 42 +ignore: + - ConnectionPoolCreated diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.json b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.json index da9357b963..b1bca0c3fd 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.json +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.json @@ -4,18 +4,21 @@ "description": "error during minPoolSize population clears pool", "runOn": [ { - "minServerVersion": "4.9.0" + "minServerVersion": "4.9.0", + "poolBackoff": false } ], "failPoint": { "configureFailPoint": "failCommand", - "mode": "alwaysOn", + "mode": { + "times": 50 + }, "data": { "failCommands": [ "isMaster", "hello" ], - "errorCode": 18, + "closeConnection": true, "appName": "poolCreateMinSizeErrorTest" } }, @@ -47,15 +50,15 @@ "type": "ConnectionCreated", "address": 42 }, + { + "type": "ConnectionPoolCleared", + "address": 42 + }, { "type": "ConnectionClosed", "address": 42, "connectionId": 42, "reason": "error" - }, - { - "type": "ConnectionPoolCleared", - "address": 42 } ], "ignore": [ diff --git a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.yml b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.yml index b658193710..5428dd1e3e 100644 --- a/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.yml +++ b/source/connection-monitoring-and-pooling/tests/cmap-format/pool-create-min-size-error.yml @@ -3,14 +3,16 @@ style: integration description: error during minPoolSize population clears pool runOn: - minServerVersion: 4.9.0 + poolBackoff: false failPoint: configureFailPoint: failCommand - mode: alwaysOn + mode: + times: 50 data: failCommands: - isMaster - hello - errorCode: 18 + closeConnection: true appName: poolCreateMinSizeErrorTest poolOptions: minPoolSize: 1 @@ -28,11 +30,11 @@ events: address: 42 - type: ConnectionCreated address: 42 + - type: ConnectionPoolCleared + address: 42 - type: ConnectionClosed address: 42 connectionId: 42 reason: error - - type: ConnectionPoolCleared - address: 42 ignore: - ConnectionPoolCreated diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json index f9f34ae95b..23e8ba60b0 100644 --- a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json +++ b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json @@ -523,7 +523,8 @@ "description": "Connection enters backoff on closed connection", "runOnRequirements": [ { - "minServerVersion": "4.4" + "minServerVersion": "4.4", + "poolBackoff": true } ], "operations": [ diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml index 985fe514bb..cb6e80d36e 100644 --- a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml +++ b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml @@ -321,6 +321,7 @@ tests: - description: Connection enters backoff on closed connection runOnRequirements: - minServerVersion: "4.4" + poolBackoff: true operations: - name: createEntities object: testRunner diff --git a/source/load-balancers/tests/sdam-error-handling-pool-backoff.json b/source/load-balancers/tests/sdam-error-handling-pool-backoff.json new file mode 100644 index 0000000000..fa04b03bfb --- /dev/null +++ b/source/load-balancers/tests/sdam-error-handling-pool-backoff.json @@ -0,0 +1,541 @@ +{ + "description": "state change errors are correctly handled", + "schemaVersion": "1.28", + "runOnRequirements": [ + { + "topologies": [ + "load-balanced" + ], + "poolBackoff": true + } + ], + "_yamlAnchors": { + "observedEvents": [ + "connectionCreatedEvent", + "connectionReadyEvent", + "connectionCheckedOutEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedInEvent", + "connectionClosedEvent", + "poolClearedEvent" + ] + }, + "createEntities": [ + { + "client": { + "id": "failPointClient", + "useMultipleMongoses": false + } + }, + { + "client": { + "id": "singleClient", + "useMultipleMongoses": false, + "uriOptions": { + "appname": "lbSDAMErrorTestClient", + "retryWrites": false + }, + "observeEvents": [ + "connectionCreatedEvent", + "connectionReadyEvent", + "connectionCheckedOutEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedInEvent", + "connectionClosedEvent", + "poolClearedEvent", + "poolBackoffEvent" + ] + } + }, + { + "database": { + "id": "singleDB", + "client": "singleClient", + "databaseName": "singleDB" + } + }, + { + "collection": { + "id": "singleColl", + "database": "singleDB", + "collectionName": "singleColl" + } + }, + { + "client": { + "id": "multiClient", + "useMultipleMongoses": true, + "uriOptions": { + "retryWrites": false + }, + "observeEvents": [ + "connectionCreatedEvent", + "connectionReadyEvent", + "connectionCheckedOutEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedInEvent", + "connectionClosedEvent", + "poolClearedEvent" + ] + } + }, + { + "database": { + "id": "multiDB", + "client": "multiClient", + "databaseName": "multiDB" + } + }, + { + "collection": { + "id": "multiColl", + "database": "multiDB", + "collectionName": "multiColl" + } + } + ], + "initialData": [ + { + "collectionName": "singleColl", + "databaseName": "singleDB", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + }, + { + "_id": 3 + } + ] + }, + { + "collectionName": "multiColl", + "databaseName": "multiDB", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + }, + { + "_id": 3 + } + ] + } + ], + "tests": [ + { + "description": "only connections for a specific serviceId are closed when pools are cleared", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], + "operations": [ + { + "name": "createFindCursor", + "object": "multiColl", + "arguments": { + "filter": {}, + "batchSize": 2 + }, + "saveResultAsEntity": "cursor0" + }, + { + "name": "createFindCursor", + "object": "multiColl", + "arguments": { + "filter": {}, + "batchSize": 2 + }, + "saveResultAsEntity": "cursor1" + }, + { + "name": "close", + "object": "cursor0" + }, + { + "name": "close", + "object": "cursor1" + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "multiClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 11600 + } + } + } + }, + { + "name": "insertOne", + "object": "multiColl", + "arguments": { + "document": { + "x": 1 + } + }, + "expectError": { + "errorCode": 11600 + } + }, + { + "name": "insertOne", + "object": "multiColl", + "arguments": { + "document": { + "x": 1 + } + } + } + ], + "expectEvents": [ + { + "client": "multiClient", + "eventType": "cmap", + "events": [ + { + "connectionCreatedEvent": {} + }, + { + "connectionReadyEvent": {} + }, + { + "connectionCheckedOutEvent": {} + }, + { + "connectionCreatedEvent": {} + }, + { + "connectionReadyEvent": {} + }, + { + "connectionCheckedOutEvent": {} + }, + { + "connectionCheckedInEvent": {} + }, + { + "connectionCheckedInEvent": {} + }, + { + "connectionCheckedOutEvent": {} + }, + { + "connectionCheckedInEvent": {} + }, + { + "connectionCheckedOutEvent": {} + }, + { + "poolClearedEvent": {} + }, + { + "connectionCheckedInEvent": {} + }, + { + "connectionClosedEvent": { + "reason": "stale" + } + }, + { + "connectionCheckedOutEvent": {} + }, + { + "connectionCheckedInEvent": {} + } + ] + } + ] + }, + { + "description": "errors during the initial connection hello trigger backoff", + "runOnRequirements": [ + { + "minServerVersion": "4.4.7" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "closeConnection": true, + "appName": "lbSDAMErrorTestClient" + } + } + } + }, + { + "name": "insertOne", + "object": "singleColl", + "arguments": { + "document": { + "x": 1 + } + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "singleClient", + "event": { + "poolBackoffEvent": {} + }, + "count": 1 + } + } + ], + "expectEvents": [ + { + "client": "singleClient", + "eventType": "cmap", + "events": [ + { + "connectionCreatedEvent": {} + }, + { + "connectionClosedEvent": { + "reason": "error" + } + }, + { + "poolBackoffEvent": { + "attempt": 1 + } + }, + { + "connectionCheckOutFailedEvent": { + "reason": "connectionError" + } + }, + { + "connectionCreatedEvent": {} + }, + { + "connectionReadyEvent": {} + }, + { + "connectionCheckedOutEvent": {} + }, + { + "connectionCheckedInEvent": {} + } + ] + } + ] + }, + { + "description": "errors during authentication are processed", + "runOnRequirements": [ + { + "auth": true + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "saslContinue" + ], + "closeConnection": true, + "appName": "lbSDAMErrorTestClient" + } + } + } + }, + { + "name": "insertOne", + "object": "singleColl", + "arguments": { + "document": { + "x": 1 + } + }, + "expectError": { + "isClientError": true + } + } + ], + "expectEvents": [ + { + "client": "singleClient", + "eventType": "cmap", + "events": [ + { + "connectionCreatedEvent": {} + }, + { + "connectionClosedEvent": { + "reason": "error" + } + }, + { + "connectionCheckOutFailedEvent": { + "reason": "connectionError" + } + }, + { + "poolClearedEvent": {} + } + ] + } + ] + }, + { + "description": "stale errors are ignored", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "getMore" + ], + "closeConnection": true + } + } + } + }, + { + "name": "createFindCursor", + "object": "singleColl", + "arguments": { + "filter": {}, + "batchSize": 2 + }, + "saveResultAsEntity": "cursor0" + }, + { + "name": "createFindCursor", + "object": "singleColl", + "arguments": { + "filter": {}, + "batchSize": 2 + }, + "saveResultAsEntity": "cursor1" + }, + { + "name": "iterateUntilDocumentOrError", + "object": "cursor0" + }, + { + "name": "iterateUntilDocumentOrError", + "object": "cursor0" + }, + { + "name": "iterateUntilDocumentOrError", + "object": "cursor0", + "expectError": { + "isClientError": true + } + }, + { + "name": "close", + "object": "cursor0" + }, + { + "name": "iterateUntilDocumentOrError", + "object": "cursor1" + }, + { + "name": "iterateUntilDocumentOrError", + "object": "cursor1" + }, + { + "name": "iterateUntilDocumentOrError", + "object": "cursor1", + "expectError": { + "isClientError": true + } + }, + { + "name": "close", + "object": "cursor1" + } + ], + "expectEvents": [ + { + "client": "singleClient", + "eventType": "cmap", + "events": [ + { + "connectionCreatedEvent": {} + }, + { + "connectionReadyEvent": {} + }, + { + "connectionCheckedOutEvent": {} + }, + { + "connectionCreatedEvent": {} + }, + { + "connectionReadyEvent": {} + }, + { + "connectionCheckedOutEvent": {} + }, + { + "poolClearedEvent": {} + }, + { + "connectionCheckedInEvent": {} + }, + { + "connectionClosedEvent": {} + }, + { + "connectionCheckedInEvent": {} + }, + { + "connectionClosedEvent": {} + } + ] + } + ] + } + ] +} diff --git a/source/load-balancers/tests/sdam-error-handling-pool-backoff.yml b/source/load-balancers/tests/sdam-error-handling-pool-backoff.yml new file mode 100644 index 0000000000..bb3f0f7df4 --- /dev/null +++ b/source/load-balancers/tests/sdam-error-handling-pool-backoff.yml @@ -0,0 +1,282 @@ +description: state change errors are correctly handled +schemaVersion: "1.28" +runOnRequirements: + - topologies: + - load-balanced + poolBackoff: true +_yamlAnchors: + observedEvents: + - connectionCreatedEvent + - connectionReadyEvent + - connectionCheckedOutEvent + - connectionCheckOutFailedEvent + - connectionCheckedInEvent + - connectionClosedEvent + - poolClearedEvent +createEntities: + - client: + id: failPointClient + useMultipleMongoses: false + - client: + id: singleClient + useMultipleMongoses: false + uriOptions: + appname: lbSDAMErrorTestClient + retryWrites: false + observeEvents: + - connectionCreatedEvent + - connectionReadyEvent + - connectionCheckedOutEvent + - connectionCheckOutFailedEvent + - connectionCheckedInEvent + - connectionClosedEvent + - poolClearedEvent + - poolBackoffEvent + - database: + id: singleDB + client: singleClient + databaseName: singleDB + - collection: + id: singleColl + database: singleDB + collectionName: singleColl + - client: + id: multiClient + useMultipleMongoses: true + uriOptions: + retryWrites: false + observeEvents: + - connectionCreatedEvent + - connectionReadyEvent + - connectionCheckedOutEvent + - connectionCheckOutFailedEvent + - connectionCheckedInEvent + - connectionClosedEvent + - poolClearedEvent + - database: + id: multiDB + client: multiClient + databaseName: multiDB + - collection: + id: multiColl + database: multiDB + collectionName: multiColl +initialData: + - collectionName: singleColl + databaseName: singleDB + documents: + - _id: 1 + - _id: 2 + - _id: 3 + - collectionName: multiColl + databaseName: multiDB + documents: + - _id: 1 + - _id: 2 + - _id: 3 +tests: + - description: only connections for a specific serviceId are closed when pools are cleared + runOnRequirements: + - serverless: forbid + operations: + - name: createFindCursor + object: multiColl + arguments: + filter: {} + batchSize: 2 + saveResultAsEntity: cursor0 + - name: createFindCursor + object: multiColl + arguments: + filter: {} + batchSize: 2 + saveResultAsEntity: cursor1 + - name: close + object: cursor0 + - name: close + object: cursor1 + - name: failPoint + object: testRunner + arguments: + client: multiClient + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - insert + errorCode: 11600 + - name: insertOne + object: multiColl + arguments: + document: + x: 1 + expectError: + errorCode: 11600 + - name: insertOne + object: multiColl + arguments: + document: + x: 1 + expectEvents: + - client: multiClient + eventType: cmap + events: + - connectionCreatedEvent: {} + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCreatedEvent: {} + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + - connectionCheckedInEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + - connectionCheckedOutEvent: {} + - poolClearedEvent: {} + - connectionCheckedInEvent: {} + - connectionClosedEvent: + reason: stale + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + - description: errors during the initial connection hello trigger backoff + runOnRequirements: + - minServerVersion: 4.4.7 + operations: + - name: failPoint + object: testRunner + arguments: + client: failPointClient + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - isMaster + - hello + closeConnection: true + appName: lbSDAMErrorTestClient + - name: insertOne + object: singleColl + arguments: + document: + x: 1 + - name: waitForEvent + object: testRunner + arguments: + client: singleClient + event: + poolBackoffEvent: {} + count: 1 + expectEvents: + - client: singleClient + eventType: cmap + events: + - connectionCreatedEvent: {} + - connectionClosedEvent: + reason: error + - poolBackoffEvent: + attempt: 1 + - connectionCheckOutFailedEvent: + reason: connectionError + - connectionCreatedEvent: {} + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCheckedInEvent: {} + - description: errors during authentication are processed + runOnRequirements: + - auth: true + operations: + - name: failPoint + object: testRunner + arguments: + client: failPointClient + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - saslContinue + closeConnection: true + appName: lbSDAMErrorTestClient + - name: insertOne + object: singleColl + arguments: + document: + x: 1 + expectError: + isClientError: true + expectEvents: + - client: singleClient + eventType: cmap + events: + - connectionCreatedEvent: {} + - connectionClosedEvent: + reason: error + - connectionCheckOutFailedEvent: + reason: connectionError + - poolClearedEvent: {} + - description: stale errors are ignored + operations: + - name: failPoint + object: testRunner + arguments: + client: failPointClient + failPoint: + configureFailPoint: failCommand + mode: + times: 2 + data: + failCommands: + - getMore + closeConnection: true + - name: createFindCursor + object: singleColl + arguments: + filter: {} + batchSize: 2 + saveResultAsEntity: cursor0 + - name: createFindCursor + object: singleColl + arguments: + filter: {} + batchSize: 2 + saveResultAsEntity: cursor1 + - name: iterateUntilDocumentOrError + object: cursor0 + - name: iterateUntilDocumentOrError + object: cursor0 + - name: iterateUntilDocumentOrError + object: cursor0 + expectError: + isClientError: true + - name: close + object: cursor0 + - name: iterateUntilDocumentOrError + object: cursor1 + - name: iterateUntilDocumentOrError + object: cursor1 + - name: iterateUntilDocumentOrError + object: cursor1 + expectError: + isClientError: true + - name: close + object: cursor1 + expectEvents: + - client: singleClient + eventType: cmap + events: + - connectionCreatedEvent: {} + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - connectionCreatedEvent: {} + - connectionReadyEvent: {} + - connectionCheckedOutEvent: {} + - poolClearedEvent: {} + - connectionCheckedInEvent: {} + - connectionClosedEvent: {} + - connectionCheckedInEvent: {} + - connectionClosedEvent: {} diff --git a/source/load-balancers/tests/sdam-error-handling.json b/source/load-balancers/tests/sdam-error-handling.json index 7654eff6fb..7c471abbda 100644 --- a/source/load-balancers/tests/sdam-error-handling.json +++ b/source/load-balancers/tests/sdam-error-handling.json @@ -1,11 +1,12 @@ { "description": "state change errors are correctly handled", - "schemaVersion": "1.28", + "schemaVersion": "1.4", "runOnRequirements": [ { "topologies": [ "load-balanced" - ] + ], + "poolBackoff": false } ], "_yamlAnchors": { @@ -41,8 +42,7 @@ "connectionCheckOutFailedEvent", "connectionCheckedInEvent", "connectionClosedEvent", - "poolClearedEvent", - "poolBackoffEvent" + "poolClearedEvent" ] } }, @@ -261,7 +261,7 @@ ] }, { - "description": "errors during the initial connection hello trigger backoff", + "description": "errors during the initial connection hello are ignored", "runOnRequirements": [ { "minServerVersion": "4.4.7" @@ -296,17 +296,9 @@ "document": { "x": 1 } - } - }, - { - "name": "waitForEvent", - "object": "testRunner", - "arguments": { - "client": "singleClient", - "event": { - "poolBackoffEvent": {} - }, - "count": 1 + }, + "expectError": { + "isClientError": true } } ], @@ -323,27 +315,10 @@ "reason": "error" } }, - { - "poolBackoffEvent": { - "attempt": 1 - } - }, { "connectionCheckOutFailedEvent": { "reason": "connectionError" } - }, - { - "connectionCreatedEvent": {} - }, - { - "connectionReadyEvent": {} - }, - { - "connectionCheckedOutEvent": {} - }, - { - "connectionCheckedInEvent": {} } ] } @@ -398,6 +373,9 @@ { "connectionCreatedEvent": {} }, + { + "poolClearedEvent": {} + }, { "connectionClosedEvent": { "reason": "error" @@ -407,9 +385,6 @@ "connectionCheckOutFailedEvent": { "reason": "connectionError" } - }, - { - "poolClearedEvent": {} } ] } diff --git a/source/load-balancers/tests/sdam-error-handling.yml b/source/load-balancers/tests/sdam-error-handling.yml index 2b2e2ea54d..4fd15a2b7a 100644 --- a/source/load-balancers/tests/sdam-error-handling.yml +++ b/source/load-balancers/tests/sdam-error-handling.yml @@ -1,8 +1,9 @@ description: state change errors are correctly handled -schemaVersion: "1.28" +schemaVersion: "1.4" runOnRequirements: - topologies: - load-balanced + poolBackoff: false _yamlAnchors: observedEvents: - connectionCreatedEvent @@ -30,7 +31,6 @@ createEntities: - connectionCheckedInEvent - connectionClosedEvent - poolClearedEvent - - poolBackoffEvent - database: id: singleDB client: singleClient @@ -139,7 +139,7 @@ tests: reason: stale - connectionCheckedOutEvent: {} - connectionCheckedInEvent: {} - - description: errors during the initial connection hello trigger backoff + - description: errors during the initial connection hello are ignored runOnRequirements: - minServerVersion: 4.4.7 operations: @@ -162,13 +162,8 @@ tests: arguments: document: x: 1 - - name: waitForEvent - object: testRunner - arguments: - client: singleClient - event: - poolBackoffEvent: {} - count: 1 + expectError: + isClientError: true expectEvents: - client: singleClient eventType: cmap @@ -176,14 +171,8 @@ tests: - connectionCreatedEvent: {} - connectionClosedEvent: reason: error - - poolBackoffEvent: - attempt: 1 - connectionCheckOutFailedEvent: reason: connectionError - - connectionCreatedEvent: {} - - connectionReadyEvent: {} - - connectionCheckedOutEvent: {} - - connectionCheckedInEvent: {} - description: errors during authentication are processed runOnRequirements: - auth: true @@ -213,11 +202,11 @@ tests: eventType: cmap events: - connectionCreatedEvent: {} + - poolClearedEvent: {} - connectionClosedEvent: reason: error - connectionCheckOutFailedEvent: reason: connectionError - - poolClearedEvent: {} - description: stale errors are ignored operations: - name: failPoint diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json index 8c6e225916..aa253edc71 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.json @@ -137,7 +137,9 @@ "arguments": { "client": "client", "event": { - "poolBackoffEvent": {} + "poolBackoffEvent": { + "attempt": 1 + } }, "count": 1 } diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml index e908f0a7a1..9b23d54093 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-failure.yml @@ -83,7 +83,8 @@ tests: arguments: client: client event: - poolBackoffEvent: {} + poolBackoffEvent: + attempt: 1 count: 1 - name: waitForEvent object: testRunner diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json deleted file mode 100644 index beeed33fb0..0000000000 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.json +++ /dev/null @@ -1,170 +0,0 @@ -{ - "description": "heartbeat-success-backoff", - "schemaVersion": "1.28", - "runOnRequirements": [ - { - "minServerVersion": "4.4", - "serverless": "forbid", - "poolBackoff": true, - "topologies": [ - "single", - "replicaset", - "sharded" - ] - } - ], - "createEntities": [ - { - "client": { - "id": "setupClient", - "useMultipleMongoses": false - } - } - ], - "initialData": [ - { - "collectionName": "heartbeat-backoff-success", - "databaseName": "sdam-tests", - "documents": [ - { - "_id": 1 - }, - { - "_id": 2 - } - ] - } - ], - "tests": [ - { - "description": "A heartbeat success during backoff not mark the pool as ready", - "operations": [ - { - "name": "createEntities", - "object": "testRunner", - "arguments": { - "entities": [ - { - "client": { - "id": "client", - "useMultipleMongoses": false, - "observeEvents": [ - "commandStartedEvent", - "poolBackoffEvent", - "poolClearedEvent", - "serverHeartbeatFailedEvent", - "serverHeartbeatSucceededEvent" - ], - "uriOptions": { - "retryWrites": false, - "heartbeatFrequencyMS": 5000, - "serverMonitoringMode": "poll", - "appname": "heartbeatBackoffSuccessTest" - } - } - }, - { - "database": { - "id": "database", - "client": "client", - "databaseName": "sdam-tests" - } - }, - { - "collection": { - "id": "collection", - "database": "database", - "collectionName": "heartbeat-backoff-success" - } - } - ] - } - }, - { - "name": "waitForEvent", - "object": "testRunner", - "arguments": { - "client": "client", - "event": { - "serverHeartbeatSucceededEvent": {} - }, - "count": 1 - } - }, - { - "name": "failPoint", - "object": "testRunner", - "arguments": { - "client": "setupClient", - "failPoint": { - "configureFailPoint": "failCommand", - "mode": "alwaysOn", - "data": { - "failCommands": [ - "isMaster", - "hello" - ], - "appName": "heartbeatBackoffSuccessTest", - "closeConnection": true - } - } - } - }, - { - "name": "insertMany", - "object": "collection", - "arguments": { - "documents": [ - { - "_id": 3 - }, - { - "_id": 4 - } - ] - }, - "expectError": { - "isError": true, - "errorLabelsContain": [ - "SystemOverloadedError", - "RetryableError" - ] - } - }, - { - "name": "waitForEvent", - "object": "testRunner", - "arguments": { - "client": "client", - "event": { - "poolBackoffEvent": {} - }, - "count": 1 - } - }, - { - "name": "failPoint", - "object": "testRunner", - "arguments": { - "client": "setupClient", - "failPoint": { - "configureFailPoint": "failCommand", - "mode": "off" - } - } - }, - { - "name": "waitForEvent", - "object": "testRunner", - "arguments": { - "client": "client", - "event": { - "serverHeartbeatSucceededEvent": {} - }, - "count": 1 - } - } - ] - } - ] -} diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml deleted file mode 100644 index dbe7715b65..0000000000 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-heartbeat-success.yml +++ /dev/null @@ -1,101 +0,0 @@ -description: heartbeat-success-backoff -schemaVersion: "1.28" -runOnRequirements: - - minServerVersion: "4.4" - serverless: forbid - poolBackoff: true - topologies: - - single - - replicaset - - sharded -createEntities: - - client: - id: setupClient - useMultipleMongoses: false -initialData: - - collectionName: heartbeat-backoff-success - databaseName: sdam-tests - documents: - - _id: 1 - - _id: 2 -tests: - - description: A heartbeat success during backoff not mark the pool as ready - operations: - - name: createEntities - object: testRunner - arguments: - entities: - - client: - id: client - useMultipleMongoses: false - observeEvents: - - commandStartedEvent - - poolBackoffEvent - - poolClearedEvent - - serverHeartbeatFailedEvent - - serverHeartbeatSucceededEvent - uriOptions: - retryWrites: false - heartbeatFrequencyMS: 5000 - serverMonitoringMode: poll - appname: heartbeatBackoffSuccessTest - - database: - id: database - client: client - databaseName: sdam-tests - - collection: - id: collection - database: database - collectionName: heartbeat-backoff-success - - name: waitForEvent - object: testRunner - arguments: - client: client - event: - serverHeartbeatSucceededEvent: {} - count: 1 - - name: failPoint - object: testRunner - arguments: - client: setupClient - failPoint: - configureFailPoint: failCommand - mode: alwaysOn - data: - failCommands: - - isMaster - - hello - appName: heartbeatBackoffSuccessTest - closeConnection: true - - name: insertMany - object: collection - arguments: - documents: - - _id: 3 - - _id: 4 - expectError: - isError: true - errorLabelsContain: - - SystemOverloadedError - - RetryableError - - name: waitForEvent - object: testRunner - arguments: - client: client - event: - poolBackoffEvent: {} - count: 1 - - name: failPoint - object: testRunner - arguments: - client: setupClient - failPoint: - configureFailPoint: failCommand - mode: off - - name: waitForEvent - object: testRunner - arguments: - client: client - event: - serverHeartbeatSucceededEvent: {} - count: 1 diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json index c0ae2adc9c..648008e944 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.json @@ -154,7 +154,9 @@ "arguments": { "client": "client", "event": { - "poolBackoffEvent": {} + "poolBackoffEvent": { + "attempt": 1 + } }, "count": 1 } @@ -262,7 +264,9 @@ "arguments": { "client": "client", "event": { - "poolBackoffEvent": {} + "poolBackoffEvent": { + "attempt": 1 + } }, "count": 1 } diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml index 7fbe8cb551..2d76b300cd 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error-fail.yml @@ -91,7 +91,8 @@ tests: arguments: client: client event: - poolBackoffEvent: {} + poolBackoffEvent: + attempt: 1 count: 1 - description: Backoff and clear the pool after network failures followed by server error operations: @@ -157,7 +158,8 @@ tests: arguments: client: client event: - poolBackoffEvent: {} + poolBackoffEvent: + attempt: 1 count: 1 - name: failPoint object: testRunner diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json index 25edcfaf93..3101451d8b 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.json @@ -147,6 +147,40 @@ }, "count": 1 } + }, + { + "name": "insertMany", + "object": "collection", + "arguments": { + "documents": [ + { + "_id": 3 + }, + { + "_id": 4 + } + ] + }, + "expectError": { + "isError": true, + "errorLabelsContain": [ + "SystemOverloadedError", + "RetryableError" + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "poolBackoffEvent": { + "attempt": 2 + } + }, + "count": 1 + } } ] } diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml index a357864ff0..b7f7d34051 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-error.yml @@ -89,3 +89,22 @@ tests: poolBackoffEvent: attempt: 1 count: 1 + - name: insertMany + object: collection + arguments: + documents: + - _id: 3 + - _id: 4 + expectError: + isError: true + errorLabelsContain: + - SystemOverloadedError + - RetryableError + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + poolBackoffEvent: + attempt: 2 + count: 1 diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json index 2cad0f685c..7969e4b908 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.json @@ -101,7 +101,7 @@ "failPoint": { "configureFailPoint": "failCommand", "mode": { - "times": 1 + "times": 2 }, "data": { "failCommands": [ @@ -141,18 +141,43 @@ "arguments": { "client": "client", "event": { - "poolBackoffEvent": {} + "poolBackoffEvent": { + "attempt": 1 + } }, "count": 1 } }, + { + "name": "insertMany", + "object": "collection", + "arguments": { + "documents": [ + { + "_id": 3 + }, + { + "_id": 4 + } + ] + }, + "expectError": { + "isError": true, + "errorLabelsContain": [ + "SystemOverloadedError", + "RetryableError" + ] + } + }, { "name": "waitForEvent", "object": "testRunner", "arguments": { "client": "client", "event": { - "poolReadyEvent": {} + "poolBackoffEvent": { + "attempt": 2 + } }, "count": 1 } diff --git a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml index 18826072ea..b173c772d8 100644 --- a/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml +++ b/source/server-discovery-and-monitoring/tests/unified/backoff-network-timeout-error.yml @@ -63,7 +63,7 @@ tests: failPoint: configureFailPoint: failCommand mode: - times: 1 + times: 2 data: failCommands: - isMaster @@ -86,12 +86,25 @@ tests: arguments: client: client event: - poolBackoffEvent: {} + poolBackoffEvent: + attempt: 1 count: 1 + - name: insertMany + object: collection + arguments: + documents: + - _id: 3 + - _id: 4 + expectError: + isError: true + errorLabelsContain: + - SystemOverloadedError + - RetryableError - name: waitForEvent object: testRunner arguments: client: client event: - poolReadyEvent: {} + poolBackoffEvent: + attempt: 2 count: 1 From a8ffc2a64fb30526200d4468f95f7e991fb94697 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 3 Nov 2025 08:28:32 -0600 Subject: [PATCH 18/23] bump schema version --- source/load-balancers/tests/sdam-error-handling.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/load-balancers/tests/sdam-error-handling.yml b/source/load-balancers/tests/sdam-error-handling.yml index 4fd15a2b7a..3e0ea1af5e 100644 --- a/source/load-balancers/tests/sdam-error-handling.yml +++ b/source/load-balancers/tests/sdam-error-handling.yml @@ -1,5 +1,5 @@ description: state change errors are correctly handled -schemaVersion: "1.4" +schemaVersion: "1.28" runOnRequirements: - topologies: - load-balanced From db4b1a3340fde6e9e99d3e0486d150e2084adc89 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 3 Nov 2025 08:33:22 -0600 Subject: [PATCH 19/23] bump schema version --- source/load-balancers/tests/sdam-error-handling.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/load-balancers/tests/sdam-error-handling.json b/source/load-balancers/tests/sdam-error-handling.json index 7c471abbda..425144a9db 100644 --- a/source/load-balancers/tests/sdam-error-handling.json +++ b/source/load-balancers/tests/sdam-error-handling.json @@ -1,6 +1,6 @@ { "description": "state change errors are correctly handled", - "schemaVersion": "1.4", + "schemaVersion": "1.28", "runOnRequirements": [ { "topologies": [ From c7608cc1a7b64a458ee215155c77963c97434ac3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 3 Nov 2025 08:35:03 -0600 Subject: [PATCH 20/23] undo yaml changes --- .../tests/sdam-error-handling.yml | 208 +++++++++--------- 1 file changed, 106 insertions(+), 102 deletions(-) diff --git a/source/load-balancers/tests/sdam-error-handling.yml b/source/load-balancers/tests/sdam-error-handling.yml index 3e0ea1af5e..98a4e01bb5 100644 --- a/source/load-balancers/tests/sdam-error-handling.yml +++ b/source/load-balancers/tests/sdam-error-handling.yml @@ -1,11 +1,13 @@ description: state change errors are correctly handled -schemaVersion: "1.28" + +schemaVersion: '1.28' + runOnRequirements: - - topologies: - - load-balanced + - topologies: [ load-balanced ] poolBackoff: false + _yamlAnchors: - observedEvents: + observedEvents: &observedEvents - connectionCreatedEvent - connectionReadyEvent - connectionCheckedOutEvent @@ -13,159 +15,155 @@ _yamlAnchors: - connectionCheckedInEvent - connectionClosedEvent - poolClearedEvent + createEntities: - client: - id: failPointClient + id: &failPointClient failPointClient useMultipleMongoses: false - client: - id: singleClient + id: &singleClient singleClient useMultipleMongoses: false uriOptions: - appname: lbSDAMErrorTestClient + appname: &singleClientAppName lbSDAMErrorTestClient retryWrites: false - observeEvents: - - connectionCreatedEvent - - connectionReadyEvent - - connectionCheckedOutEvent - - connectionCheckOutFailedEvent - - connectionCheckedInEvent - - connectionClosedEvent - - poolClearedEvent + observeEvents: *observedEvents - database: - id: singleDB - client: singleClient - databaseName: singleDB + id: &singleDB singleDB + client: *singleClient + databaseName: &singleDBName singleDB - collection: - id: singleColl - database: singleDB - collectionName: singleColl + id: &singleColl singleColl + database: *singleDB + collectionName: &singleCollName singleColl - client: - id: multiClient + id: &multiClient multiClient useMultipleMongoses: true uriOptions: retryWrites: false - observeEvents: - - connectionCreatedEvent - - connectionReadyEvent - - connectionCheckedOutEvent - - connectionCheckOutFailedEvent - - connectionCheckedInEvent - - connectionClosedEvent - - poolClearedEvent + observeEvents: *observedEvents - database: - id: multiDB - client: multiClient - databaseName: multiDB + id: &multiDB multiDB + client: *multiClient + databaseName: &multiDBName multiDB - collection: - id: multiColl - database: multiDB - collectionName: multiColl + id: &multiColl multiColl + database: *multiDB + collectionName: &multiCollName multiColl + initialData: - - collectionName: singleColl - databaseName: singleDB + - collectionName: *singleCollName + databaseName: *singleDBName documents: - _id: 1 - _id: 2 - _id: 3 - - collectionName: multiColl - databaseName: multiDB + - collectionName: *multiCollName + databaseName: *multiDBName documents: - _id: 1 - _id: 2 - _id: 3 + tests: - description: only connections for a specific serviceId are closed when pools are cleared runOnRequirements: - - serverless: forbid + # This test assumes that two sequential connections receive different serviceIDs. + # Sequential connections to a serverless instance may receive the same serviceID. + - serverless: forbid operations: + # Create two cursors to force two connections. - name: createFindCursor - object: multiColl + object: *multiColl arguments: filter: {} batchSize: 2 - saveResultAsEntity: cursor0 + saveResultAsEntity: &cursor0 cursor0 - name: createFindCursor - object: multiColl + object: *multiColl arguments: filter: {} batchSize: 2 - saveResultAsEntity: cursor1 + saveResultAsEntity: &cursor1 cursor1 + # Close both cursors to return the connections to the pool. - name: close - object: cursor0 + object: *cursor0 - name: close - object: cursor1 + object: *cursor1 + # Fail an operation with a state change error. - name: failPoint object: testRunner arguments: - client: multiClient + client: *multiClient failPoint: configureFailPoint: failCommand - mode: - times: 1 + mode: { times: 1 } data: - failCommands: - - insert - errorCode: 11600 + failCommands: [insert] + errorCode: &errorCode 11600 # InterruptedAtShutdown - name: insertOne - object: multiColl + object: *multiColl arguments: - document: - x: 1 + document: { x: 1 } expectError: - errorCode: 11600 + errorCode: *errorCode + # Do another operation to ensure the relevant connection has been closed. - name: insertOne - object: multiColl + object: *multiColl arguments: - document: - x: 1 + document: { x: 1 } expectEvents: - - client: multiClient + - client: *multiClient eventType: cmap events: + # Create cursors. - connectionCreatedEvent: {} - connectionReadyEvent: {} - connectionCheckedOutEvent: {} - connectionCreatedEvent: {} - connectionReadyEvent: {} - connectionCheckedOutEvent: {} + # Close cursors. - connectionCheckedInEvent: {} - connectionCheckedInEvent: {} + # Set failpoint. - connectionCheckedOutEvent: {} - connectionCheckedInEvent: {} + # First insertOne. - connectionCheckedOutEvent: {} - poolClearedEvent: {} - connectionCheckedInEvent: {} - connectionClosedEvent: reason: stale + # Second insertOne. - connectionCheckedOutEvent: {} - connectionCheckedInEvent: {} + + # This test uses singleClient to ensure that connection attempts are routed + # to the same mongos on which the failpoint is set. - description: errors during the initial connection hello are ignored runOnRequirements: - - minServerVersion: 4.4.7 + # Require SERVER-49336 for failCommand + appName on the initial handshake. + - minServerVersion: '4.4.7' operations: - name: failPoint object: testRunner arguments: - client: failPointClient + client: *failPointClient failPoint: configureFailPoint: failCommand - mode: - times: 1 + mode: { times: 1 } data: - failCommands: - - isMaster - - hello + failCommands: [isMaster, hello] closeConnection: true - appName: lbSDAMErrorTestClient + appName: *singleClientAppName - name: insertOne - object: singleColl + object: *singleColl arguments: - document: - x: 1 + document: { x: 1 } expectError: isClientError: true expectEvents: - - client: singleClient + - client: *singleClient eventType: cmap events: - connectionCreatedEvent: {} @@ -173,6 +171,7 @@ tests: reason: error - connectionCheckOutFailedEvent: reason: connectionError + - description: errors during authentication are processed runOnRequirements: - auth: true @@ -180,25 +179,22 @@ tests: - name: failPoint object: testRunner arguments: - client: failPointClient + client: *failPointClient failPoint: configureFailPoint: failCommand - mode: - times: 1 + mode: { times: 1 } data: - failCommands: - - saslContinue + failCommands: [saslContinue] closeConnection: true - appName: lbSDAMErrorTestClient + appName: *singleClientAppName - name: insertOne - object: singleColl + object: *singleColl arguments: - document: - x: 1 + document: { x: 1 } expectError: isClientError: true expectEvents: - - client: singleClient + - client: *singleClient eventType: cmap events: - connectionCreatedEvent: {} @@ -207,64 +203,72 @@ tests: reason: error - connectionCheckOutFailedEvent: reason: connectionError + - description: stale errors are ignored operations: - name: failPoint object: testRunner arguments: - client: failPointClient + client: *failPointClient failPoint: configureFailPoint: failCommand - mode: - times: 2 + mode: { times: 2 } data: - failCommands: - - getMore + failCommands: [getMore] closeConnection: true + # Force two connections to be checked out from the pool. - name: createFindCursor - object: singleColl + object: *singleColl arguments: filter: {} batchSize: 2 - saveResultAsEntity: cursor0 + saveResultAsEntity: &cursor0 cursor0 - name: createFindCursor - object: singleColl + object: *singleColl arguments: filter: {} batchSize: 2 - saveResultAsEntity: cursor1 + saveResultAsEntity: &cursor1 cursor1 + # Iterate cursor0 three times to force a network error. - name: iterateUntilDocumentOrError - object: cursor0 + object: *cursor0 - name: iterateUntilDocumentOrError - object: cursor0 + object: *cursor0 - name: iterateUntilDocumentOrError - object: cursor0 + object: *cursor0 expectError: isClientError: true - name: close - object: cursor0 + object: *cursor0 + # Iterate cursor1 three times to force a network error. - name: iterateUntilDocumentOrError - object: cursor1 + object: *cursor1 - name: iterateUntilDocumentOrError - object: cursor1 + object: *cursor1 - name: iterateUntilDocumentOrError - object: cursor1 + object: *cursor1 expectError: isClientError: true - name: close - object: cursor1 + object: *cursor1 expectEvents: - - client: singleClient + - client: *singleClient eventType: cmap events: + # Events for creating both cursors. - connectionCreatedEvent: {} - connectionReadyEvent: {} - connectionCheckedOutEvent: {} - connectionCreatedEvent: {} - connectionReadyEvent: {} - connectionCheckedOutEvent: {} + # Events for iterating and closing the first cursor. The failed + # getMore should cause a poolClearedEvent to be published. - poolClearedEvent: {} - connectionCheckedInEvent: {} - connectionClosedEvent: {} + # Events for iterating and closing the second cursor. The failed + # getMore should not clear the pool because the connection's + # generation number is stale. - connectionCheckedInEvent: {} - - connectionClosedEvent: {} + - connectionClosedEvent: {} \ No newline at end of file From ab2eb8784b686cceaf7c0bd9ab5155357d60a866 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 3 Nov 2025 08:38:52 -0600 Subject: [PATCH 21/23] undo connection-logging changes --- .../tests/logging/connection-logging.json | 364 +---------- .../tests/logging/connection-logging.yml | 565 ++++-------------- 2 files changed, 125 insertions(+), 804 deletions(-) diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json index 23e8ba60b0..3482f59777 100644 --- a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json +++ b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json @@ -355,7 +355,7 @@ "failCommands": [ "saslContinue" ], - "errorCode": 18, + "closeConnection": true, "appName": "clientAppName" } } @@ -368,7 +368,7 @@ "filter": {} }, "expectError": { - "isError": true + "isClientError": true } } ], @@ -446,59 +446,6 @@ } } }, - { - "level": "debug", - "component": "connection", - "data": { - "message": "Connection closed", - "driverConnectionId": { - "$$type": [ - "int", - "long" - ] - }, - "serverHost": { - "$$type": "string" - }, - "serverPort": { - "$$type": [ - "int", - "long" - ] - }, - "reason": "An error occurred while using the connection", - "error": { - "$$exists": true - } - } - }, - { - "level": "debug", - "component": "connection", - "data": { - "message": "Connection checkout failed", - "serverHost": { - "$$type": "string" - }, - "serverPort": { - "$$type": [ - "int", - "long" - ] - }, - "reason": "An error occurred while trying to establish a new connection", - "error": { - "$$exists": true - }, - "durationMS": { - "$$type": [ - "double", - "int", - "long" - ] - } - } - }, { "level": "debug", "component": "connection", @@ -514,167 +461,6 @@ ] } } - } - ] - } - ] - }, - { - "description": "Connection enters backoff on closed connection", - "runOnRequirements": [ - { - "minServerVersion": "4.4", - "poolBackoff": true - } - ], - "operations": [ - { - "name": "createEntities", - "object": "testRunner", - "arguments": { - "entities": [ - { - "client": { - "id": "client", - "uriOptions": { - "retryReads": true, - "appname": "clientAppName", - "heartbeatFrequencyMS": 5000 - }, - "observeEvents": [ - "serverHeartbeatSucceededEvent" - ], - "observeLogMessages": { - "connection": "debug" - } - } - }, - { - "database": { - "id": "database0", - "client": "client", - "databaseName": "ci-tests" - } - } - ] - } - }, - { - "name": "waitForEvent", - "object": "testRunner", - "arguments": { - "client": "client", - "event": { - "serverHeartbeatSucceededEvent": {} - }, - "count": 1 - } - }, - { - "name": "failPoint", - "object": "testRunner", - "arguments": { - "client": "failPointClient", - "failPoint": { - "configureFailPoint": "failCommand", - "mode": { - "times": 1 - }, - "data": { - "failCommands": [ - "isMaster", - "hello" - ], - "closeConnection": true, - "appName": "clientAppName" - } - } - } - }, - { - "object": "database0", - "name": "runCommand", - "arguments": { - "command": { - "find": "test" - }, - "commandName": "find" - } - } - ], - "expectLogMessages": [ - { - "client": "client", - "messages": [ - { - "level": "debug", - "component": "connection", - "data": { - "message": "Connection pool created", - "serverHost": { - "$$type": "string" - }, - "serverPort": { - "$$type": [ - "int", - "long" - ] - } - } - }, - { - "level": "debug", - "component": "connection", - "data": { - "message": "Connection pool ready", - "serverHost": { - "$$type": "string" - }, - "serverPort": { - "$$type": [ - "int", - "long" - ] - } - } - }, - { - "level": "debug", - "component": "connection", - "data": { - "message": "Connection checkout started", - "serverHost": { - "$$type": "string" - }, - "serverPort": { - "$$type": [ - "int", - "long" - ] - } - } - }, - { - "level": "debug", - "component": "connection", - "data": { - "message": "Connection created", - "driverConnectionId": { - "$$type": [ - "int", - "long" - ] - }, - "serverHost": { - "$$type": "string" - }, - "serverPort": { - "$$type": [ - "int", - "long" - ] - } - } }, { "level": "debug", @@ -702,32 +488,6 @@ } } }, - { - "level": "debug", - "component": "connection", - "data": { - "message": "Connection pool backoff", - "serverHost": { - "$$type": "string" - }, - "durationMS": { - "$$type": "int" - }, - "attempt": { - "$$type": "int" - }, - "serverPort": { - "$$type": [ - "int", - "long" - ] - }, - "reason": "Connection pool is in backoff", - "error": { - "$$exists": true - } - } - }, { "level": "debug", "component": "connection", @@ -754,128 +514,10 @@ ] } } - }, - { - "level": "debug", - "component": "connection", - "data": { - "message": "Connection checkout started", - "serverHost": { - "$$type": "string" - }, - "serverPort": { - "$$type": [ - "int", - "long" - ] - } - } - }, - { - "level": "debug", - "component": "connection", - "data": { - "message": "Connection created", - "driverConnectionId": { - "$$type": [ - "int", - "long" - ] - }, - "serverHost": { - "$$type": "string" - }, - "serverPort": { - "$$type": [ - "int", - "long" - ] - } - } - }, - { - "level": "debug", - "component": "connection", - "data": { - "message": "Connection ready", - "driverConnectionId": { - "$$type": [ - "int", - "long" - ] - }, - "serverHost": { - "$$type": "string" - }, - "serverPort": { - "$$type": [ - "int", - "long" - ] - }, - "durationMS": { - "$$type": [ - "double", - "int", - "long" - ] - } - } - }, - { - "level": "debug", - "component": "connection", - "data": { - "message": "Connection checked out", - "driverConnectionId": { - "$$type": [ - "int", - "long" - ] - }, - "serverHost": { - "$$type": "string" - }, - "serverPort": { - "$$type": [ - "int", - "long" - ] - }, - "durationMS": { - "$$type": [ - "double", - "int", - "long" - ] - } - } - }, - { - "level": "debug", - "component": "connection", - "data": { - "message": "Connection checked in", - "driverConnectionId": { - "$$type": [ - "int", - "long" - ] - }, - "serverHost": { - "$$type": "string" - }, - "serverPort": { - "$$type": [ - "int", - "long" - ] - } - } } ] } ] } ] -} +} \ No newline at end of file diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml index cb6e80d36e..435b4b8fc8 100644 --- a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml +++ b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.yml @@ -1,545 +1,224 @@ -description: connection-logging +description: "connection-logging" + schemaVersion: "1.13" + runOnRequirements: - topologies: - - single + - single # The number of log messages is different for each topology since there is a connection pool per host. + createEntities: - client: - id: failPointClient + id: &failPointClient failPointClient + tests: - - description: Create a client, run a command, and close the client + - description: "Create a client, run a command, and close the client" operations: - name: createEntities object: testRunner arguments: entities: - client: - id: client + id: &client client observeLogMessages: connection: debug - name: listDatabases - object: client + object: *client arguments: filter: {} - name: close - object: client + object: *client expectLogMessages: - - client: client + - client: *client messages: - level: debug component: connection data: - message: Connection pool created - serverHost: - $$type: string - serverPort: - $$type: - - int - - long + message: "Connection pool created" + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + - level: debug component: connection data: - message: Connection pool ready - serverHost: - $$type: string - serverPort: - $$type: - - int - - long + message: "Connection pool ready" + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + - level: debug component: connection data: - message: Connection checkout started - serverHost: - $$type: string - serverPort: - $$type: - - int - - long + message: "Connection checkout started" + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + - level: debug component: connection data: - message: Connection created - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long + message: "Connection created" + driverConnectionId: { $$type: [int, long] } + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + - level: debug component: connection data: - message: Connection ready - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - durationMS: - $$type: - - double - - int - - long + message: "Connection ready" + driverConnectionId: { $$type: [int, long] } + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + durationMS: { $$type: [double, int, long] } + - level: debug component: connection data: - message: Connection checked out - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - durationMS: - $$type: - - double - - int - - long + message: "Connection checked out" + driverConnectionId: { $$type: [int, long] } + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + durationMS: { $$type: [double, int, long] } + - level: debug component: connection data: - message: Connection checked in - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long + message: "Connection checked in" + driverConnectionId: { $$type: [int, long] } + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + + # The next three expected logs are for ending a session. - level: debug component: connection data: - message: Connection checkout started - serverHost: - $$type: string - serverPort: - $$type: - - int - - long + message: "Connection checkout started" + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + - level: debug component: connection data: - message: Connection checked out - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - durationMS: - $$type: - - double - - int - - long + message: "Connection checked out" + driverConnectionId: { $$type: [int, long] } + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + durationMS: { $$type: [double, int, long] } + - level: debug component: connection data: - message: Connection checked in - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long + message: "Connection checked in" + driverConnectionId: { $$type: [int, long] } + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + - level: debug component: connection data: - message: Connection closed - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - reason: Connection pool was closed + message: "Connection closed" + driverConnectionId: { $$type: [int, long] } + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + reason: "Connection pool was closed" + - level: debug component: connection data: - message: Connection pool closed - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - - description: Connection checkout fails due to error establishing connection + message: "Connection pool closed" + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + + # This test exists to provide coverage of checkout failed and pool cleared events. + - description: "Connection checkout fails due to error establishing connection" runOnRequirements: - auth: true - minServerVersion: "4.0" + minServerVersion: "4.0" # failCommand was added to mongod in 4.0 operations: - name: createEntities object: testRunner arguments: entities: - client: - id: client + id: &client client uriOptions: retryReads: false - appname: clientAppName + appname: &clientAppName clientAppName + # use a high heartbeatFrequencyMS to avoid a successful monitor check marking the pool as + # ready (and emitting another event) during the course of test execution. heartbeatFrequencyMS: 10000 observeLogMessages: connection: debug - name: failPoint object: testRunner arguments: - client: failPointClient + client: *failPointClient failPoint: configureFailPoint: failCommand - mode: - times: 1 + mode: { times: 1 } data: - failCommands: - - saslContinue - errorCode: 18 - appName: clientAppName + failCommands: ["saslContinue"] + closeConnection: true + appName: *clientAppName - name: listDatabases - object: client + object: *client arguments: filter: {} expectError: - isError: true + isClientError: true + expectLogMessages: - - client: client + - client: *client messages: - level: debug component: connection data: - message: Connection pool created - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - - level: debug - component: connection - data: - message: Connection pool ready - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - - level: debug - component: connection - data: - message: Connection checkout started - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - - level: debug - component: connection - data: - message: Connection created - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - - level: debug - component: connection - data: - message: Connection closed - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - reason: An error occurred while using the connection - error: - $$exists: true - - level: debug - component: connection - data: - message: Connection checkout failed - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - reason: An error occurred while trying to establish a new connection - error: - $$exists: true - durationMS: - $$type: - - double - - int - - long - - level: debug - component: connection - data: - message: Connection pool cleared - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - - description: Connection enters backoff on closed connection - runOnRequirements: - - minServerVersion: "4.4" - poolBackoff: true - operations: - - name: createEntities - object: testRunner - arguments: - entities: - - client: - id: client - uriOptions: - retryReads: true - appname: clientAppName - heartbeatFrequencyMS: 5000 - observeEvents: - - serverHeartbeatSucceededEvent - observeLogMessages: - connection: debug - - database: - id: database0 - client: client - databaseName: ci-tests - - name: waitForEvent - object: testRunner - arguments: - client: client - event: - serverHeartbeatSucceededEvent: {} - count: 1 - - name: failPoint - object: testRunner - arguments: - client: failPointClient - failPoint: - configureFailPoint: failCommand - mode: - times: 1 - data: - failCommands: - - isMaster - - hello - closeConnection: true - appName: clientAppName - - object: database0 - name: runCommand - arguments: - command: - find: test - commandName: find - expectLogMessages: - - client: client - messages: - - level: debug - component: connection - data: - message: Connection pool created - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - - level: debug - component: connection - data: - message: Connection pool ready - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - - level: debug - component: connection - data: - message: Connection checkout started - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - - level: debug - component: connection - data: - message: Connection created - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - - level: debug - component: connection - data: - message: Connection closed - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - reason: An error occurred while using the connection - error: - $$exists: true - - level: debug - component: connection - data: - message: Connection pool backoff - serverHost: - $$type: string - durationMS: - $$type: int - attempt: - $$type: int - serverPort: - $$type: - - int - - long - reason: Connection pool is in backoff - error: - $$exists: true + message: "Connection pool created" + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + - level: debug component: connection data: - message: Connection checkout failed - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - reason: An error occurred while trying to establish a new connection - error: - $$exists: true - durationMS: - $$type: - - double - - int - - long + message: "Connection pool ready" + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + - level: debug component: connection data: - message: Connection checkout started - serverHost: - $$type: string - serverPort: - $$type: - - int - - long + message: "Connection checkout started" + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + - level: debug component: connection data: - message: Connection created - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long + message: "Connection created" + driverConnectionId: { $$type: [int, long] } + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + - level: debug component: connection data: - message: Connection ready - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - durationMS: - $$type: - - double - - int - - long + message: "Connection pool cleared" + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + - level: debug component: connection data: - message: Connection checked out - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long - durationMS: - $$type: - - double - - int - - long + message: "Connection closed" + driverConnectionId: { $$type: [int, long] } + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + reason: "An error occurred while using the connection" + error: { $$exists: true } + - level: debug component: connection data: - message: Connection checked in - driverConnectionId: - $$type: - - int - - long - serverHost: - $$type: string - serverPort: - $$type: - - int - - long + message: "Connection checkout failed" + serverHost: { $$type: string } + serverPort: { $$type: [int, long] } + reason: "An error occurred while trying to establish a new connection" + error: { $$exists: true } + durationMS: { $$type: [double, int, long] } \ No newline at end of file From 7309f40105ec042f1d8724adacbf07ab7f44c9b0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 3 Nov 2025 08:41:33 -0600 Subject: [PATCH 22/23] add connection-logging-pool-backoff tests --- .../connection-logging-pool-backoff.json | 379 ++++++++++++++++++ .../connection-logging-pool-backoff.yml | 235 +++++++++++ 2 files changed, 614 insertions(+) create mode 100644 source/connection-monitoring-and-pooling/tests/logging/connection-logging-pool-backoff.json create mode 100644 source/connection-monitoring-and-pooling/tests/logging/connection-logging-pool-backoff.yml diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging-pool-backoff.json b/source/connection-monitoring-and-pooling/tests/logging/connection-logging-pool-backoff.json new file mode 100644 index 0000000000..6dcad2824d --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/logging/connection-logging-pool-backoff.json @@ -0,0 +1,379 @@ +{ + "description": "connection-logging", + "schemaVersion": "1.28", + "runOnRequirements": [ + { + "topologies": [ + "single" + ], + "poolBackoff": true + } + ], + "createEntities": [ + { + "client": { + "id": "failPointClient" + } + } + ], + "tests": [ + { + "description": "Connection enters backoff on closed connection", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "poolBackoff": true + } + ], + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "uriOptions": { + "retryReads": true, + "appname": "clientAppName", + "heartbeatFrequencyMS": 5000 + }, + "observeEvents": [ + "serverHeartbeatSucceededEvent" + ], + "observeLogMessages": { + "connection": "debug" + } + } + }, + { + "database": { + "id": "database0", + "client": "client", + "databaseName": "ci-tests" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverHeartbeatSucceededEvent": {} + }, + "count": 1 + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "isMaster", + "hello" + ], + "closeConnection": true, + "appName": "clientAppName" + } + } + } + }, + { + "object": "database0", + "name": "runCommand", + "arguments": { + "command": { + "find": "test" + }, + "commandName": "find" + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection pool created", + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection pool ready", + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection checkout started", + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection created", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection closed", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + }, + "reason": "An error occurred while using the connection", + "error": { + "$$exists": true + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection pool backoff", + "serverHost": { + "$$type": "string" + }, + "durationMS": { + "$$type": "int" + }, + "attempt": { + "$$type": "int" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + }, + "reason": "Connection pool is in backoff", + "error": { + "$$exists": true + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection checkout failed", + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + }, + "reason": "An error occurred while trying to establish a new connection", + "error": { + "$$exists": true + }, + "durationMS": { + "$$type": [ + "double", + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection checkout started", + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection created", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection ready", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + }, + "durationMS": { + "$$type": [ + "double", + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection checked out", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + }, + "durationMS": { + "$$type": [ + "double", + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "connection", + "data": { + "message": "Connection checked in", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + } + ] + } + ] + } + ] +} diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging-pool-backoff.yml b/source/connection-monitoring-and-pooling/tests/logging/connection-logging-pool-backoff.yml new file mode 100644 index 0000000000..a966b8c0c7 --- /dev/null +++ b/source/connection-monitoring-and-pooling/tests/logging/connection-logging-pool-backoff.yml @@ -0,0 +1,235 @@ +description: connection-logging +schemaVersion: "1.28" +runOnRequirements: + - topologies: + - single + poolBackoff: true +createEntities: + - client: + id: failPointClient +tests: + - description: Connection enters backoff on closed connection + runOnRequirements: + - minServerVersion: "4.4" + poolBackoff: true + operations: + - name: createEntities + object: testRunner + arguments: + entities: + - client: + id: client + uriOptions: + retryReads: true + appname: clientAppName + heartbeatFrequencyMS: 5000 + observeEvents: + - serverHeartbeatSucceededEvent + observeLogMessages: + connection: debug + - database: + id: database0 + client: client + databaseName: ci-tests + - name: waitForEvent + object: testRunner + arguments: + client: client + event: + serverHeartbeatSucceededEvent: {} + count: 1 + - name: failPoint + object: testRunner + arguments: + client: failPointClient + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - isMaster + - hello + closeConnection: true + appName: clientAppName + - object: database0 + name: runCommand + arguments: + command: + find: test + commandName: find + expectLogMessages: + - client: client + messages: + - level: debug + component: connection + data: + message: Connection pool created + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection pool ready + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection checkout started + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection created + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection closed + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + reason: An error occurred while using the connection + error: + $$exists: true + - level: debug + component: connection + data: + message: Connection pool backoff + serverHost: + $$type: string + durationMS: + $$type: int + attempt: + $$type: int + serverPort: + $$type: + - int + - long + reason: Connection pool is in backoff + error: + $$exists: true + - level: debug + component: connection + data: + message: Connection checkout failed + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + reason: An error occurred while trying to establish a new connection + error: + $$exists: true + durationMS: + $$type: + - double + - int + - long + - level: debug + component: connection + data: + message: Connection checkout started + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection created + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + - level: debug + component: connection + data: + message: Connection ready + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + durationMS: + $$type: + - double + - int + - long + - level: debug + component: connection + data: + message: Connection checked out + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long + durationMS: + $$type: + - double + - int + - long + - level: debug + component: connection + data: + message: Connection checked in + driverConnectionId: + $$type: + - int + - long + serverHost: + $$type: string + serverPort: + $$type: + - int + - long From b349f0bd7e8ca523d81c8e87c868c2cd3d49b323 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 3 Nov 2025 09:42:37 -0600 Subject: [PATCH 23/23] update json --- .../tests/logging/connection-logging.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json index 3482f59777..72103b3cab 100644 --- a/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json +++ b/source/connection-monitoring-and-pooling/tests/logging/connection-logging.json @@ -520,4 +520,4 @@ ] } ] -} \ No newline at end of file +}