diff --git a/Snowflake.Data.Tests/IntegrationTests/CertificateRevocationIT.cs b/Snowflake.Data.Tests/IntegrationTests/CertificateRevocationIT.cs index f5f7cc08f..e7fd092df 100644 --- a/Snowflake.Data.Tests/IntegrationTests/CertificateRevocationIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/CertificateRevocationIT.cs @@ -26,6 +26,7 @@ public void TestCertificate() false, false, 3, + 20, true, CertRevocationCheckMode.Enabled.ToString(), true, diff --git a/Snowflake.Data.Tests/UnitTests/HttpUtilTest.cs b/Snowflake.Data.Tests/UnitTests/HttpUtilTest.cs index 5fba7cfff..3f7e0e320 100644 --- a/Snowflake.Data.Tests/UnitTests/HttpUtilTest.cs +++ b/Snowflake.Data.Tests/UnitTests/HttpUtilTest.cs @@ -35,7 +35,7 @@ public async Task TestNonRetryableHttpExceptionThrowsError() .ThrowsAsync(new HttpRequestException("", new AuthenticationException())); var httpClient = HttpUtil.Instance.GetHttpClient( - new HttpClientConfig("fakeHost", "fakePort", "user", "password", "fakeProxyList", false, false, 7, certRevocationCheckMode: "ENABLED"), + new HttpClientConfig("fakeHost", "fakePort", "user", "password", "fakeProxyList", false, false, 7, 20, certRevocationCheckMode: "ENABLED"), handler.Object); try @@ -150,7 +150,8 @@ public void TestCreateHttpClientHandlerWithProxy() "localhost", false, false, - 7 + 7, + 20 ); // act @@ -173,6 +174,7 @@ public void TestCreateHttpClientHandlerWithoutProxy() null, false, false, + 20, 0 ); diff --git a/Snowflake.Data.Tests/UnitTests/Revocation/CertificateRevocationVerifierTest.cs b/Snowflake.Data.Tests/UnitTests/Revocation/CertificateRevocationVerifierTest.cs index 4064cc034..f5749c6ea 100644 --- a/Snowflake.Data.Tests/UnitTests/Revocation/CertificateRevocationVerifierTest.cs +++ b/Snowflake.Data.Tests/UnitTests/Revocation/CertificateRevocationVerifierTest.cs @@ -277,6 +277,7 @@ private HttpClientConfig GetHttpConfig(CertRevocationCheckMode checkMode = CertR false, false, 3, + 20, true, checkMode.ToString(), false, diff --git a/Snowflake.Data.Tests/UnitTests/Session/SFHttpClientPropertiesTest.cs b/Snowflake.Data.Tests/UnitTests/Session/SFHttpClientPropertiesTest.cs index 98f3d5ab9..eed04e4bf 100644 --- a/Snowflake.Data.Tests/UnitTests/Session/SFHttpClientPropertiesTest.cs +++ b/Snowflake.Data.Tests/UnitTests/Session/SFHttpClientPropertiesTest.cs @@ -53,6 +53,86 @@ public void TestConvertToMapOnly2Properties( Assert.AreEqual(clientStoreTemporaryCredential, parameterMap[SFSessionParameter.CLIENT_STORE_TEMPORARY_CREDENTIAL]); } + [Test] + [TestCase(1)] + [TestCase(10)] + [TestCase(100)] + public void TestSettingConnectionLimitProperty(int expectedConnectionLimit) + { + // arrange + var connectionString = $"ACCOUNT=account;USER=test;PASSWORD=test;SERVICE_POINT_CONNECTION_LIMIT={expectedConnectionLimit}"; + var properties = SFSessionProperties.ParseConnectionString(connectionString, new SessionPropertiesContext()); + + // act + var extractedProperties = SFSessionHttpClientProperties.ExtractAndValidate(properties); + + // assert + Assert.AreEqual(expectedConnectionLimit, extractedProperties._servicePointConnectionLimit); + } + + [Test] + public void TestSettingConnectionLimitPropertyToLessThan1() + { + // arrange + var connectionString = $"ACCOUNT=account;USER=test;PASSWORD=test;SERVICE_POINT_CONNECTION_LIMIT={0}"; + var properties = SFSessionProperties.ParseConnectionString(connectionString, new SessionPropertiesContext()); + + // act + var extractedProperties = SFSessionHttpClientProperties.ExtractAndValidate(properties); + + // assert + Assert.AreEqual(SFSessionHttpClientProperties.DefaultConnectionLimit, extractedProperties._servicePointConnectionLimit); + } + + [Test] + public void TestSettingConnectionLimitPropertyToGreaterThanMaxConnectionLimit() + { + // arrange + var connectionString = $"ACCOUNT=account;USER=test;PASSWORD=test;SERVICE_POINT_CONNECTION_LIMIT={SFSessionHttpClientProperties.MaxConnectionLimit + 1}"; + var properties = SFSessionProperties.ParseConnectionString(connectionString, new SessionPropertiesContext()); + + // act + var extractedProperties = SFSessionHttpClientProperties.ExtractAndValidate(properties); + + // assert + Assert.AreEqual(SFSessionHttpClientProperties.DefaultConnectionLimit, extractedProperties._servicePointConnectionLimit); + } + + [Test] + public void TestSettingConnectionLimitPropertyToNoValue() + { + // arrange + var connectionString = $"ACCOUNT=account;USER=test;PASSWORD=test;SERVICE_POINT_CONNECTION_LIMIT="; + var properties = SFSessionProperties.ParseConnectionString(connectionString, new SessionPropertiesContext()); + + // act + var extractedProperties = SFSessionHttpClientProperties.ExtractAndValidate(properties); + + // assert + Assert.AreEqual(SFSessionHttpClientProperties.DefaultConnectionLimit, extractedProperties._servicePointConnectionLimit); + } + + [Test] + [TestCase("abc")] + [TestCase("1.5")] + [TestCase("true")] + [TestCase("-2.3")] + [TestCase("null")] + public void TestThrowsExceptionWhenSettingConnectionLimitPropertyToNonIntegerValue(string nonIntegerValue) + { + // arrange + var parameterName = "SERVICE_POINT_CONNECTION_LIMIT"; + var expectedErrorMessage = $"Error: Invalid parameter value for {parameterName}"; + var connectionString = $"ACCOUNT=account;USER=test;PASSWORD=test;{parameterName}={nonIntegerValue}"; + + // act + var thrown = Assert.Throws(() => SFSessionProperties.ParseConnectionString(connectionString, new SessionPropertiesContext())); + + // assert + Assert.AreEqual(SFError.INVALID_CONNECTION_PARAMETER_VALUE.GetAttribute().errorCode, thrown.ErrorCode); + Assert.IsTrue(thrown.Message.Contains(expectedErrorMessage)); + } + [Test] public void TestBuildHttpClientConfig() { diff --git a/Snowflake.Data/Core/HttpUtil.cs b/Snowflake.Data/Core/HttpUtil.cs old mode 100755 new mode 100644 index 8cfe17baa..9cb354576 --- a/Snowflake.Data/Core/HttpUtil.cs +++ b/Snowflake.Data/Core/HttpUtil.cs @@ -27,6 +27,7 @@ public HttpClientConfig( bool disableRetry, bool forceRetryOn404, int maxHttpRetries, + int connectionLimit, bool includeRetryReason = true, string certRevocationCheckMode = "DISABLED", bool enableCRLDiskCaching = true, @@ -46,6 +47,7 @@ public HttpClientConfig( ForceRetryOn404 = forceRetryOn404; MaxHttpRetries = maxHttpRetries; IncludeRetryReason = includeRetryReason; + ConnectionLimit = connectionLimit; CertRevocationCheckMode = (CertRevocationCheckMode)Enum.Parse(typeof(CertRevocationCheckMode), certRevocationCheckMode, true); EnableCRLDiskCaching = enableCRLDiskCaching; EnableCRLInMemoryCaching = enableCRLInMemoryCaching; @@ -65,6 +67,7 @@ public HttpClientConfig( forceRetryOn404.ToString(), maxHttpRetries.ToString(), includeRetryReason.ToString(), + connectionLimit.ToString(), certRevocationCheckMode, enableCRLDiskCaching.ToString(), enableCRLInMemoryCaching.ToString(), @@ -84,6 +87,7 @@ public HttpClientConfig( public readonly bool ForceRetryOn404; public readonly int MaxHttpRetries; public readonly bool IncludeRetryReason; + public readonly int ConnectionLimit; internal readonly CertRevocationCheckMode CertRevocationCheckMode; internal readonly bool EnableCRLDiskCaching; internal readonly bool EnableCRLInMemoryCaching; @@ -161,7 +165,7 @@ private HttpClient RegisterNewHttpClientIfNecessary(HttpClientConfig config, Del internal HttpClient CreateNewHttpClient(HttpClientConfig config, DelegatingHandler customHandler = null) => new HttpClient( - new RetryHandler(SetupCustomHttpHandler(config, customHandler), config.DisableRetry, config.ForceRetryOn404, config.MaxHttpRetries, config.IncludeRetryReason)) + new RetryHandler(SetupCustomHttpHandler(config, customHandler), config.DisableRetry, config.ForceRetryOn404, config.MaxHttpRetries, config.IncludeRetryReason, config.ConnectionLimit)) { Timeout = Timeout.InfiniteTimeSpan }; @@ -434,13 +438,15 @@ private class RetryHandler : DelegatingHandler private bool forceRetryOn404; private int maxRetryCount; private bool includeRetryReason; + private int connectionLimit; - internal RetryHandler(HttpMessageHandler innerHandler, bool disableRetry, bool forceRetryOn404, int maxRetryCount, bool includeRetryReason) : base(innerHandler) + internal RetryHandler(HttpMessageHandler innerHandler, bool disableRetry, bool forceRetryOn404, int maxRetryCount, bool includeRetryReason, int connectionLimit) : base(innerHandler) { this.disableRetry = disableRetry; this.forceRetryOn404 = forceRetryOn404; this.maxRetryCount = maxRetryCount; this.includeRetryReason = includeRetryReason; + this.connectionLimit = connectionLimit; } protected override async Task SendAsync(HttpRequestMessage requestMessage, @@ -457,7 +463,7 @@ protected override async Task SendAsync(HttpRequestMessage ServicePoint p = ServicePointManager.FindServicePoint(requestMessage.RequestUri); p.Expect100Continue = false; // Saves about 100 ms per request p.UseNagleAlgorithm = false; // Saves about 200 ms per request - p.ConnectionLimit = 20; // Default value is 2, we need more connections for performing multiple parallel queries + p.ConnectionLimit = connectionLimit; // Default value is 2, we need more connections for performing multiple parallel queries TimeSpan httpTimeout = (TimeSpan)requestMessage.Properties[BaseRestRequest.HTTP_REQUEST_TIMEOUT_KEY]; TimeSpan restTimeout = (TimeSpan)requestMessage.Properties[BaseRestRequest.REST_REQUEST_TIMEOUT_KEY]; diff --git a/Snowflake.Data/Core/Session/SFSessionHttpClientProperties.cs b/Snowflake.Data/Core/Session/SFSessionHttpClientProperties.cs index b7da1fcdf..e2fcf3b45 100644 --- a/Snowflake.Data/Core/Session/SFSessionHttpClientProperties.cs +++ b/Snowflake.Data/Core/Session/SFSessionHttpClientProperties.cs @@ -21,8 +21,10 @@ internal class SFSessionHttpClientProperties public static readonly TimeSpan DefaultExpirationTimeout = TimeSpan.FromHours(1); public const bool DefaultPoolingEnabled = true; public const int DefaultMaxHttpRetries = 7; + public const int DefaultConnectionLimit = 20; public static readonly TimeSpan DefaultRetryTimeout = TimeSpan.FromSeconds(300); private static readonly SFLogger s_logger = SFLoggerFactory.GetLogger(); + internal static readonly int MaxConnectionLimit = 1000; internal bool validateDefaultParameters; internal bool clientSessionKeepAlive; @@ -41,6 +43,7 @@ internal class SFSessionHttpClientProperties private TimeSpan _expirationTimeout; private bool _poolingEnabled; internal bool _clientStoreTemporaryCredential; + internal int _servicePointConnectionLimit; internal CertRevocationCheckMode _certRevocationCheckMode; internal bool _enableCrlDiskCaching; internal bool _enableCrlInMemoryCaching; @@ -99,6 +102,7 @@ private void CheckPropertiesAreValid() ValidateHttpRetries(); ValidateMinMaxPoolSize(); ValidateWaitingForSessionIdleTimeout(); + ValidateConnectionLimit(); } catch (SnowflakeDbException) { @@ -188,6 +192,15 @@ private void ValidateWaitingForSessionIdleTimeout() } } + private void ValidateConnectionLimit() + { + if (_servicePointConnectionLimit < 1 || _servicePointConnectionLimit > MaxConnectionLimit) + { + s_logger.Warn($"Connection limit must be between 1 and {MaxConnectionLimit}. Using the default value of {DefaultConnectionLimit}"); + _servicePointConnectionLimit = DefaultConnectionLimit; + } + } + public HttpClientConfig BuildHttpClientConfig() { return new HttpClientConfig( @@ -199,6 +212,7 @@ public HttpClientConfig BuildHttpClientConfig() disableRetry, forceRetryOn404, maxHttpRetries, + _servicePointConnectionLimit, includeRetryReason, _certRevocationCheckMode.ToString(), _enableCrlDiskCaching, @@ -267,6 +281,7 @@ public SFSessionHttpClientProperties ExtractProperties(SFSessionProperties prope _poolingEnabled = extractor.ExtractBooleanWithDefaultValue(SFSessionProperty.POOLINGENABLED), _disableSamlUrlCheck = extractor.ExtractBooleanWithDefaultValue(SFSessionProperty.DISABLE_SAML_URL_CHECK), _clientStoreTemporaryCredential = Boolean.Parse(propertiesDictionary[SFSessionProperty.CLIENT_STORE_TEMPORARY_CREDENTIAL]), + _servicePointConnectionLimit = int.Parse(propertiesDictionary[SFSessionProperty.SERVICE_POINT_CONNECTION_LIMIT]), _certRevocationCheckMode = (CertRevocationCheckMode)Enum.Parse(typeof(CertRevocationCheckMode), propertiesDictionary[SFSessionProperty.CERTREVOCATIONCHECKMODE], true), _enableCrlDiskCaching = Boolean.Parse(propertiesDictionary[SFSessionProperty.ENABLECRLDISKCACHING]), _enableCrlInMemoryCaching = Boolean.Parse(propertiesDictionary[SFSessionProperty.ENABLECRLINMEMORYCACHING]), diff --git a/Snowflake.Data/Core/Session/SFSessionProperty.cs b/Snowflake.Data/Core/Session/SFSessionProperty.cs index 8d34a0695..bb98342f8 100644 --- a/Snowflake.Data/Core/Session/SFSessionProperty.cs +++ b/Snowflake.Data/Core/Session/SFSessionProperty.cs @@ -132,6 +132,8 @@ internal enum SFSessionProperty WORKLOAD_IDENTITY_ENTRA_RESOURCE, [SFSessionPropertyAttr(required = false, defaultValue = "false")] OAUTHENABLESINGLEUSEREFRESHTOKENS, + [SFSessionPropertyAttr(required = false, defaultValue = "20")] + SERVICE_POINT_CONNECTION_LIMIT, [SFSessionPropertyAttr(required = false, defaultValue = "disabled")] CERTREVOCATIONCHECKMODE, [SFSessionPropertyAttr(required = false, defaultValue = "true")] @@ -302,6 +304,7 @@ internal static SFSessionProperties ParseConnectionString(string connectionStrin ValidateAccountDomain(properties); WarnIfHttpUsed(properties); ValidateAuthenticatorFlowsProperties(properties); + ValidateServicePointConnectionLimit(properties); ValidateCrlParameters(properties); ValidateTlsParameters(properties); @@ -860,6 +863,23 @@ private static void ValidateFileTransferMaxBytesInMemoryProperty(SFSessionProper } } + private static void ValidateServicePointConnectionLimit(SFSessionProperties properties) + { + if (properties.TryGetValue(SFSessionProperty.SERVICE_POINT_CONNECTION_LIMIT, out var servicePointConnectionLimit)) + { + if (!int.TryParse(servicePointConnectionLimit, out _)) + { + var errorMessage = $"Invalid value of {SFSessionProperty.SERVICE_POINT_CONNECTION_LIMIT.ToString()} parameter"; + logger.Error(errorMessage); + throw new SnowflakeDbException( + new Exception(errorMessage), + SFError.INVALID_CONNECTION_PARAMETER_VALUE, + "", + SFSessionProperty.SERVICE_POINT_CONNECTION_LIMIT.ToString()); + } + } + } + private static bool IsRequired(SFSessionProperty sessionProperty, SFSessionProperties properties) { if (sessionProperty.Equals(SFSessionProperty.PASSWORD)) diff --git a/doc/Connecting.md b/doc/Connecting.md index 1dd986b5d..61f1fa387 100644 --- a/doc/Connecting.md +++ b/doc/Connecting.md @@ -68,6 +68,7 @@ The following table lists all valid connection properties: | CRLDOWNLOADTIMEOUT | No | When the driver's own implementation of CRL check is enabled (`enabled` or `advisory` mode) it specifies the timeout in seconds for downloading CRL files from distribution points. The default value is `10`. | | MINTLS | No | Minimum TLS version negotiated with the server when specified. Possible values are `TLS12` (default) and `TLS13`. | | MAXTLS | No | Maximum TLS version negotiated with the server when specified. Possible values are `TLS12` and `TLS13` (default). | +| SERVICE_POINT_CONNECTION_LIMIT | No | The maximum amount of connections allowed for the ServicePoint object. The default value is 20. Note: Only the connection limit from the first connection string is used. If other connection strings have different limits, they will be ignored because the HTTP client from the first connection is reused. |
**Note**: Connections should not be shared across multiple threads.