Integration tests for 6.x and refactored common bits

Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs

@@ -8,10 +8,6 @@ namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap
 	public abstract class ClientTestClusterBase : XunitClusterBase<ClientTestClusterConfiguration> 
 	{
 		protected ClientTestClusterBase(ClientTestClusterConfiguration configuration) : base(configuration) { }
-
-		public IElasticClient Client => this.GetOrAddClient(s => ConnectionSettings(s));
-
-		protected virtual ConnectionSettings ConnectionSettings(ConnectionSettings s) => s;
 	}
 
 	public class ClientTestClusterConfiguration : XunitClusterConfiguration

Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ElasticsearchSinkOptionsFactory.cs

@@ -0,0 +1,36 @@
+using System;
+using Elasticsearch.Net;
+
+namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap
+{
+    public static class ElasticsearchSinkOptionsFactory
+    {
+        public static ElasticsearchSinkOptions Create(string indexPrefix, string templateName, Action<ElasticsearchSinkOptions> alterOptions = null)
+        {
+            // make sure we run through `ipv4.fiddler` if `fiddler` or `mitmproxy` is running
+            // NOTE with the latter you need to add `ipv4.fiddler` as an alias to 127.0.0.1 in your HOSTS file manually
+            var pool = new SingleNodeConnectionPool(new Uri($"http://{ProxyDetection.LocalOrProxyHost}:9200"));
+            var options = new ElasticsearchSinkOptions(pool)
+            {
+                IndexFormat = indexPrefix + "{0:yyyy.MM.dd}",
+                TemplateName = templateName,
+            };
+
+            alterOptions?.Invoke(options);
+            // here we make sure we set a proxy on the elasticsearch connection 
+            // when we detect `mitmproxy` running. This is a cli tool similar to fiddler
+            // on *nix systems which aids debugging what goes over the wire
+            var provided = options.ModifyConnectionSettings;
+            options.ModifyConnectionSettings = configuration =>
+            {
+                if (ProxyDetection.RunningMitmProxy)
+                    configuration.Proxy(ProxyDetection.MitmProxyAddress, null, (string) null);
+                configuration = provided?.Invoke(configuration) ?? configuration;
+                return configuration;
+            };
+
+            return options;
+        }
+
+    }
+}

Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ProxyDetection.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Diagnostics;
+using System.Linq;
+
+namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap
+{
+    public static class ProxyDetection
+    {
+        public static readonly bool RunningMitmProxy = Process.GetProcessesByName("mitmproxy").Any();
+        private static readonly bool RunningFiddler = Process.GetProcessesByName("fiddler").Any();
+        public static string LocalOrProxyHost => RunningFiddler || RunningMitmProxy ? "ipv4.fiddler" : "localhost";
+
+        public static readonly Uri MitmProxyAddress = new Uri("http://localhost:8080");
+    }
+}

Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs

@@ -0,0 +1,19 @@
+using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap;
+
+namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6.Bootstrap
+{
+	public class Elasticsearch6XCluster : ClientTestClusterBase
+	{
+		public Elasticsearch6XCluster() : base(CreateConfiguration()) { }
+
+		private static ClientTestClusterConfiguration CreateConfiguration()
+		{
+			return new ClientTestClusterConfiguration("6.6.0")
+			{
+				MaxConcurrency = 1
+			};
+		}
+
+		protected override void SeedCluster() { }
+	}
+}

Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Security.Cryptography.X509Certificates;
+using Elastic.Managed.Ephemeral;
+using Elastic.Xunit;
+using Elastic.Xunit.XunitPlumbing;
+using Elasticsearch.Net6;
+using Nest6;
+using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap;
+
+namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6.Bootstrap
+{
+	public abstract class Elasticsearch6XTestBase : IClusterFixture<Elasticsearch6XCluster> 
+	{
+        protected Elasticsearch6XTestBase(Elasticsearch6XCluster cluster) => Cluster = cluster;
+
+        private Elasticsearch6XCluster Cluster { get; }
+
+        protected IElasticClient Client => this.CreateClient();
+
+		protected virtual ConnectionSettings SetClusterSettings(ConnectionSettings s) => s;
+
+        private IElasticClient CreateClient() =>
+            Cluster.GetOrAddClient(c =>
+            {
+                var clusterNodes = c.NodesUris(ProxyDetection.LocalOrProxyHost);
+                var settings = new ConnectionSettings(new StaticConnectionPool(clusterNodes));
+                if (ProxyDetection.RunningMitmProxy)
+                    settings = settings.Proxy(new Uri("http://localhost:8080"), null, null);
+                settings = SetClusterSettings(settings);
+
+                var current = (IConnectionConfigurationValues)settings;
+                var notAlreadyAuthenticated = current.BasicAuthenticationCredentials == null && current.ClientCertificates == null;
+                var noCertValidation = current.ServerCertificateValidationCallback == null;
+
+                var config = Cluster.ClusterConfiguration;
+                if (config.EnableSecurity && notAlreadyAuthenticated)
+                    settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password);
+                if (config.EnableSsl && noCertValidation)
+                {
+                    //todo use CA callback instead of allow all
+                    // ReSharper disable once UnusedVariable
+                    var ca = new X509Certificate2(config.FileSystem.CaCertificate);
+                    settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll);
+                }
+                var client = new ElasticClient(settings);
+                return client;
+            });
+    }
+
+}

Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs

@@ -0,0 +1,71 @@
+using System.Linq;
+using Elastic.Xunit.XunitPlumbing;
+using FluentAssertions;
+using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap;
+using Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6.Bootstrap;
+using Xunit;
+
+namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6
+{
+    public class Elasticsearch6X : Elasticsearch6XTestBase, IClassFixture<Elasticsearch6X.SetupSerilog>
+    {
+        private readonly SetupSerilog _setup;
+
+        public Elasticsearch6X(Elasticsearch6XCluster cluster, SetupSerilog setup) : base(cluster) => _setup = setup;
+
+
+        [I] public void AssertTemplate()
+        {
+            var templateResponse = Client.GetIndexTemplate(t=>t.Name(SetupSerilog.TemplateName));
+            templateResponse.TemplateMappings.Should().NotBeEmpty();
+            templateResponse.TemplateMappings.Keys.Should().Contain(SetupSerilog.TemplateName);
+
+            var template = templateResponse.TemplateMappings[SetupSerilog.TemplateName];
+
+            template.IndexPatterns.Should().Contain(pattern => pattern.StartsWith(SetupSerilog.IndexPrefix));
+        }
+
+        [I] public void AssertLogs()
+        {
+            var refreshed = Client.Refresh(SetupSerilog.IndexPrefix + "*");
+
+            var search = Client.Search<object>(s => s
+                .Index(SetupSerilog.IndexPrefix + "*")
+                .Type(ElasticsearchSinkOptions.DefaultTypeName)
+            );
+
+            // Informational should be filtered
+            search.Documents.Count().Should().Be(4);
+        }
+
+        // ReSharper disable once ClassNeverInstantiated.Global
+        public class SetupSerilog
+        {
+            public const string IndexPrefix = "logs-6x-";
+            public const string TemplateName = "serilog-logs-6x";
+
+            public SetupSerilog()
+            {
+                var loggerConfig = new LoggerConfiguration()
+                    .MinimumLevel.Information()
+                    .WriteTo.ColoredConsole()
+                    .WriteTo.Elasticsearch(
+                        ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o =>
+                        {
+                            o.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6;
+                            o.AutoRegisterTemplate = true;
+                        })
+                    );
+                var logger = loggerConfig.CreateLogger();
+
+                logger.Information("Hello Information");
+                logger.Debug("Hello Debug");
+                logger.Warning("Hello Warning");
+                logger.Error("Hello Error");
+                logger.Fatal("Hello Fatal");
+
+                logger.Dispose();
+            }
+        }
+    }
+}

Serilog.Sinks.Elasticsearch.IntegrationTests/Clusters/Elasticsearch7xCluster.cs → Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs

@@ -1,11 +1,7 @@
 using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap;
 
-namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Clusters
+namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7.Bootstrap
 {
-	/// <summary>
-	/// Use this cluster for APIs that do writes. If they are however intrusive or long running consider IntrusiveOperationCluster
-	/// instead.
-	/// </summary>
 	public class Elasticsearch7XCluster : ClientTestClusterBase
 	{
 		public Elasticsearch7XCluster() : base(CreateConfiguration()) { }

Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs

@@ -0,0 +1,51 @@
+using System;
+using System.Security.Cryptography.X509Certificates;
+using Elastic.Managed.Ephemeral;
+using Elastic.Xunit;
+using Elastic.Xunit.XunitPlumbing;
+using Elasticsearch.Net;
+using Nest;
+using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap;
+using Xunit;
+
+namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7.Bootstrap
+{
+	public abstract class Elasticsearch7XTestBase : IClusterFixture<Elasticsearch7XCluster> 
+	{
+        protected Elasticsearch7XTestBase(Elasticsearch7XCluster cluster) => Cluster = cluster;
+
+        private Elasticsearch7XCluster Cluster { get; }
+
+        protected IElasticClient Client => this.CreateClient();
+
+		protected virtual ConnectionSettings SetClusterSettings(ConnectionSettings s) => s;
+
+        private IElasticClient CreateClient() =>
+            Cluster.GetOrAddClient(c =>
+            {
+                var clusterNodes = c.NodesUris(ProxyDetection.LocalOrProxyHost);
+                var settings = new ConnectionSettings(new StaticConnectionPool(clusterNodes));
+                if (ProxyDetection.RunningMitmProxy)
+                    settings = settings.Proxy(new Uri("http://localhost:8080"), null, (string)null);
+                settings = SetClusterSettings(settings);
+
+                var current = (IConnectionConfigurationValues)settings;
+                var notAlreadyAuthenticated = current.BasicAuthenticationCredentials == null && current.ClientCertificates == null;
+                var noCertValidation = current.ServerCertificateValidationCallback == null;
+
+                var config = Cluster.ClusterConfiguration;
+                if (config.EnableSecurity && notAlreadyAuthenticated)
+                    settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password);
+                if (config.EnableSsl && noCertValidation)
+                {
+                    //todo use CA callback instead of allow all
+                    // ReSharper disable once UnusedVariable
+                    var ca = new X509Certificate2(config.FileSystem.CaCertificate);
+                    settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll);
+                }
+                var client = new ElasticClient(settings);
+                return client;
+            });
+    }
+
+}

Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs

@@ -0,0 +1,67 @@
+using System.Linq;
+using Elastic.Xunit.XunitPlumbing;
+using FluentAssertions;
+using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap;
+using Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7.Bootstrap;
+using Xunit;
+
+namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7
+{
+    public class Elasticsearch7X : Elasticsearch7XTestBase, IClassFixture<Elasticsearch7X.SetupSerilog>
+    {
+        private readonly SetupSerilog _setup;
+
+        public Elasticsearch7X(Elasticsearch7XCluster cluster, SetupSerilog setup) : base(cluster) => _setup = setup;
+
+        [I] public void AssertTemplate()
+        {
+            var templateResponse = Client.Indices.GetTemplate(SetupSerilog.TemplateName);
+            templateResponse.TemplateMappings.Should().NotBeEmpty();
+            templateResponse.TemplateMappings.Keys.Should().Contain(SetupSerilog.TemplateName);
+
+            var template = templateResponse.TemplateMappings[SetupSerilog.TemplateName];
+
+            template.IndexPatterns.Should().Contain(pattern => pattern.StartsWith(SetupSerilog.IndexPrefix));
+        }
+
+        [I] public void AssertLogs()
+        {
+            var refreshed = Client.Indices.Refresh(SetupSerilog.IndexPrefix + "*");
+
+            var search = Client.Search<object>(s => s.Index(SetupSerilog.IndexPrefix + "*"));
+
+            // Informational should be filtered
+            search.Documents.Count().Should().Be(4);
+        }
+
+        // ReSharper disable once ClassNeverInstantiated.Global
+        public class SetupSerilog
+        {
+            public const string IndexPrefix = "logs-7x-";
+            public const string TemplateName = "serilog-logs-7x";
+
+            public SetupSerilog()
+            {
+                var loggerConfig = new LoggerConfiguration()
+                    .MinimumLevel.Information()
+                    .WriteTo.ColoredConsole()
+                    .WriteTo.Elasticsearch(
+                        ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o =>
+                        {
+                            o.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7;
+                            o.AutoRegisterTemplate = true;
+                        })
+                    );
+                using (var logger = loggerConfig.CreateLogger())
+                {
+                    logger.Information("Hello Information");
+                    logger.Debug("Hello Debug");
+                    logger.Warning("Hello Warning");
+                    logger.Error("Hello Error");
+                    logger.Fatal("Hello Fatal");
+                }
+            }
+        }
+
+    }
+}