Added tests for 0 replicas (#303)

* Change default TypeName to '_doc'

* Added tests for 0 replicas

* Added more tests

* Fix force lowercase indexname, closes #306

* Update sample, switch to ES7.5

* Updated sample app

.env

@@ -1,3 +1,3 @@
-TAG=6.1.3
-ELASTIC_VERSION=6.1.3
+TAG=7.5.0
+ELASTIC_VERSION=7.5.0
 ELASTIC_PASSWORD=changeme

CHANGES.md

@@ -1,5 +1,9 @@
 ## Changelog
 
+8.1 
+ * Updated sample to use .NET 2.1 and ES 7.5
+ * Change default TypeName to '_doc' #298
+
 8.0
  * Adds Elasticsearch 7.0 support #256
  * Adds DetectElasticsearchVersion to the sink that will detect the running cluster version. Something we now use to support sending Esv6 templates to Elasticsearch 7.x and Esv7 templates to Elasticsearch 6.x which should simplify upgrades.

docker-compose.yml

@@ -16,10 +16,11 @@ services:
     mem_limit: 1g
     environment:
       - cluster.name=docker-cluster
+      - discovery.type=single-node
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-      - discovery.type=zen
-      - discovery.zen.ping.unicast.hosts=elasticsearch
+      - node.master=true
+      - node.data=true
       - http.cors.enabled=true
       - http.cors.allow-origin=*
       - http.cors.allow-headers=Content-Type, Access-Control-Allow-Headers, Authorization, X-Requested-With

sample/Serilog.Sinks.Elasticsearch.Sample/Program.cs

@@ -23,6 +23,7 @@ class Program
             .AddJsonFile("appsettings.json", true, true)
             .AddEnvironmentVariables()
             .Build();
+
         static void Main(string[] args)
         {
 
@@ -31,28 +32,26 @@ static void Main(string[] args)
             Log.Logger = new LoggerConfiguration()
                 .MinimumLevel.Debug()
                 .WriteTo.Console(theme: SystemConsoleTheme.Literate)
-                                //not persistant
-                                .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch"))) // for the docker-compose implementation
-                                {
-                                    AutoRegisterTemplate = true,
-                                    //BufferBaseFilename = "./buffer",
-                                    RegisterTemplateFailure = RegisterTemplateRecovery.IndexAnyway,
-                                    FailureCallback = e => Console.WriteLine("Unable to submit event " + e.MessageTemplate),
-                                    EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog |
-                                                       EmitEventFailureHandling.WriteToFailureSink |
-                                                       EmitEventFailureHandling.RaiseCallback,
-                                    FailureSink = new FileSink("./fail-{Date}.txt", new JsonFormatter(), null, null)
-                                })
-                    .CreateLogger();
-
-            //SetupLoggerWithSimplePersistantStorage();
-            //LoggingLevelSwitch levelSwitch = SetupLoggerWithPersistantStorage();
+                .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch"))) // for the docker-compose implementation
+                {
+                    AutoRegisterTemplate = true,
+                    OverwriteTemplate = true,
+                    DetectElasticsearchVersion = true,
+                    AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7,
+                    NumberOfReplicas = 1,
+                    NumberOfShards = 2,
+                    //BufferBaseFilename = "./buffer",
+                    RegisterTemplateFailure = RegisterTemplateRecovery.FailSink,
+                    FailureCallback = e => Console.WriteLine("Unable to submit event " + e.MessageTemplate),
+                    EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog |
+                                       EmitEventFailureHandling.WriteToFailureSink |
+                                       EmitEventFailureHandling.RaiseCallback,
+                    FailureSink = new FileSink("./fail-{Date}.txt", new JsonFormatter(), null, null)
+                })
+                .CreateLogger();
 
-            //Log.Debug("To high loglevel default is Information this will not be logged");
-            //levelSwitch.MinimumLevel = LogEventLevel.Debug;
             Log.Information("Hello, world!");
-            //Log.Information("To  big log row bigger than SingleEventSizePostingLimit ! {a}", new string('*', 5000));
-
+
             int a = 10, b = 0;
             try
             {
@@ -75,63 +74,6 @@ static void Main(string[] args)
             }
         }
 
-        private static LoggingLevelSwitch SetupLoggerWithPersistantStorage()
-        {
-            //persistant storage with all settings available for this mode
-            //please note that all limit settings here is set verry low for test, default values should usually work best!
-            var levelSwitch = new LoggingLevelSwitch();
-            Log.Logger = new LoggerConfiguration()
-                .MinimumLevel.Debug()
-                .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch")))
-                {
-                    AutoRegisterTemplate = true,
-                    BufferBaseFilename = "./buffer/logserilog",
-                    IndexFormat = "log-serilog-{0:yyyy.MM}",
-                    RegisterTemplateFailure = RegisterTemplateRecovery.FailSink,
-                    BufferCleanPayload = (failingEvent, statuscode, exception) =>
-                    {
-                        dynamic e = JObject.Parse(failingEvent);
-                        return JsonConvert.SerializeObject(new Dictionary<string, object>()
-                        {
-                            { "@timestamp",e["@timestamp"]},
-                            { "level",e.level},
-                            { "message","Error: "+e.message},
-                            { "messageTemplate",e.messageTemplate},
-                            { "failingStatusCode", statuscode},
-                            { "failingException", exception}
-                        });
-                    },
-                    OverwriteTemplate = true,
-                    NumberOfShards = 1,
-                    NumberOfReplicas = 1,
-                    GetTemplateContent = null,
-                    AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6,
-                    PipelineName = null,
-                    TypeName = "logevent",
-                    BufferIndexDecider = (logEvent, offset) => "log-serilog-" + (new Random().Next(0, 2)),
-                    BatchPostingLimit = 50,
-                    BufferLogShippingInterval = TimeSpan.FromSeconds(5),
-                    SingleEventSizePostingLimit = 1000,
-                    LevelSwitch = levelSwitch,
-                    BufferRetainedInvalidPayloadsLimitBytes = 2000,
-                    BufferFileSizeLimitBytes = 2000,
-                    BufferFileCountLimit = 2
-                })
-                .CreateLogger();
-            return levelSwitch;
-        }
-
-        private static void SetupLoggerWithSimplePersistantStorage()
-        {
-            //presistant
-            Log.Logger = new LoggerConfiguration()
-                .MinimumLevel.Debug()
-                .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch")))
-                {
-                    BufferBaseFilename = "./buffer/logserilogsimple",
-                    IndexFormat = "log-serilog-simple-{0:yyyy.MM}"
-                })
-                .CreateLogger();
-        }
+
     }
 }

sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj

@@ -2,14 +2,14 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp2.0</TargetFramework>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.1.1" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="2.1.1" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.1.1" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.1.1" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="2.2.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.2.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />
     <PackageReference Include="Serilog" Version="2.8.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
   </ItemGroup>

src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs

@@ -92,6 +92,7 @@ public class ElasticsearchSinkOptions
         ///<summary>
         /// The index name formatter. A string.Format using the DateTimeOffset of the event is run over this string.
         /// defaults to "logstash-{0:yyyy.MM.dd}"
+        /// Needs to be lowercased.
         /// </summary>
         public string IndexFormat { get; set; }
 
@@ -286,7 +287,7 @@ public ElasticsearchSinkOptions()
         /// The default Elasticsearch type name used for Elasticsearch versions prior to 7.
         /// <para>As of <c>Elasticsearch 7</c> and up <c>_type</c> has been removed.</para>
         /// </summary>
-        public static string DefaultTypeName { get; } = "logevent";
+        public static string DefaultTypeName { get; } = "_doc";
 
         /// <summary>
         /// Instructs the sink to auto detect the running Elasticsearch version.
@@ -382,7 +383,7 @@ public enum RegisterTemplateRecovery
         IndexAnyway = 1,
 
         ///// <summary>
-        ///// Keep buffering the data until it is written. be aware you might hit a limit here.
+        ///// Keep buffering the data until it is written. be aware you might hit a limit here.                  
         ///// </summary>
         //BufferUntilSuccess = 2,
 

src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs

@@ -82,8 +82,8 @@ private ElasticsearchSinkState(ElasticsearchSinkOptions options)
             _templateName = options.TemplateName;
             _templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2");
 
-            _indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset));
-            _bufferedIndexDecider = options.BufferIndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset));
+            _indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset).ToLowerInvariant());
+            _bufferedIndexDecider = options.BufferIndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset).ToLowerInvariant());
 
             _options = options;
 
@@ -203,13 +203,13 @@ public void RegisterTemplateIfNeeded()
 
         private PostData GetTemplatePostData()
         {
-            //PostData no longer exposes an implict cast from object.  Previously it supported that and would inspect the object Type to
-            //determine if it it was a litteral string to write directly or if it was an object that it needed to serialse.  Now the onus is 
-            //on us to tell it what type we are passing otherwise if the user specified the template as a json string it would be serialised again.
+            //PostData no longer exposes an implicit cast from object.  Previously it supported that and would inspect the object Type to
+            //determine if it it was a literal string to write directly or if it was an object that it needed to serialize.  Now the onus is 
+            //on us to tell it what type we are passing otherwise if the user specified the template as a json string it would be serialized again.
             var template = GetTemplateData();
-            if (template is string)
+            if (template is string s)
             {
-                return PostData.String((string)template);
+                return PostData.String(s);
             }
             else
             {

test/Serilog.Sinks.Elasticsearch.Tests/CustomIndexTypeNameTests.cs

@@ -52,6 +52,47 @@ public void CustomIndex_And_TypeName_EndsUpInTheOutput()
 
         }
 
+        [Fact]
+        public void UpperCasedIndex_And_TypeName_EndsUpInTheOutput()
+        {
+            //DO NOTE that you cant send objects as scalar values through Logger.*("{Scalar}", {});
+            var timestamp = new DateTimeOffset(2013, 05, 28, 22, 10, 20, 666, TimeSpan.FromHours(10));
+            const string messageTemplate = "{Song}++ @{Complex}";
+            var template = new MessageTemplateParser().Parse(messageTemplate);
+            _options.TypeName = "custom-event-type";
+            _options.IndexFormat = "Event-Index-{0:yyyy.MM.dd}";
+            using (var sink = new ElasticsearchSink(_options))
+            {
+                var properties = new List<LogEventProperty> { new LogEventProperty("Song", new ScalarValue("New Macabre")) };
+                var e = new LogEvent(timestamp, LogEventLevel.Information, null, template, properties);
+                //one off 
+                sink.Emit(e);
+
+                sink.Emit(e);
+                var exception = new ArgumentException("parameter");
+                properties = new List<LogEventProperty>
+                {
+                    new LogEventProperty("Song", new ScalarValue("Old Macabre")),
+                    new LogEventProperty("Complex", new ScalarValue(new { A  = 1, B = 2}))
+                };
+                e = new LogEvent(timestamp.AddYears(-2), LogEventLevel.Fatal, exception, template, properties);
+                sink.Emit(e);
+            }
+
+            var bulkJsonPieces = this.AssertSeenHttpPosts(_seenHttpPosts, 4);
+
+            bulkJsonPieces.Should().HaveCount(4);
+            bulkJsonPieces[0].Should().Contain(@"""_index"":""event-index-2013.05.28");
+            bulkJsonPieces[0].Should().Contain(@"""_type"":""custom-event-type");
+            bulkJsonPieces[1].Should().Contain("New Macabre");
+            bulkJsonPieces[2].Should().Contain(@"""_index"":""event-index-2011.05.28");
+            bulkJsonPieces[2].Should().Contain(@"""_type"":""custom-event-type");
+            bulkJsonPieces[3].Should().Contain("Old Macabre");
+
+            bulkJsonPieces[3].Should().Contain("Complex\":{");
+
+        }
+
     }
 
 }

test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs

@@ -69,11 +69,11 @@ protected IList<SerilogElasticsearchEvent> GetPostedLogEvents(int expectedCount)
                 }
                 catch (Exception e)
                 {
-                    throw new Exception(string.Format("Can not deserialize into BulkOperation \r\n:{0}", totalBulks[i]), e);
+                    throw new Exception($"Can not deserialize into BulkOperation \r\n:{totalBulks[i]}", e);
                 }
                 action.IndexAction.Should().NotBeNull();
                 action.IndexAction.Index.Should().NotBeNullOrEmpty().And.StartWith("logstash-");
-                action.IndexAction.Type.Should().NotBeNullOrEmpty().And.Be("logevent");
+                action.IndexAction.Type.Should().NotBeNullOrEmpty().And.Be("_doc");
 
                 SerilogElasticsearchEvent actionMetaData;
                 try
@@ -82,7 +82,8 @@ protected IList<SerilogElasticsearchEvent> GetPostedLogEvents(int expectedCount)
                 }
                 catch (Exception e)
                 {
-                    throw new Exception(string.Format("Can not deserialize into SerilogElasticsearchMessage \r\n:{0}", totalBulks[i + 1]), e);
+                    throw new Exception(
+                        $"Can not deserialize into SerilogElasticsearchMessage \r\n:{totalBulks[i + 1]}", e);
                 }
                 actionMetaData.Should().NotBeNull();
                 bulkActions.Add(actionMetaData);

test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj

@@ -24,13 +24,19 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <None Remove="Templating\template_0replicas.json" />
+    <None Remove="Templating\template_2shards.json" />
+    <None Remove="Templating\template_5replicas.json" />
     <None Remove="Templating\template_v2.json" />
     <None Remove="Templating\template_v5.json" />
     <None Remove="Templating\template_v6.json" />
     <None Remove="Templating\template_v7.json" />
   </ItemGroup>
 
   <ItemGroup>
+    <EmbeddedResource Include="Templating\template_0replicas.json" />
+    <EmbeddedResource Include="Templating\template_2shards.json" />
+    <EmbeddedResource Include="Templating\template_5replicas.json" />
     <EmbeddedResource Include="Templating\template_v7.json" />
     <EmbeddedResource Include="Templating\template_v6.json" />
     <EmbeddedResource Include="Templating\template_v5.json" />

test/Serilog.Sinks.Elasticsearch.Tests/Templating/DoNotRegisterTemplateIfItExists.cs

@@ -25,7 +25,7 @@ private void DoRegister()
         }
 
         [Fact]
-        public void WhenTempplateExists_ShoudNotSendAPutTemplate()
+        public void WhenTemplateExists_ShouldNotSendAPutTemplate()
         {
             DoRegister();
             this._seenHttpPosts.Should().NotBeNullOrEmpty().And.HaveCount(1);

test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetElasticsearchSinkOptions.cs

@@ -0,0 +1,26 @@
+using System;
+using FluentAssertions;
+using Xunit;
+
+namespace Serilog.Sinks.Elasticsearch.Tests.Templating
+{
+    public class SetElasticsearchSinkOptions : ElasticsearchSinkTestsBase
+    {
+
+        [Fact]
+        public void WhenCreatingOptions_NumberOfShardsInjected_NumberOfShardsAreSet()
+        {
+            var options = new ElasticsearchSinkOptions(new Uri("http://localhost:9100"))
+            {
+                AutoRegisterTemplate = true,
+
+                NumberOfShards = 2,
+                NumberOfReplicas = 0
+            };
+
+            options.NumberOfReplicas.Should().Be(0);
+            options.NumberOfShards.Should().Be(2);
+
+        }
+    }
+}