Add MongoDB default Guid serialization (#249)

* Added configuration for MongoDB Guid representation

* Allow access to the underlying connection string / MongoUrl for each tenant

Author: Magne Helleborg

Directory.Packages.props

@@ -42,7 +42,7 @@
     <PackageVersion Include="MongoDB.Driver" Version="3.0.0" />
     <PackageVersion Include="MongoDB.Driver.Core.Extensions.DiagnosticSources" Version="2.0.0" />
     <PackageVersion Include="Moq" Version="4.18.4" />
-    <PackageVersion Include="FluentAssertions" Version="6.12.1" />
+    <PackageVersion Include="FluentAssertions" Version="6.12.2" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="Polly" Version="8.4.1" />
     <PackageVersion Include="Proto.Actor" Version="1.7.1-alpha.0.1" />
@@ -53,10 +53,12 @@
     <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.9.0" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
     <PackageVersion Include="System.Collections.Immutable" Version="8.0.0" />
+    <PackageVersion Include="System.Formats.Asn1" Version="8.0.1" />
     <PackageVersion Include="System.Net.Http" Version="4.3.4" />
     <PackageVersion Include="System.Reactive" Version="6.0.1" />
     <PackageVersion Include="System.Security.Claims" Version="4.3.0" />
     <PackageVersion Include="System.Text.Json" Version="8.0.5" />
+    <PackageVersion Include="System.Text.RegularExpressions" Version="4.3.1" />
     <PackageVersion Include="OpenTelemetry.Api" Version="1.9.0" />
     <PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.9.0" />
     <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />

Samples/Environment/docker-compose.yml

@@ -8,7 +8,7 @@ services:
       - 27017:27017
 
   runtime:
-    image: dolittle/runtime:9.6.1
+    image: dolittle/runtime:9.7.0
     volumes:
       - ./runtime.yml:/app/.dolittle/runtime.yml
       - ./appsettings.json:/app/appsettings.json

Samples/Environment/grafana/docker-compose.yml

@@ -1,49 +1,51 @@
-version: "3"
 services:
   tempo:
-    image: grafana/tempo:latest
+    image: grafana/tempo:2.6.1
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
-      - ./tempo-local.yaml:/etc/tempo.yaml
+      - ./tempo.yaml:/etc/tempo.yaml
       - ./tempo-data:/tmp/tempo
     ports:
-      - "3200:3200"   # tempo
+      # - "3200:3200"   # tempo
+      - "127.0.0.1:3200:3200"
       # - "14268:14268"  # jaeger ingest
       # - "4317:4317"  # otlp grpc
       # - "4318:4318"  # otlp http
       # - "9411:9411"   # zipkin
 
   otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.52.0
+    image: otel/opentelemetry-collector-contrib:0.101.0
     depends_on:
       - tempo
     command: [ "--config=/etc/otel-collector.yaml", "--set=service.telemetry.logs.level=debug"]
     volumes:
       - ./otel-collector.yaml:/etc/otel-collector.yaml
     ports:
-      - "4317:4317"  # otlp grpc
+      - "127.0.0.1:4317:4317"  # otlp grpc
 
   loki:
-    image: grafana/loki:2.5.0
+    image: grafana/loki:3.2.1
     ports:
-      - "3100:3100"
+      # - "3100:3100"
+      - "127.0.0.1:3100:3100"
     command: -config.file=/etc/loki/local-config.yaml
 
   prometheus:
-    image: prom/prometheus:latest
-    command: [ "--config.file=/etc/prometheus.yaml", "--web.enable-remote-write-receiver", "--enable-feature=exemplar-storage"]
+    image: prom/prometheus:v2.54.1
+    command: [ "--config.file=/etc/prometheus.yaml", "--web.enable-remote-write-receiver", "--enable-feature=otlp-write-receiver,exemplar-storage"]
     volumes:
       - ./prometheus.yaml:/etc/prometheus.yaml
     ports:
       - "9090:9090"
 
   grafana:
-    image: grafana/grafana:8.5.3
+    image: grafana/grafana:11.2.2
     volumes:
       - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
     environment:
       - GF_AUTH_ANONYMOUS_ENABLED=true
       - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
       - GF_AUTH_DISABLE_LOGIN_FORM=true
+      - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor traceQLStreaming metricsSummary
     ports:
-      - "3000:3000"
+      - "4000:3000"

Samples/Environment/grafana/otel-collector.yaml

@@ -8,22 +8,33 @@ exporters:
     endpoint: tempo:4317
     tls:
       insecure: true
+  otlphttp:
+    endpoint: http://prometheus:9090/api/v1/otlp
+    tls:
+      insecure: true
 
   loki:
-    endpoint: http://loki:3100/loki/api/v1/push
-    format: json
+    endpoint: "http://loki:3100/loki/api/v1/push"
     tls:
       insecure: true
-    # tenant_id: "example"
-    labels:
-      resource:
-        container.name: "container_name"
-        service.name: "service_name"
-        service.version: "service_version"
-        service.instance.id: "service_instance_id"
+#    labels:
+#      resource:
+#        container.name: "container_name"
+#        service.name: "service_name"
+#        service.version: "service_version"
+#        service.instance.id: "service_instance_id"
+  # prometheusremotewrite:
+  #   endpoint: http://prometheus:9090/api/v1/write
+  #   target_info:
 
 processors:
   batch:
+    timeout: 200ms
+  resource:
+    attributes:
+    - key: environment
+      value: "dev"
+      action: upsert
 
 extensions:
   health_check:
@@ -35,9 +46,13 @@ service:
   pipelines:
     traces:
       receivers: [otlp]
-      processors: [batch]
+      processors: [resource, batch]
       exporters: [otlp]
+    metrics:
+      receivers: [otlp]
+      processors: [resource, batch]
+      exporters: [otlphttp]
     logs:
       receivers: [otlp]
-      processors: [batch]
+      processors: [resource, batch]
       exporters: [loki]

Samples/Environment/grafana/tempo-local.yaml

@@ -0,0 +1,57 @@
+stream_over_http_enabled: true
+server:
+  http_listen_port: 3200
+  log_level: info
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
+      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+        thrift_http:                   #
+        grpc:                          # for a production deployment you should only enable the receivers you need!
+        thrift_binary:
+        thrift_compact:
+    zipkin:
+    otlp:
+      protocols:
+        http:
+        grpc:
+    opencensus:
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h                # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /tmp/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /tmp/tempo/wal             # where to store the the wal locally
+    local:
+      path: /tmp/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics] # enables metrics generator

Samples/Environment/grafana/tempo.yaml

@@ -1,8 +1,10 @@
 // Copyright (c) Dolittle. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using MongoDB.Bson;
 using MongoDB.Bson.Serialization;
 using MongoDB.Bson.Serialization.Conventions;
+using MongoDB.Bson.Serialization.Serializers;
 
 namespace Dolittle.SDK.Resources.MongoDB;
 
@@ -22,7 +24,7 @@ public static class DolittleMongoConventions
     /// <summary>
     /// Ensured that the default Mongo Conventions are registered;
     /// </summary>
-    public static void EnsureConventionsAreRegistered()
+    public static void EnsureConventionsAreRegistered(GuidRepresentation defaultRepresentation)
     {
         if (_isRegistered)
         {
@@ -36,6 +38,10 @@ public static void EnsureConventionsAreRegistered()
             }
             _isRegistered = true;
             BsonSerializer.RegisterSerializationProvider(new ConceptSerializationProvider());
+            if (defaultRepresentation != GuidRepresentation.Unspecified)
+            {
+                BsonSerializer.RegisterSerializer(new GuidSerializer(defaultRepresentation));
+            }
             var pack = new ConventionPack();
             pack.AddClassMapConvention("Ignore extra elements", _ => _.SetIgnoreExtraElements(true));
             ConventionRegistry.Register(ConventionPackName, pack,  _ => true);

Source/Resources/MongoDB/DolittleMongoConventions.cs

@@ -17,4 +17,14 @@ public interface IMongoDBResource
     /// <param name="databaseSettingsCallback">The <see cref="Action{T}"/> callback for creating <see cref="MongoDatabaseSettings"/> used to create the <see cref="IMongoDatabase"/>.</param>
     /// <returns>An <see cref="IMongoDatabase"/>.</returns>
     IMongoDatabase GetDatabase(Action<MongoDatabaseSettings>? databaseSettingsCallback = default);
+    
+    /// <summary>
+    /// Gets the connection string that is used to connect to the MongoDB resource.
+    /// </summary>
+    string ConnectionString { get; }
+    
+    /// <summary>
+    /// Gets the <see cref="MongoUrl"/> that is used to connect to the MongoDB resource.
+    /// </summary>
+    MongoUrl ConnectionUrl { get; }
 }

Source/Resources/MongoDB/IMongoDBResource.cs

@@ -12,7 +12,6 @@ namespace Dolittle.SDK.Resources.MongoDB.Internal;
 /// </summary>
 public class MongoDBResource : IMongoDBResource
 {
-    readonly MongoUrl _mongoUrl;
     readonly MongoClient _mongoClient;
 
     /// <summary>
@@ -23,8 +22,9 @@ public class MongoDBResource : IMongoDBResource
     public MongoDBResource(GetMongoDBResponse runtimeMongoDbResponse,
         Action<MongoClientSettings>? clientSettingsCallback)
     {
-        _mongoUrl = MongoUrl.Create(runtimeMongoDbResponse.ConnectionString);
-        var clientSettings = MongoClientSettings.FromUrl(_mongoUrl);
+        ConnectionString = runtimeMongoDbResponse.ConnectionString;
+        ConnectionUrl = MongoUrl.Create(runtimeMongoDbResponse.ConnectionString);
+        var clientSettings = MongoClientSettings.FromUrl(ConnectionUrl);
         ConfigureSettings(clientSettingsCallback, clientSettings);
 
         _mongoClient = new MongoClient(clientSettings.Freeze());
@@ -34,12 +34,16 @@ public MongoDBResource(GetMongoDBResponse runtimeMongoDbResponse,
     /// <inheritdoc />
     public IMongoDatabase GetDatabase(Action<MongoDatabaseSettings>? databaseSettingsCallback = default)
     {
-        DolittleMongoConventions.EnsureConventionsAreRegistered();
         var databaseSettings = new MongoDatabaseSettings();
         databaseSettingsCallback?.Invoke(databaseSettings);
-        return _mongoClient.GetDatabase(_mongoUrl.DatabaseName, databaseSettings);
+        return _mongoClient.GetDatabase(ConnectionUrl.DatabaseName, databaseSettings);
     }
 
+    public string ConnectionString { get; }
+
+    /// <inheritdoc />
+    public MongoUrl ConnectionUrl { get; }
+
     /// <summary>
     /// Configures <see cref="MongoClientSettings"/> class.
     /// </summary>