Skip to content

Commit e1e04f9

Browse files
SergeyTeplyakovSergeyTeplyakov
committed
Merged PR 693228: Forward rocksdb logs to Kusto
This is a manual rollback of [this rollback](https://dev.azure.com/mseng/Domino/_git/BuildXL.Internal/pullrequest/689216).
1 parent 7bc194b commit e1e04f9

File tree

8 files changed

+133
-26
lines changed

8 files changed

+133
-26
lines changed

‎Public/Src/Cache/ContentStore/Distributed/MetadataService/RocksDbContentMetadataDatabase.cs

+21-23
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,6 @@
44
using System;
55
using System.Collections.Generic;
66
using System.Diagnostics.CodeAnalysis;
7-
using System.Diagnostics.ContractsLight;
8-
using System.Diagnostics.SymbolStore;
9-
using System.Globalization;
107
using System.IO;
118
using System.Linq;
129
using System.Runtime.CompilerServices;
@@ -17,7 +14,6 @@
1714
using BuildXL.Cache.ContentStore.Distributed.NuCache;
1815
using BuildXL.Cache.ContentStore.FileSystem;
1916
using BuildXL.Cache.ContentStore.Hashing;
20-
using BuildXL.Cache.ContentStore.Interfaces.Extensions;
2117
using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
2218
using BuildXL.Cache.ContentStore.Interfaces.Results;
2319
using BuildXL.Cache.ContentStore.Interfaces.Synchronization;
@@ -32,18 +28,16 @@
3228
using BuildXL.Native.IO;
3329
using BuildXL.Utilities;
3430
using BuildXL.Utilities.Collections;
35-
using BuildXL.Utilities.Serialization;
3631
using BuildXL.Utilities.Tasks;
3732
using RocksDbSharp;
33+
using static BuildXL.Cache.ContentStore.Distributed.MetadataService.RocksDbOperations;
3834
using static BuildXL.Engine.Cache.KeyValueStores.RocksDbStore;
3935
using AbsolutePath = BuildXL.Cache.ContentStore.Interfaces.FileSystem.AbsolutePath;
4036

4137
#nullable enable
4238

4339
namespace BuildXL.Cache.ContentStore.Distributed.MetadataService
4440
{
45-
using static RocksDbOperations;
46-
4741
public class RocksDbContentMetadataDatabaseConfiguration : RocksDbContentLocationDatabaseConfiguration
4842
{
4943
public RocksDbContentMetadataDatabaseConfiguration(AbsolutePath storeLocation)
@@ -60,7 +54,7 @@ public RocksDbContentMetadataDatabaseConfiguration(AbsolutePath storeLocation)
6054
}
6155

6256
/// <summary>
63-
/// RocksDb-based version of <see cref="ContentLocationDatabase"/>.
57+
/// RocksDb-based version of <see cref="ContentLocationDatabase"/> used by the global location store.
6458
/// </summary>
6559
public class RocksDbContentMetadataDatabase : ContentLocationDatabase
6660
{
@@ -289,23 +283,27 @@ private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool cle
289283
var dbAlreadyExists = Directory.Exists(storeLocation);
290284
Directory.CreateDirectory(storeLocation);
291285

292-
Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'. Clean={clean}, Configured Epoch='{_configuration.Epoch}'");
286+
var settings = new RocksDbStoreConfiguration(storeLocation)
287+
{
288+
AdditionalColumns = ColumnNames.SelectMany(n => n),
289+
RotateLogsMaxFileSizeBytes = 0L,
290+
RotateLogsNumFiles = 60,
291+
RotateLogsMaxAge = TimeSpan.FromHours(12),
292+
EnableStatistics = true,
293+
FastOpen = true,
294+
LeveledCompactionDynamicLevelTargetSizes = true,
295+
Compression = Compression.Zstd,
296+
UseReadOptionsWithSetTotalOrderSeekInDbEnumeration = true,
297+
UseReadOptionsWithSetTotalOrderSeekInGarbageCollection = true,
298+
MergeOperators = GetMergeOperators()
299+
};
300+
301+
RocksDbUtilities.ConfigureRocksDbTracingIfNeeded(context, _configuration, settings, Tracer, componentName: nameof(RocksDbContentMetadataDatabase));
302+
303+
Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'. Clean={clean}, Configured Epoch='{_configuration.Epoch}', TracingLevel={_configuration.RocksDbTracingLevel}");
293304

294305
var possibleStore = KeyValueStoreAccessor.Open(
295-
new RocksDbStoreConfiguration(storeLocation)
296-
{
297-
AdditionalColumns = ColumnNames.SelectMany(n => n),
298-
RotateLogsMaxFileSizeBytes = 0L,
299-
RotateLogsNumFiles = 60,
300-
RotateLogsMaxAge = TimeSpan.FromHours(12),
301-
EnableStatistics = true,
302-
FastOpen = true,
303-
LeveledCompactionDynamicLevelTargetSizes = true,
304-
Compression = Compression.Zstd,
305-
UseReadOptionsWithSetTotalOrderSeekInDbEnumeration = true,
306-
UseReadOptionsWithSetTotalOrderSeekInGarbageCollection = true,
307-
MergeOperators = GetMergeOperators()
308-
},
306+
settings,
309307
// When an exception is caught from within methods using the database, this handler is called to
310308
// decide whether the exception should be rethrown in user code, and the database invalidated. Our
311309
// policy is to only invalidate if it is an exception coming from RocksDb, but not from our code.

‎Public/Src/Cache/ContentStore/Distributed/NuCache/ContentLocationDatabaseConfiguration.cs

+9-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using BuildXL.Cache.ContentStore.Distributed.NuCache.InMemory;
66
using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
77
using BuildXL.Cache.Host.Configuration;
8+
using RocksDbSharp;
89
using static BuildXL.Utilities.ConfigurationHelper;
910

1011
#nullable enable
@@ -60,6 +61,12 @@ public abstract class ContentLocationDatabaseConfiguration
6061
/// </summary>
6162
public bool TraceOperations { get; set; } = true;
6263

64+
/// <summary>
65+
/// Ges or sets log level from RocksDb emitted to Kusto.
66+
/// Null - the tracing is off.
67+
/// </summary>
68+
public LogLevel? RocksDbTracingLevel { get; set; }
69+
6370
/// <summary>
6471
/// Specifies whether to trace touches or not.
6572
/// Tracing touches is expensive in terms of the amount of traffic to Kusto and in terms of memory traffic.
@@ -221,6 +228,7 @@ public static RocksDbContentLocationDatabaseConfiguration FromDistributedContent
221228

222229
configuration.RocksDbPerformanceSettings = settings.RocksDbPerformanceSettings;
223230

231+
ApplyIfNotNull(settings.ContentLocationDatabaseRocksDbTracingLevel, v => configuration.RocksDbTracingLevel = (LogLevel)v);
224232
ApplyIfNotNull(settings.TraceStateChangeDatabaseOperations, v => configuration.TraceOperations = v);
225233
ApplyIfNotNull(settings.TraceNoStateChangeDatabaseOperations, v => configuration.TraceNoStateChangeOperations = v);
226234

@@ -229,7 +237,7 @@ public static RocksDbContentLocationDatabaseConfiguration FromDistributedContent
229237
ApplyIfNotNull(settings.ContentLocationDatabaseMetadataGarbageCollectionMaximumSizeMb, v => configuration.MetadataGarbageCollectionMaximumSizeMb = v);
230238
ApplyIfNotNull(settings.ContentLocationDatabaseMetadataGarbageCollectionLogEnabled, v => configuration.MetadataGarbageCollectionLogEnabled = v);
231239

232-
ApplyIfNotNull(settings.ContentLocationDatabaseOpenReadOnly, v => configuration.OpenReadOnly = v && !settings.IsMasterEligible);
240+
ApplyIfNotNull(settings.ContentLocationDatabaseOpenReadOnly, v => configuration.OpenReadOnly = (v && !settings.IsMasterEligible));
233241
ApplyIfNotNull(settings.UseMergeOperatorForContentLocations, v => configuration.UseMergeOperatorForContentLocations = v);
234242
ApplyIfNotNull(settings.SortMergeableContentLocations, v => configuration.SortMergeableContentLocations = v);
235243

‎Public/Src/Cache/ContentStore/Distributed/NuCache/RocksDbContentLocationDatabase.cs

+5-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
using System.Linq;
1010
using System.Threading;
1111
using System.Threading.Tasks;
12+
using BuildXL.Cache.ContentStore.Distributed.MetadataService;
13+
using BuildXL.Cache.ContentStore.FileSystem;
1214
using BuildXL.Cache.ContentStore.Hashing;
1315
using BuildXL.Cache.ContentStore.Interfaces.Results;
1416
using BuildXL.Cache.ContentStore.Interfaces.Synchronization;
@@ -210,7 +212,7 @@ private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool cle
210212
bool dbAlreadyExists = Directory.Exists(storeLocation);
211213
Directory.CreateDirectory(storeLocation);
212214

213-
Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'. Clean={clean}, UseMergeOperators={_configuration.UseMergeOperatorForContentLocations}, Configured Epoch='{_configuration.Epoch}'");
215+
Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'. Clean={clean}, UseMergeOperators={_configuration.UseMergeOperatorForContentLocations}, Configured Epoch='{_configuration.Epoch}', TracingLevel={_configuration.RocksDbTracingLevel}");
214216

215217
var settings = new RocksDbStoreConfiguration(storeLocation)
216218
{
@@ -240,6 +242,8 @@ private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool cle
240242
UseReadOptionsWithSetTotalOrderSeekInGarbageCollection = _configuration.UseReadOptionsWithSetTotalOrderSeekInGarbageCollection,
241243
};
242244

245+
RocksDbUtilities.ConfigureRocksDbTracingIfNeeded(context, _configuration, settings, Tracer, componentName: nameof(RocksDbContentLocationDatabase));
246+
243247
if (_configuration.UseMergeOperatorForContentLocations)
244248
{
245249
var mergeContext = context.CreateNested(nameof(RocksDbContentLocationDatabase), caller: "MergeContentLocationEntries");

‎Public/Src/Cache/ContentStore/Distributed/RocksDbUtilities.cs

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using BuildXL.Cache.ContentStore.Distributed.NuCache;
6+
using BuildXL.Cache.ContentStore.Tracing;
7+
using BuildXL.Cache.ContentStore.Tracing.Internal;
8+
using BuildXL.Engine.Cache.KeyValueStores;
9+
using RocksDbSharp;
10+
11+
namespace BuildXL.Cache.ContentStore.Distributed.MetadataService
12+
{
13+
internal static class RocksDbUtilities
14+
{
15+
public static void ConfigureRocksDbTracingIfNeeded(
16+
OperationContext context,
17+
RocksDbContentLocationDatabaseConfiguration configuration,
18+
RocksDbStoreConfiguration settings,
19+
Tracer tracer,
20+
string componentName)
21+
{
22+
if (!configuration.OpenReadOnly && configuration.RocksDbTracingLevel is { } tracingLevel)
23+
{
24+
// Tracing operations only for non-readonly database.
25+
// We don't know if the current instance will be on a master machine, but the configuration is happening
26+
// before such decision is made.
27+
var traceLineHandler = CreateLogLineCallback(context, tracer, componentName: componentName);
28+
settings.LogLevel = tracingLevel;
29+
settings.HandleLogMessage = traceLineHandler;
30+
}
31+
}
32+
33+
public static LogLineCallback CreateLogLineCallback(OperationContext context, Tracer tracer, string componentName)
34+
{
35+
const string operation = "RocksDbTrace";
36+
var rocksDbContext = context.CreateNested(componentName, caller: operation).TracingContext;
37+
return (logLevel, message) =>
38+
{
39+
Action<string> targetMethod = logLevel switch
40+
{
41+
LogLevel.Debug => (string msg) => tracer.Debug(rocksDbContext, msg, operation: operation),
42+
LogLevel.Info => (string msg) => tracer.Info(rocksDbContext, msg, operation: operation),
43+
LogLevel.Warn => (string msg) => tracer.Warning(rocksDbContext, msg, operation: operation),
44+
LogLevel.Error => (string msg) => tracer.Error(rocksDbContext, msg, operation: operation),
45+
LogLevel.Fatal => (string msg) => tracer.Error(rocksDbContext, msg, operation: operation),
46+
LogLevel.Header => (string msg) => tracer.Info(rocksDbContext, msg, operation: operation),
47+
// Do nothing in case of an error
48+
_ => (string msg) => { }
49+
};
50+
51+
targetMethod(message);
52+
};
53+
}
54+
}
55+
}

‎Public/Src/Cache/ContentStore/Distributed/Services/DistributedContentStoreServices.cs

+1
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,7 @@ private RocksDbContentMetadataStore CreateRocksDbContentMetadataStore()
218218
MetadataSizeRotationThreshold = DistributedContentSettings.GlobalCacheMetadataSizeRotationThreshold,
219219
};
220220

221+
ApplyIfNotNull(DistributedContentSettings.ContentMetadataDatabaseRocksDbTracingLevel, v => dbConfig.RocksDbTracingLevel = (RocksDbSharp.LogLevel)v);
221222
ApplyIfNotNull(DistributedContentSettings.LocationEntryExpiryMinutes, v => dbConfig.ContentRotationInterval = TimeSpan.FromMinutes(v));
222223
dbConfig.MetadataRotationInterval = DistributedContentSettings.ContentMetadataServerMetadataRotationInterval;
223224

‎Public/Src/Cache/DistributedCache.Host/Configuration/DistributedContentSettings.cs

+19
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,21 @@ public void DisableRedis()
220220
[DataMember]
221221
public bool? TraceStateChangeDatabaseOperations { get; set; }
222222

223+
/// <summary>
224+
/// Defines a tracing level used by RocksDb logging adapter.
225+
/// </summary>
226+
/// <remarks>
227+
/// See RocksDbSharp.LogLevel. The levels are:
228+
/// 0 - Debug,
229+
/// 1 - Info,
230+
/// 2 - Warn,
231+
/// 3 - Error,
232+
/// 4 - Fatal,
233+
/// 5 - Header
234+
/// </remarks>
235+
[DataMember]
236+
public int? ContentLocationDatabaseRocksDbTracingLevel { get; set; } = 1;
237+
223238
[DataMember]
224239
public bool TraceTouches { get; set; } = true;
225240

@@ -961,6 +976,10 @@ public void DisableRedis()
961976
[DataMember]
962977
public bool ContentMetadataUseMergeWrites { get; set; }
963978

979+
/// <inheritdoc cref="ContentLocationDatabaseRocksDbTracingLevel"/>
980+
[DataMember]
981+
public int? ContentMetadataDatabaseRocksDbTracingLevel { get; set; }
982+
964983
[DataMember]
965984
public string ContentMetadataBlobSecretName { get; set; }
966985

‎Public/Src/Utilities/KeyValueStore/RocksDb/RocksDbStore.cs

+15
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,8 @@ private record struct ColumnFamilyInfo
113113

114114
private readonly RocksDbStoreConfiguration? m_options;
115115

116+
private readonly RocksDbLoggingAdapter? m_loggingAdapter;
117+
116118
/// <summary>
117119
/// Encapsulates <see cref="RocksDb"/> options that should be set.
118120
/// </summary>
@@ -154,6 +156,12 @@ public RocksDbStore(RocksDbStoreConfiguration configuration)
154156
// }
155157
.IncreaseParallelism(performanceConfiguration.GetBackgroundCompactionActualThreadCount());
156158

159+
if (configuration.HandleLogMessage != null)
160+
{
161+
m_loggingAdapter = new RocksDbLoggingAdapter(configuration.HandleLogMessage);
162+
m_defaults.DbOptions = m_defaults.DbOptions.SetInfoLog(m_loggingAdapter);
163+
}
164+
157165
if (performanceConfiguration.DbWriteBufferSize is { } dbWriteBufferSize)
158166
{
159167
m_defaults.DbOptions.SetDbWriteBufferSize(dbWriteBufferSize);
@@ -236,6 +244,11 @@ public RocksDbStore(RocksDbStoreConfiguration configuration)
236244
.SetPrefixExtractor(SliceTransform.CreateNoOp())
237245
.SetLevelCompactionDynamicLevelBytes(configuration.LeveledCompactionDynamicLevelTargetSizes);
238246

247+
if (configuration.HandleLogMessage != null)
248+
{
249+
options = options.SetInfoLogLevel((int)configuration.LogLevel);
250+
}
251+
239252
ColumnFamilyPerformanceConfiguration? perfConfiguration = null;
240253
if (name != null)
241254
{
@@ -952,7 +965,9 @@ public void Dispose()
952965
}
953966
}
954967

968+
// Disabling the log to avoid execution engine exceptions by calling deleted delegates
955969
m_store.Dispose();
970+
m_loggingAdapter?.Dispose();
956971
}
957972
else
958973
{

‎Public/Src/Utilities/KeyValueStore/RocksDb/RocksDbStoreConfiguration.cs

+8-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
using System;
55
using System.Collections.Generic;
66
using System.Text;
7-
using BuildXL.Utilities.Collections;
87
using RocksDbSharp;
98

109
namespace BuildXL.Engine.Cache.KeyValueStores
@@ -161,6 +160,14 @@ public record RocksDbStoreConfiguration(string StoreDirectory)
161160
/// </remarks>
162161
public bool UseReadOptionsWithSetTotalOrderSeekInGarbageCollection { get; init; } = true;
163162

163+
/// <summary>
164+
/// Log handler that will be called once rocksdb traces a message.
165+
/// </summary>
166+
public LogLineCallback? HandleLogMessage { get; set; }
167+
168+
/// <nodoc />
169+
public LogLevel LogLevel { get; set; } = LogLevel.Info;
170+
164171
/// <nodoc />
165172
public RocksDbPerformanceConfiguration PerformanceConfiguration { get; set; } = new RocksDbPerformanceConfiguration();
166173
}

0 commit comments

Comments
 (0)