Skip to content

Commit 8a8e771

Browse files
semihokursemihokur
committed
Revert "Merged PR 685164: Forward rocksdb logs to Kusto"
This reverts commit 60bffd1.
1 parent 2a109b4 commit 8a8e771

File tree

10 files changed

+31
-135
lines changed

10 files changed

+31
-135
lines changed

‎Public/Src/Cache/ContentStore/Distributed/MetadataService/RocksDbContentMetadataDatabase.cs

+23-21
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
using System;
55
using System.Collections.Generic;
66
using System.Diagnostics.CodeAnalysis;
7+
using System.Diagnostics.ContractsLight;
8+
using System.Diagnostics.SymbolStore;
9+
using System.Globalization;
710
using System.IO;
811
using System.Linq;
912
using System.Runtime.CompilerServices;
@@ -14,6 +17,7 @@
1417
using BuildXL.Cache.ContentStore.Distributed.NuCache;
1518
using BuildXL.Cache.ContentStore.FileSystem;
1619
using BuildXL.Cache.ContentStore.Hashing;
20+
using BuildXL.Cache.ContentStore.Interfaces.Extensions;
1721
using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
1822
using BuildXL.Cache.ContentStore.Interfaces.Results;
1923
using BuildXL.Cache.ContentStore.Interfaces.Synchronization;
@@ -28,16 +32,18 @@
2832
using BuildXL.Native.IO;
2933
using BuildXL.Utilities;
3034
using BuildXL.Utilities.Collections;
35+
using BuildXL.Utilities.Serialization;
3136
using BuildXL.Utilities.Tasks;
3237
using RocksDbSharp;
33-
using static BuildXL.Cache.ContentStore.Distributed.MetadataService.RocksDbOperations;
3438
using static BuildXL.Engine.Cache.KeyValueStores.RocksDbStore;
3539
using AbsolutePath = BuildXL.Cache.ContentStore.Interfaces.FileSystem.AbsolutePath;
3640

3741
#nullable enable
3842

3943
namespace BuildXL.Cache.ContentStore.Distributed.MetadataService
4044
{
45+
using static RocksDbOperations;
46+
4147
public class RocksDbContentMetadataDatabaseConfiguration : RocksDbContentLocationDatabaseConfiguration
4248
{
4349
public RocksDbContentMetadataDatabaseConfiguration(AbsolutePath storeLocation)
@@ -54,7 +60,7 @@ public RocksDbContentMetadataDatabaseConfiguration(AbsolutePath storeLocation)
5460
}
5561

5662
/// <summary>
57-
/// RocksDb-based version of <see cref="ContentLocationDatabase"/> used by the global location store.
63+
/// RocksDb-based version of <see cref="ContentLocationDatabase"/>.
5864
/// </summary>
5965
public class RocksDbContentMetadataDatabase : ContentLocationDatabase
6066
{
@@ -283,27 +289,23 @@ private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool cle
283289
var dbAlreadyExists = Directory.Exists(storeLocation);
284290
Directory.CreateDirectory(storeLocation);
285291

286-
var settings = new RocksDbStoreConfiguration(storeLocation)
287-
{
288-
AdditionalColumns = ColumnNames.SelectMany(n => n),
289-
RotateLogsMaxFileSizeBytes = 0L,
290-
RotateLogsNumFiles = 60,
291-
RotateLogsMaxAge = TimeSpan.FromHours(12),
292-
EnableStatistics = true,
293-
FastOpen = true,
294-
LeveledCompactionDynamicLevelTargetSizes = true,
295-
Compression = Compression.Zstd,
296-
UseReadOptionsWithSetTotalOrderSeekInDbEnumeration = true,
297-
UseReadOptionsWithSetTotalOrderSeekInGarbageCollection = true,
298-
MergeOperators = GetMergeOperators()
299-
};
300-
301-
RocksDbUtilities.ConfigureRocksDbTracingIfNeeded(context, _configuration, settings, Tracer, componentName: nameof(RocksDbContentMetadataDatabase));
302-
303-
Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'. Clean={clean}, Configured Epoch='{_configuration.Epoch}', TracingLevel={_configuration.RocksDbTracingLevel}");
292+
Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'. Clean={clean}, Configured Epoch='{_configuration.Epoch}'");
304293

305294
var possibleStore = KeyValueStoreAccessor.Open(
306-
settings,
295+
new RocksDbStoreConfiguration(storeLocation)
296+
{
297+
AdditionalColumns = ColumnNames.SelectMany(n => n),
298+
RotateLogsMaxFileSizeBytes = 0L,
299+
RotateLogsNumFiles = 60,
300+
RotateLogsMaxAge = TimeSpan.FromHours(12),
301+
EnableStatistics = true,
302+
FastOpen = true,
303+
LeveledCompactionDynamicLevelTargetSizes = true,
304+
Compression = Compression.Zstd,
305+
UseReadOptionsWithSetTotalOrderSeekInDbEnumeration = true,
306+
UseReadOptionsWithSetTotalOrderSeekInGarbageCollection = true,
307+
MergeOperators = GetMergeOperators()
308+
},
307309
// When an exception is caught from within methods using the database, this handler is called to
308310
// decide whether the exception should be rethrown in user code, and the database invalidated. Our
309311
// policy is to only invalidate if it is an exception coming from RocksDb, but not from our code.

‎Public/Src/Cache/ContentStore/Distributed/MetadataService/RocksDbUtilities.cs

-54
This file was deleted.

‎Public/Src/Cache/ContentStore/Distributed/NuCache/ContentLocationDatabaseConfiguration.cs

+1-9
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
using BuildXL.Cache.ContentStore.Distributed.NuCache.InMemory;
66
using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
77
using BuildXL.Cache.Host.Configuration;
8-
using RocksDbSharp;
98
using static BuildXL.Utilities.ConfigurationHelper;
109

1110
#nullable enable
@@ -61,12 +60,6 @@ public abstract class ContentLocationDatabaseConfiguration
6160
/// </summary>
6261
public bool TraceOperations { get; set; } = true;
6362

64-
/// <summary>
65-
/// Ges or sets log level from RocksDb emitted to Kusto.
66-
/// Null - the tracing is off.
67-
/// </summary>
68-
public LogLevel? RocksDbTracingLevel { get; set; }
69-
7063
/// <summary>
7164
/// Specifies whether to trace touches or not.
7265
/// Tracing touches is expensive in terms of the amount of traffic to Kusto and in terms of memory traffic.
@@ -240,7 +233,6 @@ public static RocksDbContentLocationDatabaseConfiguration FromDistributedContent
240233

241234
configuration.RocksDbPerformanceSettings = settings.RocksDbPerformanceSettings;
242235

243-
ApplyIfNotNull(settings.ContentLocationDatabaseRocksDbTracingLevel, v => configuration.RocksDbTracingLevel = (LogLevel)v);
244236
ApplyIfNotNull(settings.TraceStateChangeDatabaseOperations, v => configuration.TraceOperations = v);
245237
ApplyIfNotNull(settings.TraceNoStateChangeDatabaseOperations, v => configuration.TraceNoStateChangeOperations = v);
246238

@@ -249,7 +241,7 @@ public static RocksDbContentLocationDatabaseConfiguration FromDistributedContent
249241
ApplyIfNotNull(settings.ContentLocationDatabaseMetadataGarbageCollectionMaximumSizeMb, v => configuration.MetadataGarbageCollectionMaximumSizeMb = v);
250242
ApplyIfNotNull(settings.ContentLocationDatabaseMetadataGarbageCollectionLogEnabled, v => configuration.MetadataGarbageCollectionLogEnabled = v);
251243

252-
ApplyIfNotNull(settings.ContentLocationDatabaseOpenReadOnly, v => configuration.OpenReadOnly = (v && !settings.IsMasterEligible));
244+
ApplyIfNotNull(settings.ContentLocationDatabaseOpenReadOnly, v => configuration.OpenReadOnly = v && !settings.IsMasterEligible);
253245
ApplyIfNotNull(settings.UseMergeOperatorForContentLocations, v => configuration.UseMergeOperatorForContentLocations = v);
254246

255247
if (settings.ContentLocationDatabaseLogsBackupEnabled)

‎Public/Src/Cache/ContentStore/Distributed/NuCache/RocksDbContentLocationDatabase.cs

+2-4
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
using System.Linq;
1010
using System.Threading;
1111
using System.Threading.Tasks;
12-
using BuildXL.Cache.ContentStore.Distributed.MetadataService;
1312
using BuildXL.Cache.ContentStore.FileSystem;
1413
using BuildXL.Cache.ContentStore.Hashing;
1514
using BuildXL.Cache.ContentStore.Interfaces.Extensions;
@@ -27,6 +26,7 @@
2726
using BuildXL.Engine.Cache.KeyValueStores;
2827
using BuildXL.Native.IO;
2928
using BuildXL.Utilities;
29+
using BuildXL.Utilities.Collections;
3030
using BuildXL.Utilities.ConfigurationHelpers;
3131
using BuildXL.Utilities.Serialization;
3232
using BuildXL.Utilities.Tasks;
@@ -225,7 +225,7 @@ private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool cle
225225
bool dbAlreadyExists = Directory.Exists(storeLocation);
226226
Directory.CreateDirectory(storeLocation);
227227

228-
Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'. Clean={clean}, UseMergeOperators={_configuration.UseMergeOperatorForContentLocations}, Configured Epoch='{_configuration.Epoch}', TracingLevel={_configuration.RocksDbTracingLevel}");
228+
Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'. Clean={clean}, UseMergeOperators={_configuration.UseMergeOperatorForContentLocations}, Configured Epoch='{_configuration.Epoch}'");
229229

230230
var settings = new RocksDbStoreConfiguration(storeLocation)
231231
{
@@ -255,8 +255,6 @@ private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool cle
255255
UseReadOptionsWithSetTotalOrderSeekInGarbageCollection = _configuration.UseReadOptionsWithSetTotalOrderSeekInGarbageCollection,
256256
};
257257

258-
RocksDbUtilities.ConfigureRocksDbTracingIfNeeded(context, _configuration, settings, Tracer, componentName: nameof(RocksDbContentLocationDatabase));
259-
260258
if (_configuration.UseMergeOperatorForContentLocations)
261259
{
262260
var mergeContext = context.CreateNested(nameof(RocksDbContentLocationDatabase), caller: "MergeContentLocationEntries");

‎Public/Src/Cache/ContentStore/Distributed/Services/DistributedContentStoreServices.cs

-1
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,6 @@ private RocksDbContentMetadataStore CreateRocksDbContentMetadataStore()
218218
MetadataSizeRotationThreshold = DistributedContentSettings.GlobalCacheMetadataSizeRotationThreshold,
219219
};
220220

221-
ApplyIfNotNull(DistributedContentSettings.ContentMetadataDatabaseRocksDbTracingLevel, v => dbConfig.RocksDbTracingLevel = (RocksDbSharp.LogLevel)v);
222221
ApplyIfNotNull(DistributedContentSettings.LocationEntryExpiryMinutes, v => dbConfig.ContentRotationInterval = TimeSpan.FromMinutes(v));
223222
dbConfig.MetadataRotationInterval = DistributedContentSettings.ContentMetadataServerMetadataRotationInterval;
224223

‎Public/Src/Cache/DistributedCache.Host/Configuration/DistributedContentSettings.cs

-19
Original file line numberDiff line numberDiff line change
@@ -220,21 +220,6 @@ public void DisableRedis()
220220
[DataMember]
221221
public bool? TraceStateChangeDatabaseOperations { get; set; }
222222

223-
/// <summary>
224-
/// Defines a tracing level used by RocksDb logging adapter.
225-
/// </summary>
226-
/// <remarks>
227-
/// See RocksDbSharp.LogLevel. The levels are:
228-
/// 0 - Debug,
229-
/// 1 - Info,
230-
/// 2 - Warn,
231-
/// 3 - Error,
232-
/// 4 - Fatal,
233-
/// 5 - Header
234-
/// </remarks>
235-
[DataMember]
236-
public int? ContentLocationDatabaseRocksDbTracingLevel { get; set; } = 0;
237-
238223
[DataMember]
239224
public bool TraceTouches { get; set; } = true;
240225

@@ -967,10 +952,6 @@ public void DisableRedis()
967952
[DataMember]
968953
public bool ContentMetadataUseMergeWrites { get; set; }
969954

970-
/// <inheritdoc cref="ContentLocationDatabaseRocksDbTracingLevel"/>
971-
[DataMember]
972-
public int? ContentMetadataDatabaseRocksDbTracingLevel { get; set; } = 0;
973-
974955
[DataMember]
975956
public string ContentMetadataBlobSecretName { get; set; }
976957

‎Public/Src/Utilities/KeyValueStore/RocksDb/RocksDbStore.cs

-15
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,6 @@ private record struct ColumnFamilyInfo
113113

114114
private readonly RocksDbStoreConfiguration? m_options;
115115

116-
private readonly RocksDbLoggingAdapter? m_loggingAdapter;
117-
118116
/// <summary>
119117
/// Encapsulates <see cref="RocksDb"/> options that should be set.
120118
/// </summary>
@@ -156,12 +154,6 @@ public RocksDbStore(RocksDbStoreConfiguration configuration)
156154
// }
157155
.IncreaseParallelism(performanceConfiguration.GetBackgroundCompactionActualThreadCount());
158156

159-
if (configuration.HandleLogMessage != null)
160-
{
161-
m_loggingAdapter = new RocksDbLoggingAdapter(configuration.HandleLogMessage);
162-
m_defaults.DbOptions = m_defaults.DbOptions.SetInfoLog(m_loggingAdapter);
163-
}
164-
165157
if (performanceConfiguration.DbWriteBufferSize is { } dbWriteBufferSize)
166158
{
167159
m_defaults.DbOptions.SetDbWriteBufferSize(dbWriteBufferSize);
@@ -244,11 +236,6 @@ public RocksDbStore(RocksDbStoreConfiguration configuration)
244236
.SetPrefixExtractor(SliceTransform.CreateNoOp())
245237
.SetLevelCompactionDynamicLevelBytes(configuration.LeveledCompactionDynamicLevelTargetSizes);
246238

247-
if (configuration.HandleLogMessage != null)
248-
{
249-
options = options.SetInfoLogLevel((int)configuration.LogLevel);
250-
}
251-
252239
ColumnFamilyPerformanceConfiguration? perfConfiguration = null;
253240
if (name != null)
254241
{
@@ -965,9 +952,7 @@ public void Dispose()
965952
}
966953
}
967954

968-
// Disabling the log to avoid execution engine exceptions by calling deleted delegates
969955
m_store.Dispose();
970-
m_loggingAdapter?.Dispose();
971956
}
972957
else
973958
{

‎Public/Src/Utilities/KeyValueStore/RocksDb/RocksDbStoreConfiguration.cs

+1-8
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System;
55
using System.Collections.Generic;
66
using System.Text;
7+
using BuildXL.Utilities.Collections;
78
using RocksDbSharp;
89

910
namespace BuildXL.Engine.Cache.KeyValueStores
@@ -160,14 +161,6 @@ public record RocksDbStoreConfiguration(string StoreDirectory)
160161
/// </remarks>
161162
public bool UseReadOptionsWithSetTotalOrderSeekInGarbageCollection { get; init; } = true;
162163

163-
/// <summary>
164-
/// Log handler that will be called once rocksdb traces a message.
165-
/// </summary>
166-
public LogLineCallback? HandleLogMessage { get; set; }
167-
168-
/// <nodoc />
169-
public LogLevel LogLevel { get; set; } = LogLevel.Info;
170-
171164
/// <nodoc />
172165
public RocksDbPerformanceConfiguration PerformanceConfiguration { get; set; } = new RocksDbPerformanceConfiguration();
173166
}

‎cg/nuget/cgmanifest.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -2643,7 +2643,7 @@
26432643
"Type": "NuGet",
26442644
"NuGet": {
26452645
"Name": "RocksDbNative",
2646-
"Version": "6.10.2-b20221102.4"
2646+
"Version": "6.10.2-b20220621.2"
26472647
}
26482648
}
26492649
},
@@ -2652,7 +2652,7 @@
26522652
"Type": "NuGet",
26532653
"NuGet": {
26542654
"Name": "RocksDbSharp",
2655-
"Version": "6.10.2-b20221102.4"
2655+
"Version": "6.10.2-b20220621.2"
26562656
}
26572657
}
26582658
},

‎config.dsc

+2-2
Original file line numberDiff line numberDiff line change
@@ -178,11 +178,11 @@ config({
178178
{ id: "Microsoft.Windows.ProjFS", version: "1.2.19351.1" },
179179

180180
// RocksDb
181-
{ id: "RocksDbSharp", version: "6.10.2-b20221102.4", alias: "RocksDbSharpSigned",
181+
{ id: "RocksDbSharp", version: "6.10.2-b20220621.2", alias: "RocksDbSharpSigned",
182182
dependentPackageIdsToSkip: [ "System.Memory" ],
183183
dependentPackageIdsToIgnore: [ "System.Memory" ]
184184
},
185-
{ id: "RocksDbNative", version: "6.10.2-b20221102.4" },
185+
{ id: "RocksDbNative", version: "6.10.2-b20220621.2" },
186186

187187
{ id: "JsonDiffPatch.Net", version: "2.1.0" },
188188

0 commit comments

Comments
 (0)