Skip to content

Commit 643fdbe

Browse files
itssimonitssimon
authored
Add masking for request and response body fields (#10)
* Add masking for request and response body fields * Fix HasJsonContentType
1 parent eaf22f1 commit 643fdbe

3 files changed

Lines changed: 403 additions & 158 deletions

File tree

‎src/Apitally/ApitallyOptions.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ public class RequestLoggingOptions
3232
public bool IncludeException { get; set; } = true;
3333
public List<string> QueryParamMaskPatterns { get; set; } = [];
3434
public List<string> HeaderMaskPatterns { get; set; } = [];
35+
public List<string> BodyFieldMaskPatterns { get; set; } = [];
3536
public List<string> PathExcludePatterns { get; set; } = [];
3637

3738
/// <summary>

‎src/Apitally/RequestLogger.cs‎

Lines changed: 166 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ namespace Apitally;
44
using System.Net.Mime;
55
using System.Text;
66
using System.Text.Json;
7+
using System.Text.Json.Nodes;
78
using System.Text.RegularExpressions;
89
using Apitally.Models;
910
using Microsoft.Extensions.Hosting;
@@ -55,6 +56,17 @@ class RequestLogger(IOptions<ApitallyOptions> options, ILogger<RequestLogger> lo
5556
"token",
5657
"cookie",
5758
];
59+
private static readonly string[] MaskBodyFieldPatterns =
60+
[
61+
"password",
62+
"pwd",
63+
"token",
64+
"secret",
65+
"auth",
66+
"card[-_ ]?number",
67+
"ccv",
68+
"ssn",
69+
];
5870

5971
public static readonly string[] AllowedContentTypes =
6072
[
@@ -63,7 +75,7 @@ class RequestLogger(IOptions<ApitallyOptions> options, ILogger<RequestLogger> lo
6375
];
6476

6577
private readonly object _lock = new();
66-
private readonly ConcurrentQueue<string> _pendingWrites = new();
78+
private readonly ConcurrentQueue<RequestLogItem> _pendingWrites = new();
6779
private readonly ConcurrentQueue<TempGzipFile> _files = new();
6880
private readonly List<Regex> _compiledPathExcludePatterns = CompilePatterns(
6981
ExcludePathPatterns,
@@ -81,6 +93,10 @@ class RequestLogger(IOptions<ApitallyOptions> options, ILogger<RequestLogger> lo
8193
MaskHeaderPatterns,
8294
options.Value.RequestLogging.HeaderMaskPatterns
8395
);
96+
private readonly List<Regex> _compiledBodyFieldMaskPatterns = CompilePatterns(
97+
MaskBodyFieldPatterns,
98+
options.Value.RequestLogging.BodyFieldMaskPatterns
99+
);
84100
private readonly JsonSerializerOptions _serializerOptions = new()
85101
{
86102
DefaultIgnoreCondition =
@@ -129,81 +145,21 @@ public void LogRequest(Request request, Response response, Exception? exception
129145
return;
130146
}
131147

132-
// Process query params and URL
133-
if (!string.IsNullOrEmpty(request.Url))
134-
{
135-
var uri = new Uri(request.Url);
136-
var query = uri.Query.TrimStart('?');
137-
if (!requestLoggingOptions.IncludeQueryParams)
138-
{
139-
query = string.Empty;
140-
}
141-
else if (!string.IsNullOrEmpty(query))
142-
{
143-
query = MaskQueryParams(query);
144-
}
145-
var uriBuilder = new UriBuilder(uri) { Query = query };
146-
request.Url = uriBuilder.Uri.ToString();
147-
}
148-
149-
// Process request body
150148
if (
151149
!requestLoggingOptions.IncludeRequestBody
152150
|| !HasSupportedContentType(request.Headers)
153151
)
154152
{
155153
request.Body = null;
156154
}
157-
else if (request.Body != null)
158-
{
159-
if (request.Body.Length > MaxBodySize)
160-
{
161-
request.Body = BodyTooLarge;
162-
}
163-
else
164-
{
165-
request.Body = requestLoggingOptions.MaskRequestBody(request) ?? BodyMasked;
166-
if (request.Body.Length > MaxBodySize)
167-
{
168-
request.Body = BodyTooLarge;
169-
}
170-
}
171-
}
172-
173-
// Process response body
174155
if (
175156
!requestLoggingOptions.IncludeResponseBody
176157
|| !HasSupportedContentType(response.Headers)
177158
)
178159
{
179160
response.Body = null;
180161
}
181-
else if (response.Body != null)
182-
{
183-
if (response.Body.Length > MaxBodySize)
184-
{
185-
response.Body = BodyTooLarge;
186-
}
187-
else
188-
{
189-
response.Body =
190-
requestLoggingOptions.MaskResponseBody(request, response) ?? BodyMasked;
191-
if (response.Body.Length > MaxBodySize)
192-
{
193-
response.Body = BodyTooLarge;
194-
}
195-
}
196-
}
197-
198-
// Process headers
199-
request.Headers = requestLoggingOptions.IncludeRequestHeaders
200-
? MaskHeaders(request.Headers)
201-
: [];
202-
response.Headers = requestLoggingOptions.IncludeResponseHeaders
203-
? MaskHeaders(response.Headers)
204-
: [];
205162

206-
// Create exception info
207163
var exceptionInfo =
208164
exception != null && requestLoggingOptions.IncludeException
209165
? new ExceptionInfo
@@ -216,15 +172,14 @@ public void LogRequest(Request request, Response response, Exception? exception
216172
}
217173
: null;
218174

219-
// Create log item
175+
// Create log item and enqueue
220176
var item = new RequestLogItem
221177
{
222178
Request = request,
223179
Response = response,
224180
Exception = exceptionInfo,
225181
};
226-
var serializedItem = JsonSerializer.Serialize(item, _serializerOptions);
227-
_pendingWrites.Enqueue(serializedItem);
182+
_pendingWrites.Enqueue(item);
228183

229184
if (_pendingWrites.Count > MaxPendingWrites)
230185
{
@@ -249,7 +204,9 @@ private void WriteToFile()
249204
_currentFile ??= new TempGzipFile();
250205
while (_pendingWrites.TryDequeue(out var item))
251206
{
252-
_currentFile.WriteLine(Encoding.UTF8.GetBytes(item));
207+
ApplyMasking(item);
208+
var serializedItem = JsonSerializer.Serialize(item, _serializerOptions);
209+
_currentFile.WriteLine(Encoding.UTF8.GetBytes(serializedItem));
253210
}
254211
}
255212
}
@@ -315,6 +272,80 @@ public void Clear()
315272
}
316273
}
317274

275+
private void ApplyMasking(RequestLogItem item)
276+
{
277+
var requestLoggingOptions = options.Value.RequestLogging;
278+
var request = item.Request;
279+
var response = item.Response;
280+
281+
if (request.Body != null)
282+
{
283+
// Apply user-provided masking callback for request body
284+
request.Body = requestLoggingOptions.MaskRequestBody(request) ?? BodyMasked;
285+
286+
if (request.Body.Length > MaxBodySize)
287+
{
288+
request.Body = BodyTooLarge;
289+
}
290+
291+
// Mask request body fields (if JSON)
292+
if (
293+
!request.Body.SequenceEqual(BodyTooLarge)
294+
&& !request.Body.SequenceEqual(BodyMasked)
295+
&& RequestLogger.HasJsonContentType(request.Headers)
296+
)
297+
{
298+
request.Body = MaskJsonBody(request.Body);
299+
}
300+
}
301+
302+
if (response.Body != null)
303+
{
304+
// Apply user-provided masking callback for response body
305+
response.Body = requestLoggingOptions.MaskResponseBody(request, response) ?? BodyMasked;
306+
307+
if (response.Body.Length > MaxBodySize)
308+
{
309+
response.Body = BodyTooLarge;
310+
}
311+
312+
// Mask response body fields (if JSON)
313+
if (
314+
!response.Body.SequenceEqual(BodyTooLarge)
315+
&& !response.Body.SequenceEqual(BodyMasked)
316+
&& RequestLogger.HasJsonContentType(response.Headers)
317+
)
318+
{
319+
response.Body = MaskJsonBody(response.Body);
320+
}
321+
}
322+
323+
// Mask headers
324+
request.Headers = requestLoggingOptions.IncludeRequestHeaders
325+
? MaskHeaders(request.Headers)
326+
: [];
327+
response.Headers = requestLoggingOptions.IncludeResponseHeaders
328+
? MaskHeaders(response.Headers)
329+
: [];
330+
331+
// Mask query params
332+
if (!string.IsNullOrEmpty(request.Url))
333+
{
334+
var uri = new Uri(request.Url);
335+
var query = uri.Query.TrimStart('?');
336+
if (!requestLoggingOptions.IncludeQueryParams)
337+
{
338+
query = string.Empty;
339+
}
340+
else if (!string.IsNullOrEmpty(query))
341+
{
342+
query = MaskQueryParams(query);
343+
}
344+
var uriBuilder = new UriBuilder(uri) { Query = query };
345+
request.Url = uriBuilder.Uri.ToString();
346+
}
347+
}
348+
318349
private bool ShouldExcludePath(string? path)
319350
{
320351
return !string.IsNullOrEmpty(path)
@@ -337,6 +368,69 @@ private bool ShouldMaskHeader(string name)
337368
return _compiledHeaderMaskPatterns.Any(p => p.IsMatch(name));
338369
}
339370

371+
private bool ShouldMaskBodyField(string name)
372+
{
373+
return _compiledBodyFieldMaskPatterns.Any(p => p.IsMatch(name));
374+
}
375+
376+
private byte[] MaskJsonBody(byte[] body)
377+
{
378+
try
379+
{
380+
var json = Encoding.UTF8.GetString(body);
381+
var node = JsonNode.Parse(json);
382+
if (node != null)
383+
{
384+
MaskJsonNode(node);
385+
return Encoding.UTF8.GetBytes(node.ToJsonString(_serializerOptions));
386+
}
387+
return body;
388+
}
389+
catch
390+
{
391+
return body;
392+
}
393+
}
394+
395+
private void MaskJsonNode(JsonNode node)
396+
{
397+
switch (node)
398+
{
399+
case JsonObject jsonObject:
400+
var propertiesToMask = new List<string>();
401+
foreach (var property in jsonObject)
402+
{
403+
if (
404+
property.Value is JsonValue jsonValue
405+
&& jsonValue.TryGetValue<string>(out _)
406+
&& ShouldMaskBodyField(property.Key)
407+
)
408+
{
409+
propertiesToMask.Add(property.Key);
410+
}
411+
else if (property.Value != null)
412+
{
413+
MaskJsonNode(property.Value);
414+
}
415+
}
416+
foreach (var propertyKey in propertiesToMask)
417+
{
418+
jsonObject[propertyKey] = JsonValue.Create(Masked);
419+
}
420+
break;
421+
422+
case JsonArray jsonArray:
423+
foreach (var item in jsonArray)
424+
{
425+
if (item != null)
426+
{
427+
MaskJsonNode(item);
428+
}
429+
}
430+
break;
431+
}
432+
}
433+
340434
private string MaskQueryParams(string query)
341435
{
342436
if (string.IsNullOrEmpty(query))
@@ -375,6 +469,13 @@ private static bool HasSupportedContentType(Header[] headers)
375469
);
376470
}
377471

472+
private static bool HasJsonContentType(Header[] headers)
473+
{
474+
var contentType = GetHeaderValue(headers, "content-type");
475+
return contentType != null
476+
&& Regex.IsMatch(contentType, @"\bjson\b", RegexOptions.IgnoreCase);
477+
}
478+
378479
private static string? GetHeaderValue(Header[] headers, string name)
379480
{
380481
return headers

0 commit comments

Comments
 (0)