Skip to content

Commit f0e9452

Browse files
gjcairogjcairo
authored
Add GlitchesMonitor (#518)
Add a `GlitchesMonitor` that will terminate a connection if too many glitches are triggered. See https://github.com/HTTPWorkshop/workshop2024/blob/main/talks/1.%20Security/glitches.pdf for details.
1 parent bf0645a commit f0e9452

File tree

7 files changed

+272
-23
lines changed

7 files changed

+272
-23
lines changed

‎Sources/NIOHTTP2/ConnectionStateMachine/StateMachineResult.swift‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,11 @@ enum StateMachineResult {
3535

3636
/// An error that transitions the entire connection into a fatal error state. This should cause
3737
/// emission of GOAWAY frames.
38-
case connectionError(underlyingError: Error, type: HTTP2ErrorCode)
38+
case connectionError(
39+
underlyingError: Error,
40+
type: HTTP2ErrorCode,
41+
isMisbehavingPeer: Bool = false
42+
)
3943

4044
/// The frame itself was not valid, but it is also not an error. Drop the frame.
4145
case ignoreFrame

‎Sources/NIOHTTP2/GlitchesMonitor.swift‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the SwiftNIO open source project
4+
//
5+
// Copyright (c) 2025 Apple Inc. and the SwiftNIO project authors
6+
// Licensed under Apache License v2.0
7+
//
8+
// See LICENSE.txt for license information
9+
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
10+
//
11+
// SPDX-License-Identifier: Apache-2.0
12+
//
13+
//===----------------------------------------------------------------------===//
14+
15+
struct GlitchesMonitor {
16+
static var defaultMaximumGlitches: Int { 200 }
17+
private var stateMachine: GlitchesMonitorStateMachine
18+
19+
init(maximumGlitches: Int = GlitchesMonitor.defaultMaximumGlitches) {
20+
self.stateMachine = GlitchesMonitorStateMachine(maxGlitches: maximumGlitches)
21+
}
22+
23+
mutating func processStreamError() throws {
24+
switch self.stateMachine.recordEvent() {
25+
case .belowLimit:
26+
()
27+
28+
case .exceededLimit:
29+
throw NIOHTTP2Errors.excessiveNumberOfGlitches()
30+
}
31+
}
32+
}
33+
34+
extension GlitchesMonitor {
35+
private struct GlitchesMonitorStateMachine {
36+
enum State {
37+
case monitoring(numberOfGlitches: Int)
38+
case glitchesExceeded
39+
}
40+
41+
private var state: State
42+
private let maxGlitches: Int
43+
44+
init(maxGlitches: Int) {
45+
precondition(maxGlitches >= 0)
46+
self.state = .monitoring(numberOfGlitches: 0)
47+
self.maxGlitches = maxGlitches
48+
}
49+
50+
enum RecordEventAction {
51+
case belowLimit
52+
case exceededLimit
53+
}
54+
55+
mutating func recordEvent() -> RecordEventAction {
56+
switch self.state {
57+
case .monitoring(let numberOfGlitches):
58+
if numberOfGlitches < self.maxGlitches {
59+
self.state = .monitoring(numberOfGlitches: numberOfGlitches &+ 1)
60+
return .belowLimit
61+
} else {
62+
self.state = .glitchesExceeded
63+
return .exceededLimit
64+
}
65+
66+
case .glitchesExceeded:
67+
return .exceededLimit
68+
}
69+
}
70+
}
71+
}

‎Sources/NIOHTTP2/HTTP2ChannelHandler.swift‎

Lines changed: 83 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
7979
/// This object deploys heuristics to attempt to detect denial of service attacks.
8080
private var denialOfServiceValidator: DOSHeuristics<RealNIODeadlineClock>
8181

82+
private var glitchesMonitor: GlitchesMonitor
83+
8284
/// The mode this handler is operating in.
8385
private let mode: ParserMode
8486

@@ -236,6 +238,7 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
236238
maximumBufferedControlFrames: 10000,
237239
maximumSequentialContinuationFrames: NIOHTTP2Handler.defaultMaximumSequentialContinuationFrames,
238240
maximumRecentlyResetStreams: Self.defaultMaximumRecentlyResetFrames,
241+
maximumConnectionGlitches: GlitchesMonitor.defaultMaximumGlitches,
239242
maximumResetFrameCount: 200,
240243
resetFrameCounterWindow: .seconds(30),
241244
maximumStreamErrorCount: 200,
@@ -273,6 +276,7 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
273276
maximumBufferedControlFrames: maximumBufferedControlFrames,
274277
maximumSequentialContinuationFrames: NIOHTTP2Handler.defaultMaximumSequentialContinuationFrames,
275278
maximumRecentlyResetStreams: Self.defaultMaximumRecentlyResetFrames,
279+
maximumConnectionGlitches: GlitchesMonitor.defaultMaximumGlitches,
276280
maximumResetFrameCount: 200,
277281
resetFrameCounterWindow: .seconds(30),
278282
maximumStreamErrorCount: 200,
@@ -302,6 +306,7 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
302306
maximumBufferedControlFrames: connectionConfiguration.maximumBufferedControlFrames,
303307
maximumSequentialContinuationFrames: connectionConfiguration.maximumSequentialContinuationFrames,
304308
maximumRecentlyResetStreams: connectionConfiguration.maximumRecentlyResetStreams,
309+
maximumConnectionGlitches: connectionConfiguration.maximumConnectionGlitches,
305310
maximumResetFrameCount: streamConfiguration.streamResetFrameRateLimit.maximumCount,
306311
resetFrameCounterWindow: streamConfiguration.streamResetFrameRateLimit.windowLength,
307312
maximumStreamErrorCount: streamConfiguration.streamErrorRateLimit.maximumCount,
@@ -319,6 +324,7 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
319324
maximumBufferedControlFrames: Int,
320325
maximumSequentialContinuationFrames: Int,
321326
maximumRecentlyResetStreams: Int,
327+
maximumConnectionGlitches: Int,
322328
maximumResetFrameCount: Int,
323329
resetFrameCounterWindow: TimeAmount,
324330
maximumStreamErrorCount: Int,
@@ -348,6 +354,7 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
348354
self.tolerateImpossibleStateTransitionsInDebugMode = false
349355
self.inboundStreamMultiplexerState = .uninitializedLegacy
350356
self.maximumSequentialContinuationFrames = maximumSequentialContinuationFrames
357+
self.glitchesMonitor = GlitchesMonitor(maximumGlitches: maximumConnectionGlitches)
351358
}
352359

353360
/// Constructs a ``NIOHTTP2Handler``.
@@ -369,6 +376,7 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
369376
/// against this DoS vector we put an upper limit on this rate. Defaults to 200.
370377
/// - resetFrameCounterWindow: Controls the sliding window used to enforce the maximum permitted reset frames rate. Too many may exhaust CPU resources. To protect
371378
/// against this DoS vector we put an upper limit on this rate. 30 seconds.
379+
/// - maximumConnectionGlitches: Controls the maximum number of stream errors that can happen on a connection before the connection is reset. Defaults to 200.
372380
internal init(
373381
mode: ParserMode,
374382
initialSettings: HTTP2Settings = nioDefaultSettings,
@@ -382,7 +390,8 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
382390
maximumResetFrameCount: Int = 200,
383391
resetFrameCounterWindow: TimeAmount = .seconds(30),
384392
maximumStreamErrorCount: Int = 200,
385-
streamErrorCounterWindow: TimeAmount = .seconds(30)
393+
streamErrorCounterWindow: TimeAmount = .seconds(30),
394+
maximumConnectionGlitches: Int = GlitchesMonitor.defaultMaximumGlitches
386395
) {
387396
self.stateMachine = HTTP2ConnectionStateMachine(
388397
role: .init(mode),
@@ -408,6 +417,7 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
408417
self.tolerateImpossibleStateTransitionsInDebugMode = tolerateImpossibleStateTransitionsInDebugMode
409418
self.inboundStreamMultiplexerState = .uninitializedLegacy
410419
self.maximumSequentialContinuationFrames = maximumSequentialContinuationFrames
420+
self.glitchesMonitor = GlitchesMonitor(maximumGlitches: maximumConnectionGlitches)
411421
}
412422

413423
public func handlerAdded(context: ChannelHandlerContext) {
@@ -600,32 +610,41 @@ extension NIOHTTP2Handler {
600610
self.inboundConnectionErrorTriggered(
601611
context: context,
602612
underlyingError: NIOHTTP2Errors.unableToParseFrame(),
603-
reason: code
613+
reason: code,
614+
isMisbehavingPeer: false
604615
)
605616
return nil
606617
} catch is NIOHTTP2Errors.BadClientMagic {
607618
self.inboundConnectionErrorTriggered(
608619
context: context,
609620
underlyingError: NIOHTTP2Errors.badClientMagic(),
610-
reason: .protocolError
621+
reason: .protocolError,
622+
isMisbehavingPeer: false
611623
)
612624
return nil
613625
} catch is NIOHTTP2Errors.ExcessivelyLargeHeaderBlock {
614626
self.inboundConnectionErrorTriggered(
615627
context: context,
616628
underlyingError: NIOHTTP2Errors.excessivelyLargeHeaderBlock(),
617-
reason: .protocolError
629+
reason: .protocolError,
630+
isMisbehavingPeer: false
618631
)
619632
return nil
620633
} catch is NIOHTTP2Errors.ExcessiveContinuationFrames {
621634
self.inboundConnectionErrorTriggered(
622635
context: context,
623636
underlyingError: NIOHTTP2Errors.excessiveContinuationFrames(),
624-
reason: .enhanceYourCalm
637+
reason: .enhanceYourCalm,
638+
isMisbehavingPeer: false
625639
)
626640
return nil
627641
} catch {
628-
self.inboundConnectionErrorTriggered(context: context, underlyingError: error, reason: .internalError)
642+
self.inboundConnectionErrorTriggered(
643+
context: context,
644+
underlyingError: error,
645+
reason: .internalError,
646+
isMisbehavingPeer: false
647+
)
629648
return nil
630649
}
631650
}
@@ -733,6 +752,7 @@ extension NIOHTTP2Handler {
733752
}
734753

735754
self.processDoSRisk(frame, result: &result)
755+
self.processGlitches(result: &result)
736756
self.processStateChange(result.effect)
737757

738758
let returnValue: FrameProcessResult
@@ -744,9 +764,14 @@ extension NIOHTTP2Handler {
744764
case .ignoreFrame:
745765
// Frame is good but no action needs to be taken.
746766
returnValue = .continue
747-
case .connectionError(let underlyingError, let errorCode):
767+
case .connectionError(let underlyingError, let errorCode, let isMisbehavingPeer):
748768
// We should stop parsing on received connection errors, the connection is going away anyway.
749-
self.inboundConnectionErrorTriggered(context: context, underlyingError: underlyingError, reason: errorCode)
769+
self.inboundConnectionErrorTriggered(
770+
context: context,
771+
underlyingError: underlyingError,
772+
reason: errorCode,
773+
isMisbehavingPeer: isMisbehavingPeer
774+
)
750775
returnValue = .stop
751776
case .streamError(let streamID, let underlyingError, let errorCode):
752777
// We can continue parsing on stream errors in most cases, the frame is just ignored.
@@ -770,15 +795,20 @@ extension NIOHTTP2Handler {
770795
private func inboundConnectionErrorTriggered(
771796
context: ChannelHandlerContext,
772797
underlyingError: Error,
773-
reason: HTTP2ErrorCode
798+
reason: HTTP2ErrorCode,
799+
isMisbehavingPeer: Bool
774800
) {
775801
// A connection error brings the entire connection down. We attempt to write a GOAWAY frame, and then report this
776802
// error. It's possible that we'll be unable to write the GOAWAY frame, but that also just logs the error.
777803
// Because we don't know what data the user handled before we got this, we propose that they may have seen all of it.
778804
// The user may choose to fire a more specific error if they wish.
805+
806+
// If the peer is misbehaving, set the stream ID to the minimum allowed value (0).
807+
// This will cause all open streams for this connection to be immediately terminated.
808+
let streamID = isMisbehavingPeer ? HTTP2StreamID(0) : .maxID
779809
let goAwayFrame = HTTP2Frame(
780810
streamID: .rootStream,
781-
payload: .goAway(lastStreamID: .maxID, errorCode: reason, opaqueData: nil)
811+
payload: .goAway(lastStreamID: streamID, errorCode: reason, opaqueData: nil)
782812
)
783813
self.writeUnbufferedFrame(context: context, frame: goAwayFrame)
784814
self.flushIfNecessary(context: context)
@@ -818,7 +848,29 @@ extension NIOHTTP2Handler {
818848
()
819849
}
820850
} catch {
821-
result.result = StateMachineResult.connectionError(underlyingError: error, type: .enhanceYourCalm)
851+
result.result = StateMachineResult.connectionError(
852+
underlyingError: error,
853+
type: .enhanceYourCalm,
854+
isMisbehavingPeer: true
855+
)
856+
result.effect = nil
857+
}
858+
}
859+
860+
private func processGlitches(result: inout StateMachineResultWithEffect) {
861+
do {
862+
switch result.result {
863+
case .streamError:
864+
try self.glitchesMonitor.processStreamError()
865+
case .succeed, .ignoreFrame, .connectionError:
866+
()
867+
}
868+
} catch {
869+
result.result = .connectionError(
870+
underlyingError: error,
871+
type: .enhanceYourCalm,
872+
isMisbehavingPeer: true
873+
)
822874
result.effect = nil
823875
}
824876
}
@@ -894,7 +946,12 @@ extension NIOHTTP2Handler {
894946
}
895947
} catch let error where error is NIOHTTP2Errors.ExcessiveOutboundFrameBuffering {
896948
self.inboundStreamMultiplexer?.processedFrame(frame)
897-
self.inboundConnectionErrorTriggered(context: context, underlyingError: error, reason: .enhanceYourCalm)
949+
self.inboundConnectionErrorTriggered(
950+
context: context,
951+
underlyingError: error,
952+
reason: .enhanceYourCalm,
953+
isMisbehavingPeer: false
954+
)
898955
} catch {
899956
self.inboundStreamMultiplexer?.processedFrame(frame)
900957
promise?.fail(error)
@@ -968,7 +1025,7 @@ extension NIOHTTP2Handler {
9681025
switch result.result {
9691026
case .ignoreFrame:
9701027
preconditionFailure("Cannot be asked to ignore outbound frames.")
971-
case .connectionError(let underlyingError, _):
1028+
case .connectionError(let underlyingError, _, _):
9721029
self.outboundConnectionErrorTriggered(context: context, promise: promise, underlyingError: underlyingError)
9731030
return
9741031
case .streamError(let streamID, let underlyingError, _):
@@ -1330,6 +1387,7 @@ extension NIOHTTP2Handler {
13301387
maximumBufferedControlFrames: connectionConfiguration.maximumBufferedControlFrames,
13311388
maximumSequentialContinuationFrames: connectionConfiguration.maximumSequentialContinuationFrames,
13321389
maximumRecentlyResetStreams: connectionConfiguration.maximumRecentlyResetStreams,
1390+
maximumConnectionGlitches: connectionConfiguration.maximumConnectionGlitches,
13331391
maximumResetFrameCount: streamConfiguration.streamResetFrameRateLimit.maximumCount,
13341392
resetFrameCounterWindow: streamConfiguration.streamResetFrameRateLimit.windowLength,
13351393
maximumStreamErrorCount: streamConfiguration.streamErrorRateLimit.maximumCount,
@@ -1362,6 +1420,7 @@ extension NIOHTTP2Handler {
13621420
maximumBufferedControlFrames: connectionConfiguration.maximumBufferedControlFrames,
13631421
maximumSequentialContinuationFrames: connectionConfiguration.maximumSequentialContinuationFrames,
13641422
maximumRecentlyResetStreams: connectionConfiguration.maximumRecentlyResetStreams,
1423+
maximumConnectionGlitches: connectionConfiguration.maximumConnectionGlitches,
13651424
maximumResetFrameCount: streamConfiguration.streamResetFrameRateLimit.maximumCount,
13661425
resetFrameCounterWindow: streamConfiguration.streamResetFrameRateLimit.windowLength,
13671426
maximumStreamErrorCount: streamConfiguration.streamErrorRateLimit.maximumCount,
@@ -1386,6 +1445,17 @@ extension NIOHTTP2Handler {
13861445
public var maximumBufferedControlFrames: Int = 10000
13871446
public var maximumSequentialContinuationFrames: Int = NIOHTTP2Handler.defaultMaximumSequentialContinuationFrames
13881447
public var maximumRecentlyResetStreams: Int = NIOHTTP2Handler.defaultMaximumRecentlyResetFrames
1448+
1449+
/// The maximum number of glitches that are allowed on a connection before it's forcefully closed.
1450+
///
1451+
/// A glitch is defined as some suspicious event on a connection, i.e., similar to a DoS attack.
1452+
/// A running count of the number of glitches occurring on each connection will be kept.
1453+
/// When the number of glitches reaches this threshold, the connection will be closed.
1454+
///
1455+
/// For more information, see the relevant presentation of the 2024 HTTP Workshop:
1456+
/// https://github.com/HTTPWorkshop/workshop2024/blob/main/talks/1.%20Security/glitches.pdf
1457+
public var maximumConnectionGlitches: Int = GlitchesMonitor.defaultMaximumGlitches
1458+
13891459
public init() {}
13901460
}
13911461

‎Sources/NIOHTTP2/HTTP2Error.swift‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,18 @@ public enum NIOHTTP2Errors {
487487
ExcessiveEmptyDataFrames(file: file, line: line)
488488
}
489489

490+
/// Creates a ``ExcessiveNumberOfGlitches`` error with appropriate source context.
491+
///
492+
/// - Parameters:
493+
/// - file: Source file of the caller.
494+
/// - line: Source line number of the caller.
495+
public static func excessiveNumberOfGlitches(
496+
file: String = #fileID,
497+
line: UInt = #line
498+
) -> ExcessiveNumberOfGlitches {
499+
ExcessiveNumberOfGlitches(file: file, line: line)
500+
}
501+
490502
/// Creates a ``ExcessivelyLargeHeaderBlock`` error with appropriate source context.
491503
///
492504
/// - Parameters:
@@ -1832,6 +1844,31 @@ public enum NIOHTTP2Errors {
18321844
}
18331845
}
18341846

1847+
/// The remote peer has triggered too many glitches on this connection.
1848+
public struct ExcessiveNumberOfGlitches: NIOHTTP2Error {
1849+
private let file: String
1850+
private let line: UInt
1851+
1852+
/// The location where the error was thrown.
1853+
public var location: String {
1854+
_location(file: self.file, line: self.line)
1855+
}
1856+
1857+
@available(*, deprecated, renamed: "excessiveNumberOfGlitches")
1858+
public init() {
1859+
self.init(file: #fileID, line: #line)
1860+
}
1861+
1862+
fileprivate init(file: String, line: UInt) {
1863+
self.file = file
1864+
self.line = line
1865+
}
1866+
1867+
public static func == (lhs: ExcessiveNumberOfGlitches, rhs: ExcessiveNumberOfGlitches) -> Bool {
1868+
true
1869+
}
1870+
}
1871+
18351872
/// The remote peer has sent a header block so large that ``NIOHTTP2`` refuses to buffer any more data than that.
18361873
public struct ExcessivelyLargeHeaderBlock: NIOHTTP2Error {
18371874
private let file: String

0 commit comments

Comments
 (0)