Skip to content

Commit e54568c

Browse files
sagigrimbergsagigrimberg
committed
mountstats/linux: fix multiple transports statistics labeling
When mounting nfs with nconnect, mountstats will show multiple transports. Currently the mountstats exporter fails to create metrics from this information due to repeated entries. Fix this by adding transport metrics a "transport" label enumerating their order in the mountstats output. Also, an unrelated hunk that `end-to-end.sh u` applied for me Signed-off-by: Sagi Grimberg <[email protected]>
1 parent 38d32a3 commit e54568c

File tree

2 files changed

+49
-45
lines changed

2 files changed

+49
-45
lines changed

‎collector/fixtures/e2e-output.txt

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# HELP go_gc_duration_seconds A summary of the wall-time pause (stop-the-world) duration in garbage collection cycles.
22
# TYPE go_gc_duration_seconds summary
3-
# HELP go_gc_gogc_percent Heap size target percentage configured by the user, otherwise 100. This value is set by the GOGC environment variable, and the runtime/debug.SetGCPercent function. Sourced from /gc/gogc:percent.
3+
# HELP go_gc_gogc_percent Heap size target percentage configured by the user, otherwise 100. This value is set by the GOGC environment variable, and the runtime/debug.SetGCPercent function. Sourced from /gc/gogc:percent
44
# TYPE go_gc_gogc_percent gauge
5-
# HELP go_gc_gomemlimit_bytes Go runtime memory limit configured by the user, otherwise math.MaxInt64. This value is set by the GOMEMLIMIT environment variable, and the runtime/debug.SetMemoryLimit function. Sourced from /gc/gomemlimit:bytes.
5+
# HELP go_gc_gomemlimit_bytes Go runtime memory limit configured by the user, otherwise math.MaxInt64. This value is set by the GOMEMLIMIT environment variable, and the runtime/debug.SetMemoryLimit function. Sourced from /gc/gomemlimit:bytes
66
# TYPE go_gc_gomemlimit_bytes gauge
77
# HELP go_goroutines Number of goroutines that currently exist.
88
# TYPE go_goroutines gauge
@@ -52,7 +52,7 @@
5252
# TYPE go_memstats_stack_sys_bytes gauge
5353
# HELP go_memstats_sys_bytes Number of bytes obtained from system. Equals to /memory/classes/total:byte.
5454
# TYPE go_memstats_sys_bytes gauge
55-
# HELP go_sched_gomaxprocs_threads The current runtime.GOMAXPROCS setting, or the number of operating system threads that can execute user-level Go code simultaneously. Sourced from /sched/gomaxprocs:threads.
55+
# HELP go_sched_gomaxprocs_threads The current runtime.GOMAXPROCS setting, or the number of operating system threads that can execute user-level Go code simultaneously. Sourced from /sched/gomaxprocs:threads
5656
# TYPE go_sched_gomaxprocs_threads gauge
5757
# HELP go_threads Number of OS threads created.
5858
# TYPE go_threads gauge
@@ -2289,44 +2289,44 @@ node_mountstats_nfs_total_write_bytes_total{export="192.168.1.1:/srv/test",mount
22892289
node_mountstats_nfs_total_write_bytes_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp"} 0
22902290
# HELP node_mountstats_nfs_transport_backlog_queue_total Total number of items added to the RPC backlog queue.
22912291
# TYPE node_mountstats_nfs_transport_backlog_queue_total counter
2292-
node_mountstats_nfs_transport_backlog_queue_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp"} 0
2293-
node_mountstats_nfs_transport_backlog_queue_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp"} 0
2292+
node_mountstats_nfs_transport_backlog_queue_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp",transport="0"} 0
2293+
node_mountstats_nfs_transport_backlog_queue_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp",transport="0"} 0
22942294
# HELP node_mountstats_nfs_transport_bad_transaction_ids_total Number of times the NFS server sent a response with a transaction ID unknown to this client.
22952295
# TYPE node_mountstats_nfs_transport_bad_transaction_ids_total counter
2296-
node_mountstats_nfs_transport_bad_transaction_ids_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp"} 0
2297-
node_mountstats_nfs_transport_bad_transaction_ids_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp"} 0
2296+
node_mountstats_nfs_transport_bad_transaction_ids_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp",transport="0"} 0
2297+
node_mountstats_nfs_transport_bad_transaction_ids_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp",transport="0"} 0
22982298
# HELP node_mountstats_nfs_transport_bind_total Number of times the client has had to establish a connection from scratch to the NFS server.
22992299
# TYPE node_mountstats_nfs_transport_bind_total counter
2300-
node_mountstats_nfs_transport_bind_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp"} 0
2301-
node_mountstats_nfs_transport_bind_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp"} 0
2300+
node_mountstats_nfs_transport_bind_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp",transport="0"} 0
2301+
node_mountstats_nfs_transport_bind_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp",transport="0"} 0
23022302
# HELP node_mountstats_nfs_transport_connect_total Number of times the client has made a TCP connection to the NFS server.
23032303
# TYPE node_mountstats_nfs_transport_connect_total counter
2304-
node_mountstats_nfs_transport_connect_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp"} 1
2305-
node_mountstats_nfs_transport_connect_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp"} 0
2304+
node_mountstats_nfs_transport_connect_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp",transport="0"} 1
2305+
node_mountstats_nfs_transport_connect_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp",transport="0"} 0
23062306
# HELP node_mountstats_nfs_transport_idle_time_seconds Duration since the NFS mount last saw any RPC traffic, in seconds.
23072307
# TYPE node_mountstats_nfs_transport_idle_time_seconds gauge
2308-
node_mountstats_nfs_transport_idle_time_seconds{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp"} 11
2309-
node_mountstats_nfs_transport_idle_time_seconds{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp"} 0
2308+
node_mountstats_nfs_transport_idle_time_seconds{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp",transport="0"} 11
2309+
node_mountstats_nfs_transport_idle_time_seconds{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp",transport="0"} 0
23102310
# HELP node_mountstats_nfs_transport_maximum_rpc_slots Maximum number of simultaneously active RPC requests ever used.
23112311
# TYPE node_mountstats_nfs_transport_maximum_rpc_slots gauge
2312-
node_mountstats_nfs_transport_maximum_rpc_slots{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp"} 24
2313-
node_mountstats_nfs_transport_maximum_rpc_slots{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp"} 24
2312+
node_mountstats_nfs_transport_maximum_rpc_slots{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp",transport="0"} 24
2313+
node_mountstats_nfs_transport_maximum_rpc_slots{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp",transport="0"} 24
23142314
# HELP node_mountstats_nfs_transport_pending_queue_total Total number of items added to the RPC transmission pending queue.
23152315
# TYPE node_mountstats_nfs_transport_pending_queue_total counter
2316-
node_mountstats_nfs_transport_pending_queue_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp"} 5726
2317-
node_mountstats_nfs_transport_pending_queue_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp"} 5726
2316+
node_mountstats_nfs_transport_pending_queue_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp",transport="0"} 5726
2317+
node_mountstats_nfs_transport_pending_queue_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp",transport="0"} 5726
23182318
# HELP node_mountstats_nfs_transport_receives_total Number of RPC responses for this mount received from the NFS server.
23192319
# TYPE node_mountstats_nfs_transport_receives_total counter
2320-
node_mountstats_nfs_transport_receives_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp"} 6428
2321-
node_mountstats_nfs_transport_receives_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp"} 6428
2320+
node_mountstats_nfs_transport_receives_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp",transport="0"} 6428
2321+
node_mountstats_nfs_transport_receives_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp",transport="0"} 6428
23222322
# HELP node_mountstats_nfs_transport_sending_queue_total Total number of items added to the RPC transmission sending queue.
23232323
# TYPE node_mountstats_nfs_transport_sending_queue_total counter
2324-
node_mountstats_nfs_transport_sending_queue_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp"} 26
2325-
node_mountstats_nfs_transport_sending_queue_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp"} 26
2324+
node_mountstats_nfs_transport_sending_queue_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp",transport="0"} 26
2325+
node_mountstats_nfs_transport_sending_queue_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp",transport="0"} 26
23262326
# HELP node_mountstats_nfs_transport_sends_total Number of RPC requests for this mount sent to the NFS server.
23272327
# TYPE node_mountstats_nfs_transport_sends_total counter
2328-
node_mountstats_nfs_transport_sends_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp"} 6428
2329-
node_mountstats_nfs_transport_sends_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp"} 6428
2328+
node_mountstats_nfs_transport_sends_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp",transport="0"} 6428
2329+
node_mountstats_nfs_transport_sends_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="udp",transport="0"} 6428
23302330
# HELP node_mountstats_nfs_write_bytes_total Number of bytes written using the write() syscall.
23312331
# TYPE node_mountstats_nfs_write_bytes_total counter
23322332
node_mountstats_nfs_write_bytes_total{export="192.168.1.1:/srv/test",mountaddr="192.168.1.1",protocol="tcp"} 0

‎collector/mountstats_linux.go

Lines changed: 26 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package collector
1919
import (
2020
"fmt"
2121
"log/slog"
22+
"strconv"
2223

2324
"github.com/prometheus/client_golang/prometheus"
2425
"github.com/prometheus/procfs"
@@ -127,8 +128,9 @@ func NewMountStatsCollector(logger *slog.Logger) (Collector, error) {
127128
)
128129

129130
var (
130-
labels = []string{"export", "protocol", "mountaddr"}
131-
opLabels = []string{"export", "protocol", "mountaddr", "operation"}
131+
labels = []string{"export", "protocol", "mountaddr"}
132+
opLabels = []string{"export", "protocol", "mountaddr", "operation"}
133+
translabels = []string{"export", "protocol", "mountaddr", "transport"}
132134
)
133135

134136
return &mountStatsCollector{
@@ -198,70 +200,70 @@ func NewMountStatsCollector(logger *slog.Logger) (Collector, error) {
198200
NFSTransportBindTotal: prometheus.NewDesc(
199201
prometheus.BuildFQName(namespace, subsystem, "transport_bind_total"),
200202
"Number of times the client has had to establish a connection from scratch to the NFS server.",
201-
labels,
203+
translabels,
202204
nil,
203205
),
204206

205207
NFSTransportConnectTotal: prometheus.NewDesc(
206208
prometheus.BuildFQName(namespace, subsystem, "transport_connect_total"),
207209
"Number of times the client has made a TCP connection to the NFS server.",
208-
labels,
210+
translabels,
209211
nil,
210212
),
211213

212214
NFSTransportIdleTimeSeconds: prometheus.NewDesc(
213215
prometheus.BuildFQName(namespace, subsystem, "transport_idle_time_seconds"),
214216
"Duration since the NFS mount last saw any RPC traffic, in seconds.",
215-
labels,
217+
translabels,
216218
nil,
217219
),
218220

219221
NFSTransportSendsTotal: prometheus.NewDesc(
220222
prometheus.BuildFQName(namespace, subsystem, "transport_sends_total"),
221223
"Number of RPC requests for this mount sent to the NFS server.",
222-
labels,
224+
translabels,
223225
nil,
224226
),
225227

226228
NFSTransportReceivesTotal: prometheus.NewDesc(
227229
prometheus.BuildFQName(namespace, subsystem, "transport_receives_total"),
228230
"Number of RPC responses for this mount received from the NFS server.",
229-
labels,
231+
translabels,
230232
nil,
231233
),
232234

233235
NFSTransportBadTransactionIDsTotal: prometheus.NewDesc(
234236
prometheus.BuildFQName(namespace, subsystem, "transport_bad_transaction_ids_total"),
235237
"Number of times the NFS server sent a response with a transaction ID unknown to this client.",
236-
labels,
238+
translabels,
237239
nil,
238240
),
239241

240242
NFSTransportBacklogQueueTotal: prometheus.NewDesc(
241243
prometheus.BuildFQName(namespace, subsystem, "transport_backlog_queue_total"),
242244
"Total number of items added to the RPC backlog queue.",
243-
labels,
245+
translabels,
244246
nil,
245247
),
246248

247249
NFSTransportMaximumRPCSlots: prometheus.NewDesc(
248250
prometheus.BuildFQName(namespace, subsystem, "transport_maximum_rpc_slots"),
249251
"Maximum number of simultaneously active RPC requests ever used.",
250-
labels,
252+
translabels,
251253
nil,
252254
),
253255

254256
NFSTransportSendingQueueTotal: prometheus.NewDesc(
255257
prometheus.BuildFQName(namespace, subsystem, "transport_sending_queue_total"),
256258
"Total number of items added to the RPC transmission sending queue.",
257-
labels,
259+
translabels,
258260
nil,
259261
),
260262

261263
NFSTransportPendingQueueTotal: prometheus.NewDesc(
262264
prometheus.BuildFQName(namespace, subsystem, "transport_pending_queue_total"),
263265
"Total number of items added to the RPC transmission pending queue.",
264-
labels,
266+
translabels,
265267
nil,
266268
),
267269

@@ -618,74 +620,76 @@ func (c *mountStatsCollector) updateNFSStats(ch chan<- prometheus.Metric, s *pro
618620
)
619621

620622
for i := range s.Transport {
623+
translabelValues := []string{export, protocol, mountAddress, strconv.Itoa(i)}
624+
621625
ch <- prometheus.MustNewConstMetric(
622626
c.NFSTransportBindTotal,
623627
prometheus.CounterValue,
624628
float64(s.Transport[i].Bind),
625-
labelValues...,
629+
translabelValues...,
626630
)
627631

628632
ch <- prometheus.MustNewConstMetric(
629633
c.NFSTransportConnectTotal,
630634
prometheus.CounterValue,
631635
float64(s.Transport[i].Connect),
632-
labelValues...,
636+
translabelValues...,
633637
)
634638

635639
ch <- prometheus.MustNewConstMetric(
636640
c.NFSTransportIdleTimeSeconds,
637641
prometheus.GaugeValue,
638642
float64(s.Transport[i].IdleTimeSeconds%float64Mantissa),
639-
labelValues...,
643+
translabelValues...,
640644
)
641645

642646
ch <- prometheus.MustNewConstMetric(
643647
c.NFSTransportSendsTotal,
644648
prometheus.CounterValue,
645649
float64(s.Transport[i].Sends),
646-
labelValues...,
650+
translabelValues...,
647651
)
648652

649653
ch <- prometheus.MustNewConstMetric(
650654
c.NFSTransportReceivesTotal,
651655
prometheus.CounterValue,
652656
float64(s.Transport[i].Receives),
653-
labelValues...,
657+
translabelValues...,
654658
)
655659

656660
ch <- prometheus.MustNewConstMetric(
657661
c.NFSTransportBadTransactionIDsTotal,
658662
prometheus.CounterValue,
659663
float64(s.Transport[i].BadTransactionIDs),
660-
labelValues...,
664+
translabelValues...,
661665
)
662666

663667
ch <- prometheus.MustNewConstMetric(
664668
c.NFSTransportBacklogQueueTotal,
665669
prometheus.CounterValue,
666670
float64(s.Transport[i].CumulativeBacklog),
667-
labelValues...,
671+
translabelValues...,
668672
)
669673

670674
ch <- prometheus.MustNewConstMetric(
671675
c.NFSTransportMaximumRPCSlots,
672676
prometheus.GaugeValue,
673677
float64(s.Transport[i].MaximumRPCSlotsUsed),
674-
labelValues...,
678+
translabelValues...,
675679
)
676680

677681
ch <- prometheus.MustNewConstMetric(
678682
c.NFSTransportSendingQueueTotal,
679683
prometheus.CounterValue,
680684
float64(s.Transport[i].CumulativeSendingQueue),
681-
labelValues...,
685+
translabelValues...,
682686
)
683687

684688
ch <- prometheus.MustNewConstMetric(
685689
c.NFSTransportPendingQueueTotal,
686690
prometheus.CounterValue,
687691
float64(s.Transport[i].CumulativePendingQueue),
688-
labelValues...,
692+
translabelValues...,
689693
)
690694
}
691695

0 commit comments

Comments
 (0)