ON-16037: added functions to read stats for interfaces

Author: Richard Drinkwater

src/include/zf/zf.h

@@ -25,6 +25,7 @@
 #include <zf/types.h>
 #include <zf/zf_alts.h>
 #include <zf/zf_ds.h>
+#include <zf/zf_stats.h>
 
 #undef __IN_ZF_TOP_H__
 #endif  /* __ZF_TOP_H__ */

src/include/zf/zf_stats.h

@@ -0,0 +1,96 @@
+/* SPDX-License-Identifier: MIT */
+/* SPDX-FileCopyrightText: (c) Advanced Micro Devices, Inc. */
+/**************************************************************************\
+*//*! \file
+**  \brief  TCPDirect stats API
+*//*
+\**************************************************************************/
+
+#ifndef __ZF_STATS_H__
+#define __ZF_STATS_H__
+
+#ifndef __IN_ZF_TOP_H__
+# error "Please include zf.h to use TCPDirect."
+#endif
+
+
+/*! \brief Layout for a field of statistics */
+typedef struct {
+  /** Name of statistics field */
+  char* evsfl_name;
+  /** Offset of statistics field, in bytes */
+  int   evsfl_offset;
+  /** Size of statistics field, in bytes */
+  int   evsfl_size;
+} zf_stats_field_layout;
+
+/*! \brief Layout for statistics */
+typedef struct {
+  /** Size of data for statistics */
+  int                      evsl_data_size;
+  /** Number of fields of statistics */
+  int                      evsl_fields_num;
+  /** Array of fields of statistics */
+  zf_stats_field_layout    evsl_fields[];
+} zf_stats_layout;
+
+/*! \brief  */
+typedef struct {
+  /** Number of underlying interfaces present in stack */
+  int               num_intfs;
+  /** Size of memory needed to query the stats in bytes */
+  int               total_data_size;
+  /** Array of layouts, one per interface */
+  zf_stats_layout** layout;
+} zf_layout_collection;
+
+/*! \brief Allocate and retrieve layout for available statistics
+**
+** \param stack      The stack to query.
+** \param collection Pointer to an zf_stats_collection, that is allocated and
+**                   updated on return with the layout for available
+**                   statistics. This must be released when no longer needed
+**                   using zf_stats_free_query_layout().
+**
+** \return Zero on success or negative error code.
+**
+** Retrieve layout for available statistics.
+*/
+ZF_LIBENTRY int zf_stats_alloc_layout_collection(struct zf_stack* stack,
+                                            zf_layout_collection** collection);
+
+/*! \brief Release the layout allocated by zf_stats_alloc_query_layout().
+**
+** \param collection Pointer to an zf_stats_collection that was allocated using
+**                   zf_stats_alloc_layout_collection().
+*/
+ZF_LIBENTRY void zf_stats_free_layout_collection(
+                                            zf_layout_collection* collection);
+
+/*! \brief Retrieve a set of statistic values
+**
+** \param stack      The stack to query.
+** \param data       Pointer to a buffer, into which the statistics are
+**                   retrieved.
+**                   The size of this buffer must be equal to the value of
+**                   total_data_size in the zf_layout_collection structure.
+** \param collection Pointer to a zf_layout_collection, that was generated for this
+**                   stack by zf_stats_query_layout.
+** \param do_reset   True to reset the statistics after retrieving them.
+**
+** \return zero or a negative error code.
+**
+** Retrieve a set of statistic values.
+**
+** If do_reset is true, the statistics are reset after reading.
+**
+** \note This requires full feature firmware. If used with low-latency
+** firmware, no error is given, and the statistics are invalid (typically
+** all zeroes).
+*/
+ZF_LIBENTRY int zf_stats_query(struct zf_stack* stack, void* data,
+                               zf_layout_collection* collection,
+                               int do_reset);
+
+#endif /* __ZF_STATS_H__ */
+/** @} */

src/lib/zf/Makefile.inc

@@ -4,7 +4,7 @@
 PUBLIC_LIB_SRCS := zf.c log.c attr.c stack.c pool.c udp_rx.c rx.c udp_tx.c \
 	    muxer.c tcp.c tcp_in.c tcp_out.c tcp_core.c rx_table.c zf_alts.c \
 	    lazy_alloc.c x86.c timers.c cplane.c zf_tcp.c zf_stackdump.c \
-	    dshm.c zf_alt_buffer_model.c zf_ds.c bond.c tx_warm.c
+	    dshm.c zf_alt_buffer_model.c zf_ds.c bond.c tx_warm.c zf_stats.c
 
 # Source files that are not distributed, living in private/.
 PRIVATE_LIB_SRCS := stack_fast.c stack_alloc.c tcp_fast.c reactor.c

src/lib/zf/zf_stats.c

@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: MIT */
+/* SPDX-FileCopyrightText: (c) Advanced Micro Devices, Inc. */
+#include <zf_internal/zf_stack.h>
+
+/* ensure ef_vi/zf structures are the same size in case
+ * of adding/removing elements
+ */
+static_assert( sizeof(zf_stats_field_layout) == sizeof(zf_stats_field_layout),
+               "ef_vi_stats_field_layout and zf_stats_field_layout size mismatch");
+static_assert( sizeof(ef_vi_stats_layout) == sizeof(zf_stats_layout),
+               "ef_vi_stats_layout and zf_stats_layout size mismatch");
+
+static_assert( offsetof(zf_stats_field_layout, evsfl_name) == offsetof(ef_vi_stats_field_layout, evsfl_name),
+               "zf_stats_field_layout member evsfl_name does not match ef_vi_stats_field_layout");
+static_assert( offsetof(zf_stats_field_layout, evsfl_offset) == offsetof(ef_vi_stats_field_layout, evsfl_offset),
+               "zf_stats_field_layout member evsfl_name does not match ef_vi_stats_field_layout");
+static_assert( offsetof(zf_stats_field_layout, evsfl_size) == offsetof(ef_vi_stats_field_layout, evsfl_size),
+               "zf_stats_field_layout member evsfl_name does not match ef_vi_stats_field_layout");
+
+static_assert( offsetof(zf_stats_layout, evsl_data_size) == offsetof(ef_vi_stats_layout, evsl_data_size),
+               "zf_stats_field_layout member evsfl_name does not match ef_vi_stats_field_layout");
+static_assert( offsetof(zf_stats_layout, evsl_fields_num) == offsetof(ef_vi_stats_layout, evsl_fields_num),
+               "zf_stats_field_layout member evsfl_name does not match ef_vi_stats_field_layout");
+static_assert( offsetof(zf_stats_layout, evsl_fields) == offsetof(ef_vi_stats_layout, evsl_fields),
+               "zf_stats_field_layout member evsfl_name does not match ef_vi_stats_field_layout");
+
+
+int
+zf_stats_alloc_layout_collection(struct zf_stack* stack,
+                                 zf_layout_collection** collection)
+{
+  int i, malloc_sz, rc = 0;
+  int num_nics = stack->nics_n;
+  zf_layout_collection* c;
+
+  c = (zf_layout_collection*)malloc(sizeof(zf_layout_collection));
+  if( c == NULL )
+    return -ENOMEM;
+
+  malloc_sz = num_nics * sizeof(zf_stats_layout *);
+  c->layout = (zf_stats_layout**)malloc(malloc_sz);
+  if( c->layout == NULL ) {
+    free(c);
+    return -ENOMEM;
+  }
+  c->num_intfs = num_nics;
+  c->total_data_size = 0;
+
+  for( i = 0; i < num_nics && rc == 0; i++ ) {
+    ef_vi* vi = &stack->nic[i].vi;
+    rc = ef_vi_stats_query_layout(vi,
+        (const ef_vi_stats_layout** const) &c->layout[i]);
+    c->total_data_size += c->layout[i]->evsl_data_size;
+  }
+  
+
+  if( rc < 0 ) {
+    zf_stats_free_layout_collection(c);
+    return rc;
+  }
+  *collection = c;
+  return 0;
+}
+
+
+void
+zf_stats_free_layout_collection(zf_layout_collection* collection)
+{
+  free(collection->layout);
+  free(collection);
+}
+
+
+int
+zf_stats_query(struct zf_stack* stack, void* data,
+               zf_layout_collection* collection, int do_reset)
+{
+  int i, rc = 0;
+  int num_intfs = stack->nics_n;
+  char* pData = (char*)data;
+
+  assert(num_intfs == collection->num_intfs);
+
+  for( i = 0; i < num_intfs && rc == 0; i++ ) {
+    ef_vi* vi = &stack->nic[i].vi;
+    rc = ef_vi_stats_query(vi, vi->dh, (void*)pData, do_reset);
+    pData += collection->layout[i]->evsl_data_size;
+  }
+
+  return rc;
+}

src/tests/zf_apps/zfsink.c

@@ -34,6 +34,7 @@ struct resources {
 
 static bool cfg_quiet = false;
 static bool cfg_rx_timestamping = false;
+static bool cfg_intf_stats = false;
 static struct resources res;
 
 /* Mutex to protect printing from different threads */
@@ -52,6 +53,7 @@ static void usage_msg(FILE* f)
   fprintf(f, "  -r       Enable rx timestamping\n");
   fprintf(f, "  -q       Quiet -- do not emit progress messages\n");
   fprintf(f, "  -p       Print zf attributes after stack startup\n");
+  fprintf(f, "  -s       Print interfaces drop stats\n");
 }
 
 
@@ -61,18 +63,6 @@ static void usage_err(void)
   exit(1);
 }
 
-
-static void vlog(const char* fmt, ...)
-{
-  va_list args;
-  va_start(args, fmt);
-  pthread_mutex_lock(&printf_mutex);
-  vprintf(fmt, args);
-  pthread_mutex_unlock(&printf_mutex);
-  va_end(args);
-}
-
-
 static void try_recv(struct zfur* ur)
 {
   struct {
@@ -98,10 +88,12 @@ static void try_recv(struct zfur* ur)
       struct timespec ts;
       int rc = zfur_pkt_get_timestamp(ur, &rd.msg, &ts, 0, &flags);
 
+      pthread_mutex_lock(&printf_mutex);
       if( rc == 0 )
-        vlog("Hardware timestamp: %lld.%.9ld\n", ts.tv_sec, ts.tv_nsec);
+        printf("Hardware timestamp: %ld.%.9ld\n", ts.tv_sec, ts.tv_nsec);
       else
-        vlog("Error retrieving timestamp! Return code: %d\n", rc);
+        printf("Error retrieving timestamp! Return code: %d\n", rc);
+      pthread_mutex_unlock(&printf_mutex);
     }
 
     zfur_zc_recv_done(ur, &rd.msg);
@@ -243,14 +235,72 @@ void print_attrs(struct zf_attr* attr)
 }
 
 
-static void monitor()
+static void zf_stats_header_print(struct zf_stack* stack,
+                                  zf_layout_collection* stats_collection)
+{
+  int i;
+
+  if( stats_collection->num_intfs > 1 )
+    printf("\n#");
+
+  /* only display names for the first NIC and assume others are the same */
+  for( i = 0; i < stats_collection->layout[0]->evsl_fields_num; ++i )
+    printf("  %10s", stats_collection->layout[0]->evsl_fields[i].evsfl_name);
+}
+
+
+static void zf_stats_print(struct zf_stack* stack, uint8_t* stats_data,
+                           zf_layout_collection* stats_collection)
+{
+  int i, n, n_pad;
+  uint8_t* cur_data = stats_data;
+  const zf_stats_layout * layout;
+
+  for( n = 0; n < stats_collection->num_intfs; n++ ) {
+    if( stats_collection->num_intfs > 1 )
+      printf("\n ");
+
+    layout = stats_collection->layout[n];
+    for( i = 0; i < layout->evsl_fields_num; ++i ) {
+      const zf_stats_field_layout* f = &layout->evsl_fields[i];
+      n_pad = strlen(f->evsfl_name);
+      if( n_pad < 10 )
+        n_pad = 10;
+      switch( f->evsfl_size ) {
+        case sizeof(uint32_t):
+          printf("  %*d", n_pad, *(uint32_t*)(cur_data + f->evsfl_offset));
+          break;
+        default:
+          printf("  %*s", n_pad, ".");
+      };
+    }
+    cur_data += layout->evsl_data_size;
+  }
+}
+
+
+static void monitor(struct zf_stack* stack)
 {
   uint64_t now_bytes, prev_bytes;
   struct timeval start, end;
   uint64_t prev_pkts, now_pkts;
   int ms, pkt_rate, mbps;
 
-  vlog("#%9s %16s %16s", "pkt-rate", "bandwidth(Mbps)", "total-pkts\n");
+  zf_layout_collection* stats_collection;
+  uint8_t* stats_data = NULL;
+
+  pthread_mutex_lock(&printf_mutex);
+  printf("#%9s %16s %16s", "pkt-rate", "bandwidth(Mbps)", "total-pkts");
+
+  if( cfg_intf_stats ) {
+    ZF_TEST(zf_stats_alloc_layout_collection(stack, &stats_collection) == 0);
+    zf_stats_header_print(stack, stats_collection);
+
+    ZF_TEST((stats_data = malloc(stats_collection->total_data_size)) != NULL);
+  }
+
+  printf("\n");
+  pthread_mutex_unlock(&printf_mutex);
 
   prev_pkts = res.n_rx_pkts;
   prev_bytes = res.n_rx_bytes;
@@ -265,18 +315,35 @@ static void monitor()
     ms += (end.tv_usec - start.tv_usec) / 1000;
     pkt_rate = (int) ((now_pkts - prev_pkts) * 1000 / ms);
     mbps = (int) ((now_bytes - prev_bytes) * 8 / 1000 / ms);
-    vlog("%10d %16d %16"PRIu64"\n", pkt_rate, mbps, now_pkts);
+    
+    if( cfg_intf_stats )
+      zf_stats_query(stack, stats_data, stats_collection, 1);
+
+    pthread_mutex_lock(&printf_mutex);
+    printf("%10d %16d %16"PRIu64, pkt_rate, mbps, now_pkts);
+
+    if( cfg_intf_stats )
+      zf_stats_print(stack, stats_data, stats_collection);
+
+    printf("\n");
     fflush(stdout);
+    pthread_mutex_unlock(&printf_mutex);
+    
     prev_pkts = now_pkts;
     prev_bytes = now_bytes;
     start = end;
   }
+
+  if( cfg_intf_stats ) {
+    free(stats_data);
+    zf_stats_free_layout_collection(stats_collection);
+  }
 }
 
 
 static void* monitor_fn(void* arg)
 {
-  monitor();
+  monitor( (struct zf_stack*)arg );
   return NULL;
 }
 
@@ -289,7 +356,7 @@ int main(int argc, char* argv[])
   bool cfg_print_attrs = false;
 
   int c;
-  while( (c = getopt(argc, argv, "hmrwqp")) != -1 )
+  while( (c = getopt(argc, argv, "hmrwqps")) != -1 )
     switch( c ) {
     case 'h':
       usage_msg(stdout);
@@ -309,6 +376,9 @@ int main(int argc, char* argv[])
     case 'p':
       cfg_print_attrs = true;
       break;
+    case 's':
+      cfg_intf_stats = true;
+      break;
     case '?':
       exit(1);
     default:
@@ -374,7 +444,7 @@ int main(int argc, char* argv[])
   res.n_rx_pkts = 0;
 
   if( ! cfg_quiet )
-    ZF_TRY(pthread_create(&thread_id, NULL, monitor_fn, NULL) == 0);
+    ZF_TRY(pthread_create(&thread_id, NULL, monitor_fn, stack) == 0);
 
   if( cfg_waitable_fd )
     ev_loop_waitable_fd(stack, muxer);