Revert retransmit logic changes

src/app/shared_dev/commands/bench/fd_benchs.c

@@ -232,7 +232,7 @@ populate_quic_limits( fd_quic_limits_t * limits ) {
   limits->conn_cnt = 2;
   limits->handshake_cnt = limits->conn_cnt;
   limits->conn_id_cnt = 16;
-  limits->inflight_frame_cnt = 1500;
+  limits->inflight_pkt_cnt = 1500;
   limits->tx_buf_sz = FD_TXN_MTU;
   limits->stream_pool_cnt = 1UL<<16;
   limits->stream_id_cnt = 1UL<<16;

src/app/shared_dev/commands/txn.c

@@ -146,13 +146,13 @@ txn_cmd_fn( args_t *   args,
   ready_cmd_fn( args, config );
 
   fd_quic_limits_t quic_limits = {
-    .conn_cnt           =  1UL,
-    .handshake_cnt      =  1UL,
-    .conn_id_cnt        =  4UL,
-    .stream_id_cnt      =  64UL,
-    .inflight_frame_cnt =  64UL,
-    .tx_buf_sz          =  fd_ulong_pow2_up( FD_TXN_MTU ),
-    .stream_pool_cnt    =  16UL
+    .conn_cnt         =  1UL,
+    .handshake_cnt    =  1UL,
+    .conn_id_cnt      =  4UL,
+    .stream_id_cnt    = 64UL,
+    .inflight_pkt_cnt = 64UL,
+    .tx_buf_sz        = fd_ulong_pow2_up( FD_TXN_MTU ),
+    .stream_pool_cnt  = 16UL
   };
   ulong quic_footprint = fd_quic_footprint( &quic_limits );
   FD_TEST( quic_footprint );

src/disco/quic/fd_quic_tile.c

@@ -37,9 +37,8 @@ quic_limits( fd_topo_tile_t const * tile ) {
     /* fd_quic will not issue nor use any new connection IDs after
        completing a handshake.  Connection migration is not supported
        either. */
-    .conn_id_cnt                 = FD_QUIC_MIN_CONN_ID_CNT,
-    .inflight_frame_cnt          = 64UL * tile->quic.max_concurrent_connections,
-    .min_inflight_frame_cnt_conn = 32UL
+    .conn_id_cnt      = FD_QUIC_MIN_CONN_ID_CNT,
+    .inflight_pkt_cnt = 16UL,
   };
   if( FD_UNLIKELY( !fd_quic_footprint( &limits ) ) ) {
     FD_LOG_ERR(( "Invalid QUIC limits in config" ));

src/waltz/quic/fd_quic.h

@@ -111,10 +111,9 @@ struct __attribute__((aligned(16UL))) fd_quic_limits {
   ulong  handshake_cnt;             /* instance-wide, max concurrent handshake count  */
   ulong  log_depth;                 /* instance-wide, depth of shm log cache          */
 
-  ulong  conn_id_cnt;                 /* per-conn, max conn ID count (min 4UL)          */
-  ulong  stream_id_cnt;               /* per-conn, max concurrent stream ID count       */
-  ulong  inflight_frame_cnt;          /* instance-wide, total max inflight frame count  */
-  ulong  min_inflight_frame_cnt_conn; /* per-conn, min inflight frame count             */
+  ulong  conn_id_cnt;               /* per-conn, max conn ID count (min 4UL)          */
+  ulong  stream_id_cnt;             /* per-conn, max concurrent stream ID count       */
+  ulong  inflight_pkt_cnt;          /* instance-wide, total max inflight packet count */
 
   ulong  tx_buf_sz;                 /* per-stream, tx buf sz in bytes                 */
   /* the user consumes rx directly from the network buffer */
@@ -136,7 +135,6 @@ struct fd_quic_layout {
   ulong hs_pool_off;       /* offset of the handshake pool     */
   ulong stream_pool_off;   /* offset of the stream pool        */
   ulong svc_timers_off;    /* offset of the service timers     */
-  ulong pkt_meta_pool_off; /* offset of the pkt_meta pool      */
 };
 
 typedef struct fd_quic_layout fd_quic_layout_t;

src/waltz/quic/fd_quic_conn.c

@@ -17,12 +17,16 @@
 struct fd_quic_conn_layout {
   int   stream_map_lg;
   ulong stream_map_off;
+  ulong pkt_meta_off;
 };
 typedef struct fd_quic_conn_layout fd_quic_conn_layout_t;
 
+/* TODO maybe introduce a separate parameter for size of pkt_meta
+   pool? */
 ulong
 fd_quic_conn_align( void ) {
   ulong align = fd_ulong_max( alignof( fd_quic_conn_t ), alignof( fd_quic_stream_t ) );
+  align = fd_ulong_max( align, alignof( fd_quic_pkt_meta_t ) );
   align = fd_ulong_max( align, fd_quic_stream_map_align() );
   return align;
 }
@@ -31,6 +35,9 @@ static ulong
 fd_quic_conn_footprint_ext( fd_quic_limits_t const * limits,
                             fd_quic_conn_layout_t *  layout ) {
 
+  ulong inflight_pkt_cnt = limits->inflight_pkt_cnt;
+  if( FD_UNLIKELY( inflight_pkt_cnt==0UL ) ) return 0UL;
+
   ulong stream_id_cnt = limits->stream_id_cnt;
 
   ulong off  = 0;
@@ -54,7 +61,12 @@ fd_quic_conn_footprint_ext( fd_quic_limits_t const * limits,
     layout->stream_map_off = 0UL;
   }
 
-  return fd_ulong_align_up( off, fd_quic_conn_align() );
+  /* allocate space for packet metadata */
+  off                   = fd_ulong_align_up( off, alignof(fd_quic_pkt_meta_t) );
+  layout->pkt_meta_off  = off;
+  off                  += inflight_pkt_cnt * sizeof(fd_quic_pkt_meta_t);
+
+  return off;
 }
 
 ulong
@@ -104,7 +116,7 @@ fd_quic_conn_new( void *                   mem,
   fd_memset( conn, 0, sizeof(fd_quic_conn_t) );
 
   conn->quic  = quic;
-  fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
+  conn->state = FD_QUIC_CONN_STATE_INVALID;
 
   /* Initialize streams */
 
@@ -119,12 +131,14 @@ fd_quic_conn_new( void *                   mem,
     if( FD_UNLIKELY( !conn->stream_map ) ) return NULL;
   }
 
-  /* Initialize packet meta tracker */
-  fd_quic_state_t * state = fd_quic_get_state( quic );
-  fd_quic_pkt_meta_tracker_init( &conn->pkt_meta_tracker,
-                                quic->limits.inflight_frame_cnt,
-                                state->pkt_meta_pool );
+  /* Initialize packet meta pool */
+
+  ulong                pkt_meta_cnt = limits->inflight_pkt_cnt;
+  fd_quic_pkt_meta_t * pkt_meta     = (fd_quic_pkt_meta_t *)( (ulong)mem + layout.pkt_meta_off );
+  fd_memset( pkt_meta, 0, pkt_meta_cnt*sizeof(fd_quic_pkt_meta_t) );
 
+  /* store pointer to storage and size */
+  conn->pkt_meta_mem = pkt_meta;
 
   /* Initialize service timers */
   fd_quic_svc_timers_init_conn( conn );

src/waltz/quic/fd_quic_conn.h

@@ -201,7 +201,9 @@ struct fd_quic_conn {
   ulong             unacked_sz;  /* Number of received stream frame payload bytes pending ACK */
                                  /* Resets to zero when conn is rescheduled or ACKs are sent */
 
-  fd_quic_pkt_meta_tracker_t pkt_meta_tracker;
+  /* TODO find better name than pool */
+  fd_quic_pkt_meta_pool_t pkt_meta_pool;
+  fd_quic_pkt_meta_t *    pkt_meta_mem;    /* owns the memory */
 
   /* flow control */
   ulong                tx_max_data;        /* the limit on the number of bytes we are allowed
@@ -243,8 +245,6 @@ struct fd_quic_conn {
   uchar token[ FD_QUIC_RETRY_MAX_TOKEN_SZ ];
 
   fd_quic_conn_stream_rx_t srx[1];
-
-  ulong used_pkt_meta;
 };
 
 inline static void

src/waltz/quic/fd_quic_pkt_meta.c

@@ -1,76 +1,108 @@
 #include "fd_quic_pkt_meta.h"
 
-void *
-fd_quic_pkt_meta_tracker_init( fd_quic_pkt_meta_tracker_t * tracker,
-                               ulong                        total_meta_cnt,
-                               fd_quic_pkt_meta_t         * pool ) {
-  for( ulong enc_level=0; enc_level<4; enc_level++ ) {
-    void* mem = fd_quic_pkt_meta_treap_new( &tracker->sent_pkt_metas[enc_level],
-                                            total_meta_cnt );
-    mem = fd_quic_pkt_meta_treap_join( mem );
-    if( FD_UNLIKELY( !mem ) ) return NULL;
+/* initialize pool with existing array of pkt_meta */
+void
+fd_quic_pkt_meta_pool_init( fd_quic_pkt_meta_pool_t * pool,
+                            fd_quic_pkt_meta_t *      pkt_meta_array,
+                            ulong                     pkt_meta_array_sz ) {
+  /* initialize all to zeros */
+  fd_memset( pool, 0, sizeof( *pool ) );
+
+  /* free list */
+  fd_quic_pkt_meta_list_t * free = &pool->free;
+
+  /* initialize free list of packet metadata */
+  for( ulong j = 0; j < pkt_meta_array_sz; ++j ) {
+    fd_quic_pkt_meta_push_back( free, &pkt_meta_array[j] );
   }
-  tracker->pool = pool;
 
-  return tracker;
 }
 
+
+/* pop from front of list */
+fd_quic_pkt_meta_t *
+fd_quic_pkt_meta_pop_front( fd_quic_pkt_meta_list_t * list ) {
+  fd_quic_pkt_meta_t * front = list->head;
+  if( front ) {
+    list->head = front->next;
+  }
+  return front;
+}
+
+
+/* push onto front of list */
 void
-fd_quic_pkt_meta_ds_init_pool( fd_quic_pkt_meta_t * pool,
-                               ulong                total_meta_cnt ) {
-  fd_quic_pkt_meta_treap_seed( pool, total_meta_cnt, (ulong)fd_log_wallclock() );
+fd_quic_pkt_meta_push_front( fd_quic_pkt_meta_list_t * list,
+                             fd_quic_pkt_meta_t *      pkt_meta ) {
+  pkt_meta->next = list->head;
+  list->head     = pkt_meta;
 }
 
+
+/* push onto back of list */
 void
-fd_quic_pkt_meta_insert( fd_quic_pkt_meta_ds_t * ds,
-                         fd_quic_pkt_meta_t    * pkt_meta,
-                         fd_quic_pkt_meta_t    * pool ) {
-  fd_quic_pkt_meta_treap_ele_insert( ds, pkt_meta, pool );
+fd_quic_pkt_meta_push_back( fd_quic_pkt_meta_list_t * list,
+                            fd_quic_pkt_meta_t *      pkt_meta ) {
+  fd_quic_pkt_meta_t * tail = list->tail;
+  if( tail ) {
+    tail->next = pkt_meta;
+    list->tail = pkt_meta;
+  } else {
+    list->head = list->tail = pkt_meta;
+  }
+
+  pkt_meta->next = NULL;
 }
 
+/* remove from list
+   requires the prior element */
+void
+fd_quic_pkt_meta_remove( fd_quic_pkt_meta_list_t * list,
+                         fd_quic_pkt_meta_t *      pkt_meta_prior,
+                         fd_quic_pkt_meta_t *      pkt_meta ) {
+  fd_quic_pkt_meta_t * pkt_meta_next = pkt_meta->next;
 
-ulong
-fd_quic_pkt_meta_remove_range( fd_quic_pkt_meta_ds_t * ds,
-                               fd_quic_pkt_meta_t    * pool,
-                               ulong                   pkt_number_lo,
-                               ulong                   pkt_number_hi ) {
-
-  fd_quic_pkt_meta_ds_fwd_iter_t    l_iter =  fd_quic_pkt_meta_ds_idx_ge( ds, pkt_number_lo, pool );
-  fd_quic_pkt_meta_t              * prev   =  NULL;
-  ulong                        cnt_removed =  0;
-
-  for( fd_quic_pkt_meta_ds_fwd_iter_t iter = l_iter;
-                                            !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
-                                            iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
-    fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
-    if( FD_UNLIKELY( e->key.pkt_num > pkt_number_hi ) ) break;
-    if( FD_LIKELY( prev ) ) {
-      fd_quic_pkt_meta_treap_ele_remove( ds, prev, pool );
-      fd_quic_pkt_meta_pool_ele_release( pool, prev );
-      cnt_removed++;
+  if( pkt_meta_prior == NULL ) {
+    if( pkt_meta_next == NULL ) {
+      /* at tail... then head = tail = NULL */
+      list->head = list->tail = NULL;
+    } else {
+      /* at head... move it to next */
+      list->head = pkt_meta_next;
     }
-    prev = e;
-  }
-  if( FD_LIKELY( prev ) ) {
-    fd_quic_pkt_meta_treap_ele_remove( ds, prev, pool );
-    fd_quic_pkt_meta_pool_ele_release( pool, prev );
-    cnt_removed++;
+  } else {
+    if( pkt_meta_next == NULL ) {
+      /* we're removing the last, so move tail */
+      list->tail = pkt_meta_prior;
+    }
+
+    /* not head, make pkt_meta_prior point to next */
+    pkt_meta_prior->next = pkt_meta_next;
   }
-  return cnt_removed;
+
+  pkt_meta->next = NULL;
 }
 
-fd_quic_pkt_meta_t *
-fd_quic_pkt_meta_min( fd_quic_pkt_meta_ds_t * ds,
-                      fd_quic_pkt_meta_t    * pool ) {
 
-  fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( ds, pool );
-  if( FD_UNLIKELY( fd_quic_pkt_meta_ds_fwd_iter_done( iter ) ) ) return NULL;
-  return fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
+/* allocate a pkt_meta
+   obtains a free pkt_meta from the free list, and returns it
+   returns NULL if none is available */
+fd_quic_pkt_meta_t *
+fd_quic_pkt_meta_allocate( fd_quic_pkt_meta_pool_t * pool ) {
+  fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pop_front( &pool->free );
+  if( FD_LIKELY( pkt_meta ) ) {
+    fd_memset( pkt_meta, 0, sizeof( *pkt_meta ) );
+  }
+  return pkt_meta;
 }
 
+
+/* free a pkt_meta
+   returns a pkt_meta to the free list, ready to be allocated again */
 void
-fd_quic_pkt_meta_ds_clear( fd_quic_pkt_meta_tracker_t * tracker,
-                        uint                         enc_level ) {
-  ulong ele_max = fd_quic_pkt_meta_treap_ele_max( &tracker->sent_pkt_metas[enc_level] );
-  fd_quic_pkt_meta_treap_new( &tracker->sent_pkt_metas[enc_level], ele_max );
+fd_quic_pkt_meta_deallocate( fd_quic_pkt_meta_pool_t * pool, fd_quic_pkt_meta_t * pkt_meta ) {
+  /* pushing to the front should help cache usage */
+  fd_quic_pkt_meta_push_front( &pool->free, pkt_meta );
 }
+
+