diff --git a/src/app/shared_dev/commands/bench/fd_benchs.c b/src/app/shared_dev/commands/bench/fd_benchs.c index 70188cac20..d25f162317 100644 --- a/src/app/shared_dev/commands/bench/fd_benchs.c +++ b/src/app/shared_dev/commands/bench/fd_benchs.c @@ -232,7 +232,7 @@ populate_quic_limits( fd_quic_limits_t * limits ) { limits->conn_cnt = 2; limits->handshake_cnt = limits->conn_cnt; limits->conn_id_cnt = 16; - limits->inflight_frame_cnt = 1500; + limits->inflight_pkt_cnt = 1500; limits->tx_buf_sz = FD_TXN_MTU; limits->stream_pool_cnt = 1UL<<16; limits->stream_id_cnt = 1UL<<16; diff --git a/src/app/shared_dev/commands/txn.c b/src/app/shared_dev/commands/txn.c index edb9988bb0..db3666c006 100644 --- a/src/app/shared_dev/commands/txn.c +++ b/src/app/shared_dev/commands/txn.c @@ -146,13 +146,13 @@ txn_cmd_fn( args_t * args, ready_cmd_fn( args, config ); fd_quic_limits_t quic_limits = { - .conn_cnt = 1UL, - .handshake_cnt = 1UL, - .conn_id_cnt = 4UL, - .stream_id_cnt = 64UL, - .inflight_frame_cnt = 64UL, - .tx_buf_sz = fd_ulong_pow2_up( FD_TXN_MTU ), - .stream_pool_cnt = 16UL + .conn_cnt = 1UL, + .handshake_cnt = 1UL, + .conn_id_cnt = 4UL, + .stream_id_cnt = 64UL, + .inflight_pkt_cnt = 64UL, + .tx_buf_sz = fd_ulong_pow2_up( FD_TXN_MTU ), + .stream_pool_cnt = 16UL }; ulong quic_footprint = fd_quic_footprint( &quic_limits ); FD_TEST( quic_footprint ); diff --git a/src/disco/quic/fd_quic_tile.c b/src/disco/quic/fd_quic_tile.c index 22ec44a7a1..2c8d482375 100644 --- a/src/disco/quic/fd_quic_tile.c +++ b/src/disco/quic/fd_quic_tile.c @@ -37,9 +37,8 @@ quic_limits( fd_topo_tile_t const * tile ) { /* fd_quic will not issue nor use any new connection IDs after completing a handshake. Connection migration is not supported either. */ - .conn_id_cnt = FD_QUIC_MIN_CONN_ID_CNT, - .inflight_frame_cnt = 64UL * tile->quic.max_concurrent_connections, - .min_inflight_frame_cnt_conn = 32UL + .conn_id_cnt = FD_QUIC_MIN_CONN_ID_CNT, + .inflight_pkt_cnt = 16UL, }; if( FD_UNLIKELY( !fd_quic_footprint( &limits ) ) ) { FD_LOG_ERR(( "Invalid QUIC limits in config" )); diff --git a/src/waltz/quic/fd_quic.c b/src/waltz/quic/fd_quic.c index 1b1760e45e..d373d84dc5 100644 --- a/src/waltz/quic/fd_quic.c +++ b/src/waltz/quic/fd_quic.c @@ -64,19 +64,17 @@ fd_quic_footprint_ext( fd_quic_limits_t const * limits, memset( layout, 0, sizeof(fd_quic_layout_t) ); if( FD_UNLIKELY( !limits ) ) return 0UL; - ulong conn_cnt = limits->conn_cnt; - ulong conn_id_cnt = limits->conn_id_cnt; - ulong log_depth = limits->log_depth; - ulong handshake_cnt = limits->handshake_cnt; - ulong inflight_frame_cnt = limits->inflight_frame_cnt; - ulong tx_buf_sz = limits->tx_buf_sz; - ulong stream_pool_cnt = limits->stream_pool_cnt; - ulong inflight_res_cnt = limits->min_inflight_frame_cnt_conn * conn_cnt; - if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL; - if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL; - if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL; - - if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL; + ulong conn_cnt = limits->conn_cnt; + ulong conn_id_cnt = limits->conn_id_cnt; + ulong log_depth = limits->log_depth; + ulong handshake_cnt = limits->handshake_cnt; + ulong inflight_pkt_cnt = limits->inflight_pkt_cnt; + ulong tx_buf_sz = limits->tx_buf_sz; + ulong stream_pool_cnt = limits->stream_pool_cnt; + + if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL; + if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL; + if( FD_UNLIKELY( inflight_pkt_cnt==0UL ) ) return 0UL; if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT )) return 0UL; @@ -129,17 +127,6 @@ fd_quic_footprint_ext( fd_quic_limits_t const * limits, layout->stream_pool_off = 0UL; } - /* allocate space for pkt_meta_pool */ - if( inflight_frame_cnt ) { - offs = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() ); - layout->pkt_meta_pool_off = offs; - ulong pkt_meta_footprint = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt ); - if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; } - offs += pkt_meta_footprint; - } else { - layout->pkt_meta_pool_off = 0UL; - } - /* allocate space for quic_log_buf */ offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() ); layout->log_off = offs; @@ -195,10 +182,10 @@ fd_quic_new( void * mem, return NULL; } - if( FD_UNLIKELY( ( limits->conn_cnt ==0UL ) - | ( limits->conn_cnt >=UINT_MAX ) - | ( limits->handshake_cnt ==0UL ) - | ( limits->inflight_frame_cnt==0UL ) ) ) { + if( FD_UNLIKELY( ( limits->conn_cnt ==0UL ) + | ( limits->conn_cnt >=UINT_MAX ) + | ( limits->handshake_cnt ==0UL ) + | ( limits->inflight_pkt_cnt==0UL ) ) ) { FD_LOG_WARNING(( "invalid limits" )); return NULL; } @@ -250,12 +237,12 @@ fd_quic_limits_from_env( int * pargc, if( FD_UNLIKELY( !limits ) ) return NULL; - limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL ); - limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL ); - limits->stream_pool_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL ); - limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL ); - limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL ); - limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL ); + limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL ); + limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL ); + limits->stream_pool_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL ); + limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL ); + limits->inflight_pkt_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL ); + limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL ); return limits; } @@ -438,15 +425,6 @@ fd_quic_init( fd_quic_t * quic ) { FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" )); } - /* State: Initialize packet meta pool */ - if( layout.pkt_meta_pool_off ) { - ulong pkt_meta_cnt = limits->inflight_frame_cnt; - ulong pkt_meta_laddr = (ulong)quic + layout.pkt_meta_pool_off; - fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt ); - state->pkt_meta_pool = fd_quic_pkt_meta_pool_join( pkt_meta_pool ); - fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt ); - } - /* State: initialize each connection, and add to free list */ ulong conn_laddr = (ulong)quic + layout.conns_off; @@ -600,9 +578,6 @@ fd_quic_init( fd_quic_t * quic ) { FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay, max_ack_delay_ms_u ); /* */tp->disable_active_migration_present = 1; - /* Compute max inflight pkt cnt per conn */ - state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1); - return quic; } @@ -766,7 +741,7 @@ fd_quic_conn_error1( fd_quic_conn_t * conn, uint reason ) { if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return; - fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT ); + conn->state = FD_QUIC_CONN_STATE_ABORT; conn->reason = reason; /* set connection to be serviced ASAP */ @@ -1241,48 +1216,38 @@ fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) { /* packet processing */ /* fd_quic_abandon_enc_level frees all resources associated encryption - levels less or equal to enc_level. Returns the number of freed - pkt_meta. */ + levels less or equal to enc_level. */ -ulong +void fd_quic_abandon_enc_level( fd_quic_conn_t * conn, uint enc_level ) { - if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL; + if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return; FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); ) - ulong freed = 0UL; - fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level ); - fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker; - fd_quic_pkt_meta_t * pool = tracker->pool; + fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool; for( uint j = 0; j <= enc_level; ++j ) { conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j ); + /* treat all packets as ACKed (freeing handshake data, etc.) */ - fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[j]; - - fd_quic_pkt_meta_t * prev = NULL; - for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_treap_fwd_iter_init( sent, pool ); - !fd_quic_pkt_meta_ds_fwd_iter_done( iter ); - iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) { - fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool ); - if( FD_LIKELY( prev ) ) { - fd_quic_pkt_meta_pool_ele_release( pool, prev ); - } - fd_quic_reclaim_pkt_meta( conn, e, j ); - prev = e; - } - if( FD_LIKELY( prev ) ) { - fd_quic_pkt_meta_pool_ele_release( pool, prev ); - } + fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[j]; + fd_quic_pkt_meta_t * pkt_meta = sent->head; + fd_quic_pkt_meta_t * prior = NULL; /* there is no prior, as this is the head */ + while( pkt_meta ) { + fd_quic_reclaim_pkt_meta( conn, pkt_meta, j ); - freed += fd_quic_pkt_meta_ds_ele_cnt( sent ); - conn->used_pkt_meta -= fd_quic_pkt_meta_ds_ele_cnt( sent ); - fd_quic_pkt_meta_ds_clear( tracker, j ); - } + /* remove from list */ + fd_quic_pkt_meta_remove( sent, prior, pkt_meta ); + + /* put pkt_meta back in free list */ + fd_quic_pkt_meta_deallocate( pool, pkt_meta ); - return freed; + /* head should have been reclaimed, so fetch new head */ + pkt_meta = pool->sent_pkt_meta[j].head; + } + } } static void @@ -2196,6 +2161,7 @@ fd_quic_process_quic_packet_v1( fd_quic_t * quic, /* initialize packet number to unused value */ pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED; + /* long_packet_type is 2 bits, so only four possibilities */ switch( long_packet_type ) { case FD_QUIC_PKT_TYPE_INITIAL: rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz ); @@ -2723,7 +2689,7 @@ fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs, return; } conn->handshake_complete = 1; - fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ); + conn->state = FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE; return; default: @@ -2835,7 +2801,7 @@ fd_quic_svc_poll( fd_quic_t * quic, conn->server?"SERVER":"CLIENT", (void *)conn, conn->conn_idx, (double)conn->idle_timeout / 1e6 )); ) - fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); + conn->state = FD_QUIC_CONN_STATE_DEAD; quic->metrics.conn_timeout_cnt++; } } else if( quic->config.keep_alive ) { @@ -2847,6 +2813,15 @@ fd_quic_svc_poll( fd_quic_t * quic, } } + if( now > conn->last_ack + (ulong)conn->rtt->rtt_period_ticks ) { + /* send PING */ + if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) + && conn->state == FD_QUIC_CONN_STATE_ACTIVE ) { + conn->flags |= FD_QUIC_CONN_FLAGS_PING; + conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */ + } + } + if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) { fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */ fd_quic_conn_free( quic, conn ); @@ -3029,38 +3004,12 @@ fd_quic_tx_buffered( fd_quic_t * quic, conn->host.udp_port); } -/* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker. - Value and type take the passed args; all other fields are copied - from pkt_meta_tmpl. Returns 1 if successful, 0 if not. - Failure reasons include empty pkt_meta pool, or this conn reached - its pkt_meta limit. Theoretically only need latter, but let's be safe! */ -static inline int -fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t * pkt_meta_tmpl, - uchar type, - fd_quic_pkt_meta_value_t value, - fd_quic_pkt_meta_tracker_t * tracker, - fd_quic_conn_t * conn ) { - fd_quic_state_t * state = fd_quic_get_state( conn->quic ); - if( !fd_quic_pkt_meta_pool_free( tracker->pool ) || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) { - conn->quic->metrics.pkt_tx_alloc_fail_cnt++; - return 0; - } - - conn->used_pkt_meta++; - fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool ); - *pkt_meta = *pkt_meta_tmpl; - FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type ); - pkt_meta->val = value; - fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool ); - return 1; -} - static ulong -fd_quic_gen_close_frame( fd_quic_conn_t * conn, - uchar * payload_ptr, - uchar * payload_end, - const fd_quic_pkt_meta_t * pkt_meta_tmpl, - fd_quic_pkt_meta_tracker_t * tracker ) { +fd_quic_gen_close_frame( fd_quic_conn_t * conn, + uchar * payload_ptr, + uchar * payload_end, + fd_quic_pkt_meta_t * pkt_meta, + ulong now ) { if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL; conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT; @@ -3090,33 +3039,22 @@ fd_quic_gen_close_frame( fd_quic_conn_t * conn, return 0UL; } - /* create and save pkt_meta, return 0 if fail */ - if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl, - FD_QUIC_PKT_META_TYPE_CLOSE, - (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */ - tracker, - conn )) return 0UL; - + /* update packet meta */ + pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_CLOSE; + pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, fd_quic_calc_expiry( conn, now ) ); return frame_sz; } static uchar * -fd_quic_gen_handshake_frames( fd_quic_conn_t * conn, - uchar * payload_ptr, - uchar * payload_end, - const fd_quic_pkt_meta_t * pkt_meta_tmpl, - fd_quic_pkt_meta_tracker_t * tracker ) { - uint enc_level = pkt_meta_tmpl->enc_level; +fd_quic_gen_handshake_frames( fd_quic_conn_t * conn, + uchar * payload_ptr, + uchar * payload_end, + uint enc_level, + fd_quic_pkt_meta_t * pkt_meta, + ulong now ) { fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level ); if( !hs_data ) return payload_ptr; - /* confirm we have pkt_meta space */ - fd_quic_state_t * state = fd_quic_get_state( conn->quic ); - if( !fd_quic_pkt_meta_pool_free( tracker->pool ) || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) { - conn->quic->metrics.pkt_tx_alloc_fail_cnt++; - return payload_ptr; - } - ulong hs_offset = 0; /* offset within the current hs_data */ ulong sent_offset = conn->hs_sent_bytes[enc_level]; ulong ackd_offset = conn->hs_ackd_bytes[enc_level]; @@ -3176,56 +3114,65 @@ fd_quic_gen_handshake_frames( fd_quic_conn_t * conn, /* update packet meta */ if( offset_hi > offset_lo ) { - fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl, - FD_QUIC_PKT_META_TYPE_HS_DATA, - (fd_quic_pkt_meta_value_t){ - .range = { - .offset_lo = offset_lo, - .offset_hi = offset_hi - } - }, - tracker, - conn ); + pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_HS_DATA; + pkt_meta->range.offset_lo = offset_lo; + pkt_meta->range.offset_hi = offset_hi; + pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, fd_quic_calc_expiry( conn, now ) ); } return payload_ptr; } static ulong -fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn, - uchar * payload_ptr, - uchar * payload_end, - const fd_quic_pkt_meta_t * pkt_meta_tmpl, - fd_quic_pkt_meta_tracker_t * tracker ) { - FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id ); +fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn, + uchar * payload_ptr, + uchar * payload_end, + fd_quic_pkt_meta_t * pkt_meta, + ulong now ) { if( conn->handshake_done_send==0 ) return 0UL; conn->handshake_done_send = 0; if( FD_UNLIKELY( conn->handshake_done_ackd ) ) return 0UL; if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL; /* send handshake done frame */ + pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_HS_DONE; + pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, fd_quic_calc_expiry( conn, now ) ); payload_ptr[0] = 0x1E; - - /* record the send for retx */ - if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl, - FD_QUIC_PKT_META_TYPE_HS_DONE, - (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */ - tracker, - conn) ) return 0UL; - return 1UL; } +static void +fd_quic_gen_frame_update_pkt_meta( fd_quic_conn_t * conn, + fd_quic_pkt_meta_t * pkt_meta, + uint pm_flag, + uint var_key_flag, + ulong value, + ulong now ) { + pkt_meta->flags |= pm_flag; + pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, fd_quic_calc_expiry( conn, now ) ); + pkt_meta->var[pkt_meta->var_sz].key = (fd_quic_pkt_meta_key_t){ + .type = FD_QUIC_PKT_META_TYPE_OTHER, + .flags = var_key_flag + }; + pkt_meta->var[pkt_meta->var_sz].value = value; + pkt_meta->var_sz = (uchar)( pkt_meta->var_sz + 1 ); +} + static ulong -fd_quic_gen_max_data_frame( fd_quic_conn_t * conn, - uchar * payload_ptr, - uchar * payload_end, - const fd_quic_pkt_meta_t * pkt_meta_tmpl, - fd_quic_pkt_meta_tracker_t * tracker ) { +fd_quic_gen_max_data_frame( fd_quic_conn_t * conn, + uchar * payload_ptr, + uchar * payload_end, + fd_quic_pkt_meta_t * pkt_meta, + ulong pkt_number, + ulong now ) { fd_quic_conn_stream_rx_t * srx = conn->srx; if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL; if( srx->rx_max_data <= srx->rx_max_data_ackd ) return 0UL; /* peer would ignore anyway */ + if( FD_UNLIKELY( pkt_meta->var_sz >= FD_QUIC_PKT_META_VAR_MAX ) ) { + return 0UL; + } + /* send max_data frame */ fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data }; @@ -3235,25 +3182,25 @@ fd_quic_gen_max_data_frame( fd_quic_conn_t * conn, &frame ); if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL; - /* acquire and set a pkt_meta, return 0 if not successful */ - if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl, - FD_QUIC_PKT_META_TYPE_MAX_DATA, - (fd_quic_pkt_meta_value_t){ - .scalar = srx->rx_max_data - }, - tracker, - conn ) ) return 0UL; + /* set flag on pkt meta */ + fd_quic_gen_frame_update_pkt_meta( conn, + pkt_meta, + FD_QUIC_PKT_META_FLAGS_MAX_DATA, + FD_QUIC_CONN_FLAGS_MAX_DATA, + srx->rx_max_data, + now ); - conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num; + conn->upd_pkt_number = pkt_number; return frame_sz; } static ulong -fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn, - uchar * payload_ptr, - uchar * payload_end, - const fd_quic_pkt_meta_t * pkt_meta_tmpl, - fd_quic_pkt_meta_tracker_t * tracker ) { +fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn, + uchar * payload_ptr, + uchar * payload_end, + fd_quic_pkt_meta_t * pkt_meta, + ulong pkt_number, + ulong now ) { fd_quic_conn_stream_rx_t * srx = conn->srx; /* 0x02 Client-Initiated, Unidirectional @@ -3266,6 +3213,12 @@ fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn, if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL; } + if( FD_UNLIKELY( pkt_meta->var_sz >= FD_QUIC_PKT_META_VAR_MAX ) ) { + return 0UL; + } + + conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR); + fd_quic_max_streams_frame_t max_streams = { .type = 0x13, /* unidirectional */ .max_streams = max_streams_unidir @@ -3275,23 +3228,24 @@ fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn, &max_streams ); if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL; - if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl, - FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR, - (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */ - tracker, - conn ) ) return 0UL; + fd_quic_gen_frame_update_pkt_meta( conn, + pkt_meta, + FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR, + FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR, + max_streams.max_streams, + now ); - conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR); - conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num; + conn->upd_pkt_number = pkt_number; return frame_sz; } static ulong -fd_quic_gen_ping_frame( fd_quic_conn_t * conn, - uchar * payload_ptr, - uchar * payload_end, - const fd_quic_pkt_meta_t * pkt_meta_tmpl, - fd_quic_pkt_meta_tracker_t * tracker ) { +fd_quic_gen_ping_frame( fd_quic_conn_t * conn, + uchar * payload_ptr, + uchar * payload_end, + fd_quic_pkt_meta_t * pkt_meta, + ulong pkt_number, + ulong now ) { if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING ) return 0UL; if( conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT ) return 0UL; @@ -3304,37 +3258,34 @@ fd_quic_gen_ping_frame( fd_quic_conn_t * conn, conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT; conn->flags &= ~FD_QUIC_CONN_FLAGS_PING; - conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num; - /* record the send for retx, 0 if fail */ - if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl, - FD_QUIC_PKT_META_TYPE_PING, - (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */ - tracker, - conn ) ) return 0UL; + conn->upd_pkt_number = pkt_number; + + /* update packet metadata */ + pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_PING; + pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, fd_quic_calc_expiry( conn, now ) ); return frame_sz; } uchar * -fd_quic_gen_stream_frames( fd_quic_conn_t * conn, - uchar * payload_ptr, - uchar * payload_end, - fd_quic_pkt_meta_t * pkt_meta_tmpl, - fd_quic_pkt_meta_tracker_t * tracker ) { +fd_quic_gen_stream_frames( fd_quic_conn_t * conn, + uchar * payload_ptr, + uchar * payload_end, + fd_quic_pkt_meta_t * pkt_meta, + ulong pkt_number, + ulong now ) { /* loop serves two purposes: 1. finds a stream with data to send 2. appends max_stream_data frames as necessary */ fd_quic_stream_t * sentinel = conn->send_streams; fd_quic_stream_t * cur_stream = sentinel->next; - ulong pkt_num = pkt_meta_tmpl->key.pkt_num; - fd_quic_state_t * state = fd_quic_get_state( conn->quic ); - while( !cur_stream->sentinel ) { + while( !cur_stream->sentinel && pkt_meta->var_sz < FD_QUIC_PKT_META_VAR_MAX ) { /* required, since cur_stream may get removed from list */ fd_quic_stream_t * nxt_stream = cur_stream->next; _Bool sent_all_data = 1u; - if( cur_stream->upd_pkt_number >= pkt_num ) { + if( cur_stream->upd_pkt_number >= pkt_number ) { /* any stream data? */ if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) { @@ -3353,12 +3304,6 @@ fd_quic_gen_stream_frames( fd_quic_conn_t * conn, (Buffer should fit max stream header size and at least 1 byte of data) */ if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break; - /* check pkt_meta availability */ - if( !fd_quic_pkt_meta_pool_free( tracker->pool ) || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) { - conn->quic->metrics.pkt_tx_alloc_fail_cnt++; - break; - } - /* Leave placeholder for frame/stream type */ uchar * const frame_type_p = payload_ptr++; uint frame_type = 0x0a; /* stream frame with length */ @@ -3396,21 +3341,17 @@ fd_quic_gen_stream_frames( fd_quic_conn_t * conn, /* Update stream metadata */ cur_stream->tx_sent += data_sz; - cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING ); + cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_number, FD_QUIC_PKT_NUM_PENDING ); cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX ); /* Packet metadata for potential retransmits */ - pkt_meta_tmpl->key.stream_id = cur_stream->stream_id; - fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl, - FD_QUIC_PKT_META_TYPE_STREAM, - (fd_quic_pkt_meta_value_t){ - .range = { - .offset_lo = stream_off, - .offset_hi = stream_off + data_sz - } - }, - tracker, - conn ); + pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_STREAM; + pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, fd_quic_calc_expiry( conn, now ) ); + pkt_meta->var[pkt_meta->var_sz].key = + FD_QUIC_PKT_META_KEY( FD_QUIC_PKT_META_TYPE_STREAM_DATA, 0, cur_stream->stream_id ); + pkt_meta->var[pkt_meta->var_sz].range.offset_lo = stream_off; + pkt_meta->var[pkt_meta->var_sz].range.offset_hi = stream_off + data_sz; + pkt_meta->var_sz = (uchar)( pkt_meta->var_sz + 1 ); } } @@ -3427,11 +3368,13 @@ fd_quic_gen_stream_frames( fd_quic_conn_t * conn, } uchar * -fd_quic_gen_frames( fd_quic_conn_t * conn, - uchar * payload_ptr, - uchar * payload_end, - fd_quic_pkt_meta_t * pkt_meta_tmpl, - ulong now ) { +fd_quic_gen_frames( fd_quic_conn_t * conn, + uchar * payload_ptr, + uchar * payload_end, + uint enc_level, + fd_quic_pkt_meta_t * pkt_meta, + ulong pkt_number, + ulong now ) { uint closing = 0U; switch( conn->state ) { @@ -3441,24 +3384,22 @@ fd_quic_gen_frames( fd_quic_conn_t * conn, closing = 1u; } - fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker; - - payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now, (float)conn->quic->config.tick_per_us ); + payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, enc_level, now, (float)conn->quic->config.tick_per_us ); if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL; if( FD_UNLIKELY( closing ) ) { - payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker ); + payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta, now ); } else { - payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker ); - if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) { - payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker ); - if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) { - payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker ); - payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker ); - payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker ); + payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, enc_level, pkt_meta, now ); + if( enc_level == fd_quic_enc_level_appdata_id ) { + payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta, now ); + if( conn->upd_pkt_number >= pkt_number ) { + payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now ); + payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now ); + payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now ); } if( FD_LIKELY( !conn->tls_hs ) ) { - payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker ); + payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now ); } } } @@ -3475,12 +3416,12 @@ fd_quic_gen_frames( fd_quic_conn_t * conn, ping stream data */ static void -fd_quic_conn_tx( fd_quic_t * quic, +fd_quic_conn_tx( fd_quic_t * quic, fd_quic_conn_t * conn ) { if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return; - fd_quic_state_t * state = fd_quic_get_state( quic ); + fd_quic_state_t * state = fd_quic_get_state( quic ); /* used for encoding frames into before encrypting */ uchar * crypt_scratch = state->crypt_scratch; @@ -3490,6 +3431,8 @@ fd_quic_conn_tx( fd_quic_t * quic, /* TODO probably should be called tx_max_udp_payload_sz */ ulong tx_max_datagram_sz = conn->tx_max_datagram_sz; + fd_quic_pkt_meta_t * pkt_meta = NULL; + if( conn->tx_ptr != conn->tx_buf_conn ) { fd_quic_tx_buffered( quic, conn ); fd_quic_svc_prep_schedule( conn, state->now ); @@ -3525,17 +3468,35 @@ fd_quic_conn_tx( fd_quic_t * quic, /* get time, and set reschedule time for at most the idle timeout */ ulong now = fd_quic_get_state( quic )->now; - /* initialize expiry and tx_time */ - fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = now+500000000UL, .tx_time = now}}; - // pkt_meta_tmpl->expiry = fd_quic_calc_expiry( conn, now ); - //ulong margin = (ulong)(conn->rtt->smoothed_rtt) + (ulong)(3 * conn->rtt->var_rtt); - //if( margin < pkt_meta->expiry ) { - // pkt_meta->expiry -= margin; - //} - while( enc_level != ~0u ) { uint initial_pkt = 0; /* is this the first initial packet? */ + /* do we have space for pkt_meta? */ + if( !pkt_meta ) { + pkt_meta = fd_quic_pkt_meta_allocate( &conn->pkt_meta_pool ); + if( FD_UNLIKELY( !pkt_meta ) ) { + /* when there is no pkt_meta, it's best to keep processing acks + until some pkt_meta are returned */ + FD_DEBUG( FD_LOG_DEBUG(( "Failed to alloc pkt_meta" )); ) + quic->metrics.pkt_tx_alloc_fail_cnt++; + return; + } + } else { + /* reuse packet number */ + conn->pkt_number[pkt_meta->pn_space] = pkt_meta->pkt_number; + } + + *pkt_meta = (fd_quic_pkt_meta_t){0}; + + /* initialize expiry */ + pkt_meta->expiry = now + (ulong)500e6; + //ulong margin = (ulong)(conn->rtt->smoothed_rtt) + (ulong)(3 * conn->rtt->var_rtt); + //if( margin < pkt_meta->expiry ) { + // pkt_meta->expiry -= margin; + //} + + /* initialize tx_time */ + pkt_meta->tx_time = now; /* remaining in datagram */ /* invariant: tx_ptr >= tx_buf */ @@ -3554,15 +3515,13 @@ fd_quic_conn_tx( fd_quic_t * quic, cur_sz = fd_ulong_min( cur_sz, datagram_rem ); /* determine pn_space */ - uint pn_space = fd_quic_enc_level_to_pn_space( enc_level ); - pkt_meta_tmpl->pn_space = (uchar)pn_space; - pkt_meta_tmpl->enc_level = (uchar)(enc_level&0x3); + uint pn_space = fd_quic_enc_level_to_pn_space( enc_level ); /* get next packet number Returned to pool if not sent as gaps are harmful for ACK frame compression. */ ulong pkt_number = conn->pkt_number[pn_space]++; - FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number ); + /* are we the client initial packet? */ ulong hs_data_offset = conn->hs_sent_bytes[enc_level]; @@ -3610,7 +3569,6 @@ fd_quic_conn_tx( fd_quic_t * quic, hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial ); hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */ - FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id ); break; } @@ -3632,7 +3590,6 @@ fd_quic_conn_tx( fd_quic_t * quic, hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake ); hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */ - FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id ); break; } @@ -3648,7 +3605,6 @@ fd_quic_conn_tx( fd_quic_t * quic, one_rtt.pkt_num = pkt_number; hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt ); - FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num ); break; } @@ -3687,7 +3643,7 @@ fd_quic_conn_tx( fd_quic_t * quic, uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ; uchar * const frame_start = payload_ptr; - payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now ); + payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, enc_level, pkt_meta, pkt_number, now ); if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" )); /* did we add any frames? */ @@ -3769,6 +3725,29 @@ fd_quic_conn_tx( fd_quic_t * quic, conn->tx_ptr += cipher_text_sz; #endif + /* update packet metadata with summary info */ + pkt_meta->pkt_number = pkt_number; + pkt_meta->pn_space = (uchar)pn_space; + pkt_meta->enc_level = (uchar)enc_level; + + if( pkt_meta->flags & FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) { + pkt_meta->flags &= ~FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR; + } + + /* add to sent list */ + if( pkt_meta->flags ) { + fd_quic_pkt_meta_push_back( &conn->pkt_meta_pool.sent_pkt_meta[enc_level], pkt_meta ); + + /* update rescheduling variable */ + fd_quic_svc_prep_schedule( conn, pkt_meta->expiry ); + + /* clear pkt_meta for next loop */ + pkt_meta = NULL; + } else { + /* next iteration should skip the current packet number */ + pkt_meta->pkt_number++; + } + if( enc_level == fd_quic_enc_level_appdata_id ) { /* short header must be last in datagram so send in packet immediately */ @@ -3795,6 +3774,13 @@ fd_quic_conn_tx( fd_quic_t * quic, FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); ) } + /* unused pkt_meta? deallocate */ + if( FD_UNLIKELY( pkt_meta ) ) { + conn->pkt_number[pkt_meta->pn_space] = pkt_meta->pkt_number; + fd_quic_pkt_meta_deallocate( &conn->pkt_meta_pool, pkt_meta ); + pkt_meta = NULL; + } + /* try to send? */ fd_quic_tx_buffered( quic, conn ); } @@ -3804,16 +3790,6 @@ fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, ulong now ) { (void)now; conn->svc_meta.next_timeout = ULONG_MAX; - /* Send new rtt measurement probe? */ - if( FD_UNLIKELY(now > conn->last_ack + (ulong)conn->rtt->rtt_period_ticks) ) { - /* send PING */ - if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) - && conn->state == FD_QUIC_CONN_STATE_ACTIVE ) { - conn->flags |= FD_QUIC_CONN_FLAGS_PING; - conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */ - } - } - /* handle expiry on pkt_meta */ fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* enc_level */ ); @@ -3834,7 +3810,7 @@ fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, ulong now ) { conn->handshake_done_send = 1; /* move straight to ACTIVE */ - fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE ); + conn->state = FD_QUIC_CONN_STATE_ACTIVE; /* RFC 9001 4.9.2. Discarding Handshake Keys > An endpoint MUST discard its Handshake keys when the @@ -3876,7 +3852,7 @@ fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, ulong now ) { fd_quic_conn_tx( quic, conn ); /* schedule another fd_quic_conn_service to free the conn */ - fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */ + conn->state = FD_QUIC_CONN_STATE_DEAD; /* TODO need draining state wait for 3 * TPO */ quic->metrics.conn_closed_cnt++; break; @@ -3886,7 +3862,7 @@ fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, ulong now ) { fd_quic_conn_tx( quic, conn ); /* schedule another fd_quic_conn_service to free the conn */ - fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); + conn->state = FD_QUIC_CONN_STATE_DEAD; quic->metrics.conn_aborted_cnt++; break; @@ -3921,7 +3897,7 @@ fd_quic_conn_free( fd_quic_t * quic, } FD_COMPILER_MFENCE(); - fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID ); + conn->state = FD_QUIC_CONN_STATE_INVALID; FD_COMPILER_MFENCE(); fd_quic_state_t * state = fd_quic_get_state( quic ); @@ -3998,7 +3974,7 @@ fd_quic_conn_free( fd_quic_t * quic, /* put connection back in free list */ conn->free_conn_next = state->free_conn_list; state->free_conn_list = conn->conn_idx; - fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID ); + conn->state = FD_QUIC_CONN_STATE_INVALID; quic->metrics.conn_active_cnt--; @@ -4203,6 +4179,9 @@ fd_quic_conn_create( fd_quic_t * quic, conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id ); + /* initialize the pkt_meta pool with data */ + fd_quic_pkt_meta_pool_init( &conn->pkt_meta_pool, conn->pkt_meta_mem, quic->limits.inflight_pkt_cnt ); + /* rfc9000: s12.3: Packet numbers in each packet space start at 0. Subsequent packets sent in the same packet number space @@ -4224,7 +4203,7 @@ fd_quic_conn_create( fd_quic_t * quic, conn->key_phase = 0; conn->key_update = 0; - fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE ); + conn->state = FD_QUIC_CONN_STATE_HANDSHAKE; conn->reason = 0; conn->app_reason = 0; conn->flags = 0; @@ -4351,24 +4330,21 @@ fd_quic_pkt_meta_retry( fd_quic_t * quic, /* count of freed pkt_meta */ ulong cnt_freed = 0u; - fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker; - fd_quic_pkt_meta_t * pool = tracker->pool; - - /* used for metric tracking */ - ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul }; + /* obtain pointer to pkt_meta pool */ + fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool; while(1) { - /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */ + /* find earliest sent pkt_meta over all of the enc_levels */ uint enc_level = arg_enc_level; uint peer_enc_level = conn->peer_enc_level; ulong expiry = ~0ul; if( arg_enc_level == ~0u ) { for( uint j = 0u; j < 4u; ++j ) { - /* TODO this only checks smallest pkt number, - assuming that pkt numbers are monotonically increasing - over time. So it checks in 'sent' time order, but not expiry time. */ + /* TODO this only checks the head of each enc_level + assuming that pkt_meta is in time order. It IS + is time order, but not expiry time. */ #if 1 - fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool ); + fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head; if( !pkt_meta ) continue; if( enc_level == ~0u || pkt_meta->expiry < expiry ) { @@ -4389,7 +4365,7 @@ fd_quic_pkt_meta_retry( fd_quic_t * quic, #endif } } else { - fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool ); + fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[enc_level].head; if( !pkt_meta ) { return; } @@ -4415,35 +4391,55 @@ fd_quic_pkt_meta_retry( fd_quic_t * quic, return; }; - fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool ); + fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level]; + fd_quic_pkt_meta_t * pkt_meta = sent->head; + fd_quic_pkt_meta_t * prior = NULL; /* prior is always null, since we always look at head */ /* already moved to another enc_level */ if( enc_level < peer_enc_level ) { - cnt_freed += fd_quic_abandon_enc_level( conn, peer_enc_level ); + /* free pkt_meta */ + + /* treat the original packet as-if it were ack'ed */ + fd_quic_reclaim_pkt_meta( conn, + pkt_meta, + enc_level ); + + /* remove from list */ + fd_quic_pkt_meta_remove( sent, prior, pkt_meta ); + + /* put pkt_meta back in free list */ + fd_quic_pkt_meta_deallocate( pool, pkt_meta ); + + cnt_freed++; + continue; } - quic->metrics.pkt_retransmissions_cnt += !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]); - prev_retx_pkt_num[enc_level] = pkt_meta->key.pkt_num; + quic->metrics.pkt_retransmissions_cnt++; - FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, (ulong)pkt_meta->key.pkt_num, pkt_meta->expiry, (uchar)pkt_meta->key.type); + FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, pkt_meta->pkt_number, pkt_meta->expiry, pkt_meta->flags); /* set the data to retry */ - uint type = pkt_meta->key.type; - switch( type ) { - case FD_QUIC_PKT_META_TYPE_HS_DATA: - do { - ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo ); - if( offset < conn->hs_sent_bytes[enc_level] ) { - conn->hs_sent_bytes[enc_level] = offset; - conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; - } - } while(0); - break; + uint flags = pkt_meta->flags; + if( flags & FD_QUIC_PKT_META_FLAGS_HS_DATA ) { + /* find handshake data to retry */ + /* reset offset to beginning of retried range if necessary */ + ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->range.offset_lo ); + if( offset < conn->hs_sent_bytes[enc_level] ) { + conn->hs_sent_bytes[enc_level] = offset; + conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; + } + } + if( flags & FD_QUIC_PKT_META_FLAGS_STREAM ) { + /* iterate thru the variable section of the pkt_meta + * and set the max_stream_data to resend for each + * appropriate entry */ + ulong var_sz = pkt_meta->var_sz; - case FD_QUIC_PKT_META_TYPE_STREAM: - do { - ulong stream_id = pkt_meta->key.stream_id; + /* probably we should consolidate these loops over pkt_meta->var[j] */ + for( ulong j = 0UL; j < var_sz; ++j ) { + if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_STREAM_DATA ) { + ulong stream_id = FD_QUIC_PKT_META_STREAM_ID( pkt_meta->var[j].key ); /* find the stream */ fd_quic_stream_t * stream = NULL; @@ -4453,7 +4449,7 @@ fd_quic_pkt_meta_retry( fd_quic_t * quic, stream = stream_entry->stream; /* do not try sending data that has been acked */ - ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->tx_buf.tail ); + ulong offset = fd_ulong_max( pkt_meta->var[j].range.offset_lo, stream->tx_buf.tail ); /* any data left to retry? */ stream->tx_sent = fd_ulong_min( stream->tx_sent, offset ); @@ -4474,59 +4470,56 @@ fd_quic_pkt_meta_retry( fd_quic_t * quic, fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END ); } } - } while(0); - break; - - case FD_QUIC_PKT_META_TYPE_HS_DONE: - if( FD_LIKELY( !conn->handshake_done_ackd ) ) { - conn->handshake_done_send = 1; - conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; - } - break; - - case FD_QUIC_PKT_META_TYPE_MAX_DATA: - if( srx->rx_max_data_ackd < srx->rx_max_data ) { - conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA; - conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; } - break; - - case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR: - do { - /* do we still need to send? */ - /* get required value */ - ulong max_streams_unidir = srx->rx_sup_stream_id >> 2; - - if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) { - /* set the data to go out on the next packet */ - conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR; - conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; - } - } while(0); - break; - - case FD_QUIC_PKT_META_TYPE_CLOSE: - conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT; - conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; - break; - - case FD_QUIC_PKT_META_TYPE_PING: - conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT ) - | FD_QUIC_CONN_FLAGS_PING; + } + } + if( flags & FD_QUIC_PKT_META_FLAGS_HS_DONE ) { + /* do we need to resend the handshake done flag? + only send if it hasn't already been acked */ + if( FD_LIKELY( !conn->handshake_done_ackd ) ) { + conn->handshake_done_send = 1; + conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; + } + } + if( flags & FD_QUIC_PKT_META_FLAGS_MAX_DATA ) { + /* set max_data to be sent only if unacked */ + if( srx->rx_max_data_ackd < srx->rx_max_data ) { + conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA; conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; - break; + } + } + if( flags & FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) { + /* do we still need to send? */ + /* get required value */ + ulong max_streams_unidir = srx->rx_sup_stream_id >> 2; + + if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) { + /* set the data to go out on the next packet */ + conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR; + conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; + } + } + if( flags & FD_QUIC_PKT_META_FLAGS_CLOSE ) { + conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT; + conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; + } + if( flags & FD_QUIC_PKT_META_FLAGS_PING ) { + conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT ) + | FD_QUIC_CONN_FLAGS_PING; + conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; } /* reschedule to ensure the data gets processed */ fd_quic_svc_prep_schedule_now( conn ); /* free pkt_meta */ - fd_quic_pkt_meta_remove_range( &tracker->sent_pkt_metas[enc_level], - pool, - pkt_meta->key.pkt_num, - pkt_meta->key.pkt_num ); - conn->used_pkt_meta -= 1; + /* remove from list */ + fd_quic_pkt_meta_remove( sent, prior, pkt_meta ); + + /* put pkt_meta back in free list */ + fd_quic_pkt_meta_deallocate( pool, pkt_meta ); + cnt_freed++; } } @@ -4537,110 +4530,123 @@ void fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn, fd_quic_pkt_meta_t * pkt_meta, uint enc_level ) { + fd_quic_conn_stream_rx_t * srx = conn->srx; - uint type = pkt_meta->key.type; - fd_quic_range_t range = pkt_meta->val.range; + uint flags = pkt_meta->flags; + fd_quic_range_t range = pkt_meta->range; - switch( type ) { + if( flags & FD_QUIC_PKT_META_FLAGS_PING ) { + conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ); + } - case FD_QUIC_PKT_META_TYPE_PING: - do { - conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ); - } while(0); - break; + if( flags & FD_QUIC_PKT_META_FLAGS_HS_DATA ) { + /* Note that tls_hs could already be freed */ + /* is this ack'ing the next consecutive bytes? + if so, we can increase the ack'd bytes + if not, we retransmit the bytes expected to be ack'd + we assume a gap means a dropped packet, and + this policy allows us to free up the pkt_meta here */ + ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]; + if( range.offset_lo <= hs_ackd_bytes ) { + hs_ackd_bytes = conn->hs_ackd_bytes[enc_level] + = fd_ulong_max( hs_ackd_bytes, range.offset_hi ); - case FD_QUIC_PKT_META_TYPE_HS_DATA: - do { - /* Note that tls_hs could already be freed */ - /* is this ack'ing the next consecutive bytes? - if so, we can increase the ack'd bytes - if not, we retransmit the bytes expected to be ack'd - we assume a gap means a dropped packet, and - this policy allows us to free up the pkt_meta here */ - ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]; - if( range.offset_lo <= hs_ackd_bytes ) { - hs_ackd_bytes = conn->hs_ackd_bytes[enc_level] - = fd_ulong_max( hs_ackd_bytes, range.offset_hi ); - - /* remove any unused hs_data */ - fd_quic_tls_hs_data_t * hs_data = NULL; - - hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level ); - while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) { - fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level ); - hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level ); - } - } else { - conn->hs_sent_bytes[enc_level] = - fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes ); - conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; - } - } while(0); - break; + /* remove any unused hs_data */ + fd_quic_tls_hs_data_t * hs_data = NULL; - case FD_QUIC_PKT_META_TYPE_HS_DONE: - do { - conn->handshake_done_ackd = 1; - conn->handshake_done_send = 0; - if( FD_LIKELY( conn->tls_hs ) ) { - fd_quic_state_t * state = fd_quic_get_state( conn->quic ); - fd_quic_tls_hs_delete( conn->tls_hs ); - fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool ); - fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs ); - conn->tls_hs = NULL; - } - } while(0); - break; + hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level ); + while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) { + fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level ); + hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level ); + } + } else { + conn->hs_sent_bytes[enc_level] = + fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes ); + conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; + } + } - case FD_QUIC_PKT_META_TYPE_MAX_DATA: - do { - ulong max_data_ackd = pkt_meta->val.scalar; + if( flags & FD_QUIC_PKT_META_FLAGS_HS_DONE ) { + conn->handshake_done_ackd = 1; + conn->handshake_done_send = 0; + if( FD_LIKELY( conn->tls_hs ) ) { + fd_quic_state_t * state = fd_quic_get_state( conn->quic ); + fd_quic_tls_hs_delete( conn->tls_hs ); + /* Remove the handshake from the cache before releasing it */ + fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool ); + fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs ); + conn->tls_hs = NULL; + } + } - /* ack can only increase max_data_ackd */ - max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd ); + if( flags & FD_QUIC_PKT_META_FLAGS_MAX_DATA ) { + ulong max_data_ackd = 0UL; + for( ulong j = 0UL; j < pkt_meta->var_sz; ++j ) { + if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_OTHER && + pkt_meta->var[j].key.flags == FD_QUIC_PKT_META_FLAGS_MAX_DATA ) { + max_data_ackd = pkt_meta->var[j].value; + break; + } + } - /* max_data_ackd > rx_max_data is a protocol violation */ - if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) { - /* this is a protocol violation, so inform the peer */ - fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ ); - return; - } + /* ack can only increase max_data_ackd */ + max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd ); - /* clear flag only if acked value == current value */ - if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) { - conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA; - } + /* max_data_ackd > rx_max_data is a protocol violation */ + if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) { + /* this is a protocol violation, so inform the peer */ + fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ ); + return; + } - /* set the ackd value */ - srx->rx_max_data_ackd = max_data_ackd; - } while(0); - break; + /* clear flag only if acked value == current value */ + if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) { + conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA; + } + + /* set the ackd value */ + srx->rx_max_data_ackd = max_data_ackd; + } - case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR: - do { - ulong max_streams_unidir_ackd = pkt_meta->val.scalar; + if( flags & FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) { + ulong var_sz = pkt_meta->var_sz; - /* ack can only increase max_streams_unidir_ackd */ - max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd ); + /* find the value ackd */ + ulong max_streams_unidir_ackd = 0UL; + for( ulong j = 0UL; j < var_sz; ++j ) { + if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_OTHER && + pkt_meta->var[j].key.flags == FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) { + max_streams_unidir_ackd = pkt_meta->var[j].value; + break; + } + } - /* get required value */ - ulong max_streams_unidir = srx->rx_sup_stream_id >> 2; + /* ack can only increase max_streams_unidir_ackd */ + max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd ); - /* clear flag only if acked value == current value */ - if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) { - conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR; - } + /* get required value */ + ulong max_streams_unidir = srx->rx_sup_stream_id >> 2; - /* set the ackd value */ - srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd; - } while(0); - break; + /* clear flag only if acked value == current value */ + if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) { + conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR; + } - case FD_QUIC_PKT_META_TYPE_STREAM: - do { - ulong stream_id = pkt_meta->key.stream_id; - fd_quic_range_t range = pkt_meta->val.range; + /* set the ackd value */ + srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd; + } + + if( flags & FD_QUIC_PKT_META_FLAGS_STREAM ) { + /* iterate thru the variable section of the pkt_meta + * and set the max_stream_data to resend for each + * appropriate entry */ + ulong var_sz = pkt_meta->var_sz; + + /* probably we should consolidate these loops over pkt_meta->var[j] */ + for( ulong j = 0UL; j < var_sz; ++j ) { + if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_STREAM_DATA ) { + ulong stream_id = FD_QUIC_PKT_META_STREAM_ID( pkt_meta->var[j].key ); /* find the stream */ fd_quic_stream_t * stream = NULL; @@ -4650,6 +4656,7 @@ fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn, stream = stream_entry->stream; /* do not try sending data that has been acked */ + fd_quic_range_t range = pkt_meta->var[j].range; ulong tx_tail = stream->tx_buf.tail; ulong tx_sent = stream->tx_sent; @@ -4732,7 +4739,7 @@ fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn, uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN; /* move up tail, and adjust to maintain circular queue invariants, and send - max_data and max_stream_data, if necessary */ + max_data and max_stream_data, if necessary */ if( tx_tail > stream->tx_buf.tail ) { stream->tx_buf.tail = tx_tail; @@ -4764,8 +4771,8 @@ fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn, /* we could retransmit (timeout) the bytes which have not been acked (by implication) */ } } - } while(0); - break; + } + } } } /* process lost packets @@ -4773,31 +4780,31 @@ fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn, void fd_quic_process_lost( fd_quic_conn_t * conn, uint enc_level, ulong cnt ) { /* start at oldest sent */ - fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker; - fd_quic_pkt_meta_t * pool = tracker->pool; - fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level]; - ulong j = 0; - - for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool ); - !fd_quic_pkt_meta_ds_fwd_iter_done( iter ); - iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) { - fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool ); + fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool; + fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level]; + fd_quic_pkt_meta_t * pkt_meta = sent->head; + + ulong j = 0; + + while( FD_LIKELY( pkt_meta ) ) { if( FD_LIKELY( j < cnt ) ) { pkt_meta->expiry = 0; /* force expiry */ + pkt_meta = pkt_meta->next; } else { break; } + j++; } - /* trigger the retries */ + /* do the processing */ fd_quic_pkt_meta_retry( conn->quic, conn, 0 /* don't force */, enc_level ); } /* process ack range applies to pkt_number in [largest_ack - ack_range, largest_ack] */ void -fd_quic_process_ack_range( fd_quic_conn_t * conn, +fd_quic_process_ack_range( fd_quic_conn_t * conn, fd_quic_frame_ctx_t * context, uint enc_level, ulong largest_ack, @@ -4805,6 +4812,7 @@ fd_quic_process_ack_range( fd_quic_conn_t * conn, int is_largest, ulong now, ulong ack_delay ) { + /* FIXME: This would benefit from algorithmic improvements */ /* FIXME: Close connection if peer ACKed a higher packet number than we sent */ fd_quic_pkt_t * pkt = context->pkt; @@ -4814,34 +4822,62 @@ fd_quic_process_ack_range( fd_quic_conn_t * conn, ulong lo = largest_ack - ack_range; FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi ); - fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker; - fd_quic_pkt_meta_t * pool = tracker->pool; - fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level]; - /* start at oldest sent */ - for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool ); - !fd_quic_pkt_meta_ds_fwd_iter_done( iter ); - iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) { - fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool ); - if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break; - if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) { - pkt->rtt_pkt_number = hi; - pkt->rtt_ack_time = now - e->tx_time; /* in ticks */ - pkt->rtt_ack_delay = ack_delay; /* in peer units */ + fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool; + fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level]; + fd_quic_pkt_meta_t * pkt_meta = sent->head; + fd_quic_pkt_meta_t * prior = NULL; + + while( pkt_meta ) { + if( FD_UNLIKELY( pkt_meta->pkt_number < lo ) ) { + /* go to next, keeping track of prior */ + prior = pkt_meta; + pkt_meta = pkt_meta->next; + continue; } - fd_quic_reclaim_pkt_meta( conn, e, enc_level ); - } - conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi ); + /* keep pkt_meta->next for later */ + fd_quic_pkt_meta_t * pkt_meta_next = pkt_meta->next; + + /* packet number is in range, so reclaim the resources */ + if( pkt_meta->pkt_number <= hi ) { + + /* note: rtt_pkt_number is zero when unused, so using >= for the test */ + if( is_largest && pkt_meta->pkt_number == hi && hi >= pkt->rtt_pkt_number ) { + pkt->rtt_pkt_number = hi; + pkt->rtt_ack_time = now - pkt_meta->tx_time; /* in ticks */ + pkt->rtt_ack_delay = ack_delay; /* in peer units */ + } + + fd_quic_reclaim_pkt_meta( conn, + pkt_meta, + enc_level ); + + /* remove from list */ + fd_quic_pkt_meta_remove( sent, prior, pkt_meta ); + + /* put pkt_meta back in free list */ + fd_quic_pkt_meta_deallocate( pool, pkt_meta ); + + /* we removed one, so keep prior the same and move pkt_meta up */ + pkt_meta = pkt_meta_next; + + continue; + } + + /* pkt_number > hi, so break */ + break; + } } static ulong -fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context, - fd_quic_ack_frame_t * data, - uchar const * p, - ulong p_sz ) { - fd_quic_conn_t * conn = context->conn; - uint enc_level = context->pkt->enc_level; +fd_quic_handle_ack_frame( + fd_quic_frame_ctx_t * context, + fd_quic_ack_frame_t * data, + uchar const * p, + ulong p_sz ) { + fd_quic_conn_t * conn = context->conn; + uint enc_level = context->pkt->enc_level; if( FD_UNLIKELY( data->first_ack_range > data->largest_ack ) ) { /* this is a protocol violation, so inform the peer */ @@ -4940,19 +4976,15 @@ fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context, /* process lost packets */ { - fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker; - fd_quic_pkt_meta_t * pool = tracker->pool; - fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level]; - fd_quic_pkt_meta_t * min_meta = fd_quic_pkt_meta_min( sent, pool ); - - if( FD_UNLIKELY( min_meta && min_meta->key.pkt_num < low_ack_pkt_number ) ) { - ulong skipped = 0; - for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool ); - !fd_quic_pkt_meta_ds_fwd_iter_done( iter ); - iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) { - fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool ); - if( FD_UNLIKELY( e->key.pkt_num >= low_ack_pkt_number ) ) break; + fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool; + fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level]; + fd_quic_pkt_meta_t * pkt_meta = sent->head; + + if( FD_UNLIKELY( pkt_meta && pkt_meta->pkt_number < low_ack_pkt_number ) ) { + ulong skipped = 0UL; + while( FD_LIKELY( pkt_meta && pkt_meta->pkt_number < low_ack_pkt_number ) ) { skipped++; + pkt_meta = pkt_meta->next; } if( FD_UNLIKELY( skipped > 3 ) ) { @@ -5296,7 +5328,7 @@ fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) { return; default: - fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE ); + conn->state = FD_QUIC_CONN_STATE_PEER_CLOSE; } conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; @@ -5422,7 +5454,7 @@ fd_quic_handle_handshake_done_frame( } /* we shouldn't be receiving this unless handshake is complete */ - fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE ); + conn->state = FD_QUIC_CONN_STATE_ACTIVE; /* user callback */ fd_quic_cb_conn_hs_complete( conn->quic, conn ); @@ -5454,7 +5486,7 @@ fd_quic_conn_close( fd_quic_conn_t * conn, default: { - fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING ); + conn->state = FD_QUIC_CONN_STATE_CLOSE_PENDING; conn->app_reason = app_reason; } } diff --git a/src/waltz/quic/fd_quic.h b/src/waltz/quic/fd_quic.h index af0a8303e3..d3f9a453f9 100644 --- a/src/waltz/quic/fd_quic.h +++ b/src/waltz/quic/fd_quic.h @@ -111,10 +111,9 @@ struct __attribute__((aligned(16UL))) fd_quic_limits { ulong handshake_cnt; /* instance-wide, max concurrent handshake count */ ulong log_depth; /* instance-wide, depth of shm log cache */ - ulong conn_id_cnt; /* per-conn, max conn ID count (min 4UL) */ - ulong stream_id_cnt; /* per-conn, max concurrent stream ID count */ - ulong inflight_frame_cnt; /* instance-wide, total max inflight frame count */ - ulong min_inflight_frame_cnt_conn; /* per-conn, min inflight frame count */ + ulong conn_id_cnt; /* per-conn, max conn ID count (min 4UL) */ + ulong stream_id_cnt; /* per-conn, max concurrent stream ID count */ + ulong inflight_pkt_cnt; /* instance-wide, total max inflight packet count */ ulong tx_buf_sz; /* per-stream, tx buf sz in bytes */ /* the user consumes rx directly from the network buffer */ @@ -136,7 +135,6 @@ struct fd_quic_layout { ulong hs_pool_off; /* offset of the handshake pool */ ulong stream_pool_off; /* offset of the stream pool */ ulong svc_timers_off; /* offset of the service timers */ - ulong pkt_meta_pool_off; /* offset of the pkt_meta pool */ }; typedef struct fd_quic_layout fd_quic_layout_t; diff --git a/src/waltz/quic/fd_quic_conn.c b/src/waltz/quic/fd_quic_conn.c index 1cf46d050d..4cd91fb3ee 100644 --- a/src/waltz/quic/fd_quic_conn.c +++ b/src/waltz/quic/fd_quic_conn.c @@ -17,12 +17,16 @@ struct fd_quic_conn_layout { int stream_map_lg; ulong stream_map_off; + ulong pkt_meta_off; }; typedef struct fd_quic_conn_layout fd_quic_conn_layout_t; +/* TODO maybe introduce a separate parameter for size of pkt_meta + pool? */ ulong fd_quic_conn_align( void ) { ulong align = fd_ulong_max( alignof( fd_quic_conn_t ), alignof( fd_quic_stream_t ) ); + align = fd_ulong_max( align, alignof( fd_quic_pkt_meta_t ) ); align = fd_ulong_max( align, fd_quic_stream_map_align() ); return align; } @@ -31,6 +35,9 @@ static ulong fd_quic_conn_footprint_ext( fd_quic_limits_t const * limits, fd_quic_conn_layout_t * layout ) { + ulong inflight_pkt_cnt = limits->inflight_pkt_cnt; + if( FD_UNLIKELY( inflight_pkt_cnt==0UL ) ) return 0UL; + ulong stream_id_cnt = limits->stream_id_cnt; ulong off = 0; @@ -54,7 +61,12 @@ fd_quic_conn_footprint_ext( fd_quic_limits_t const * limits, layout->stream_map_off = 0UL; } - return fd_ulong_align_up( off, fd_quic_conn_align() ); + /* allocate space for packet metadata */ + off = fd_ulong_align_up( off, alignof(fd_quic_pkt_meta_t) ); + layout->pkt_meta_off = off; + off += inflight_pkt_cnt * sizeof(fd_quic_pkt_meta_t); + + return off; } ulong @@ -104,7 +116,7 @@ fd_quic_conn_new( void * mem, fd_memset( conn, 0, sizeof(fd_quic_conn_t) ); conn->quic = quic; - fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID ); + conn->state = FD_QUIC_CONN_STATE_INVALID; /* Initialize streams */ @@ -119,12 +131,14 @@ fd_quic_conn_new( void * mem, if( FD_UNLIKELY( !conn->stream_map ) ) return NULL; } - /* Initialize packet meta tracker */ - fd_quic_state_t * state = fd_quic_get_state( quic ); - fd_quic_pkt_meta_tracker_init( &conn->pkt_meta_tracker, - quic->limits.inflight_frame_cnt, - state->pkt_meta_pool ); + /* Initialize packet meta pool */ + + ulong pkt_meta_cnt = limits->inflight_pkt_cnt; + fd_quic_pkt_meta_t * pkt_meta = (fd_quic_pkt_meta_t *)( (ulong)mem + layout.pkt_meta_off ); + fd_memset( pkt_meta, 0, pkt_meta_cnt*sizeof(fd_quic_pkt_meta_t) ); + /* store pointer to storage and size */ + conn->pkt_meta_mem = pkt_meta; /* Initialize service timers */ fd_quic_svc_timers_init_conn( conn ); diff --git a/src/waltz/quic/fd_quic_conn.h b/src/waltz/quic/fd_quic_conn.h index c48bb5077f..8ffd19347c 100644 --- a/src/waltz/quic/fd_quic_conn.h +++ b/src/waltz/quic/fd_quic_conn.h @@ -201,7 +201,9 @@ struct fd_quic_conn { ulong unacked_sz; /* Number of received stream frame payload bytes pending ACK */ /* Resets to zero when conn is rescheduled or ACKs are sent */ - fd_quic_pkt_meta_tracker_t pkt_meta_tracker; + /* TODO find better name than pool */ + fd_quic_pkt_meta_pool_t pkt_meta_pool; + fd_quic_pkt_meta_t * pkt_meta_mem; /* owns the memory */ /* flow control */ ulong tx_max_data; /* the limit on the number of bytes we are allowed @@ -243,8 +245,6 @@ struct fd_quic_conn { uchar token[ FD_QUIC_RETRY_MAX_TOKEN_SZ ]; fd_quic_conn_stream_rx_t srx[1]; - - ulong used_pkt_meta; }; inline static void diff --git a/src/waltz/quic/fd_quic_pkt_meta.c b/src/waltz/quic/fd_quic_pkt_meta.c index 5512e01445..4151246c4e 100644 --- a/src/waltz/quic/fd_quic_pkt_meta.c +++ b/src/waltz/quic/fd_quic_pkt_meta.c @@ -1,76 +1,108 @@ #include "fd_quic_pkt_meta.h" -void * -fd_quic_pkt_meta_tracker_init( fd_quic_pkt_meta_tracker_t * tracker, - ulong total_meta_cnt, - fd_quic_pkt_meta_t * pool ) { - for( ulong enc_level=0; enc_level<4; enc_level++ ) { - void* mem = fd_quic_pkt_meta_treap_new( &tracker->sent_pkt_metas[enc_level], - total_meta_cnt ); - mem = fd_quic_pkt_meta_treap_join( mem ); - if( FD_UNLIKELY( !mem ) ) return NULL; +/* initialize pool with existing array of pkt_meta */ +void +fd_quic_pkt_meta_pool_init( fd_quic_pkt_meta_pool_t * pool, + fd_quic_pkt_meta_t * pkt_meta_array, + ulong pkt_meta_array_sz ) { + /* initialize all to zeros */ + fd_memset( pool, 0, sizeof( *pool ) ); + + /* free list */ + fd_quic_pkt_meta_list_t * free = &pool->free; + + /* initialize free list of packet metadata */ + for( ulong j = 0; j < pkt_meta_array_sz; ++j ) { + fd_quic_pkt_meta_push_back( free, &pkt_meta_array[j] ); } - tracker->pool = pool; - return tracker; } + +/* pop from front of list */ +fd_quic_pkt_meta_t * +fd_quic_pkt_meta_pop_front( fd_quic_pkt_meta_list_t * list ) { + fd_quic_pkt_meta_t * front = list->head; + if( front ) { + list->head = front->next; + } + return front; +} + + +/* push onto front of list */ void -fd_quic_pkt_meta_ds_init_pool( fd_quic_pkt_meta_t * pool, - ulong total_meta_cnt ) { - fd_quic_pkt_meta_treap_seed( pool, total_meta_cnt, (ulong)fd_log_wallclock() ); +fd_quic_pkt_meta_push_front( fd_quic_pkt_meta_list_t * list, + fd_quic_pkt_meta_t * pkt_meta ) { + pkt_meta->next = list->head; + list->head = pkt_meta; } + +/* push onto back of list */ void -fd_quic_pkt_meta_insert( fd_quic_pkt_meta_ds_t * ds, - fd_quic_pkt_meta_t * pkt_meta, - fd_quic_pkt_meta_t * pool ) { - fd_quic_pkt_meta_treap_ele_insert( ds, pkt_meta, pool ); +fd_quic_pkt_meta_push_back( fd_quic_pkt_meta_list_t * list, + fd_quic_pkt_meta_t * pkt_meta ) { + fd_quic_pkt_meta_t * tail = list->tail; + if( tail ) { + tail->next = pkt_meta; + list->tail = pkt_meta; + } else { + list->head = list->tail = pkt_meta; + } + + pkt_meta->next = NULL; } +/* remove from list + requires the prior element */ +void +fd_quic_pkt_meta_remove( fd_quic_pkt_meta_list_t * list, + fd_quic_pkt_meta_t * pkt_meta_prior, + fd_quic_pkt_meta_t * pkt_meta ) { + fd_quic_pkt_meta_t * pkt_meta_next = pkt_meta->next; -ulong -fd_quic_pkt_meta_remove_range( fd_quic_pkt_meta_ds_t * ds, - fd_quic_pkt_meta_t * pool, - ulong pkt_number_lo, - ulong pkt_number_hi ) { - - fd_quic_pkt_meta_ds_fwd_iter_t l_iter = fd_quic_pkt_meta_ds_idx_ge( ds, pkt_number_lo, pool ); - fd_quic_pkt_meta_t * prev = NULL; - ulong cnt_removed = 0; - - for( fd_quic_pkt_meta_ds_fwd_iter_t iter = l_iter; - !fd_quic_pkt_meta_ds_fwd_iter_done( iter ); - iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) { - fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool ); - if( FD_UNLIKELY( e->key.pkt_num > pkt_number_hi ) ) break; - if( FD_LIKELY( prev ) ) { - fd_quic_pkt_meta_treap_ele_remove( ds, prev, pool ); - fd_quic_pkt_meta_pool_ele_release( pool, prev ); - cnt_removed++; + if( pkt_meta_prior == NULL ) { + if( pkt_meta_next == NULL ) { + /* at tail... then head = tail = NULL */ + list->head = list->tail = NULL; + } else { + /* at head... move it to next */ + list->head = pkt_meta_next; } - prev = e; - } - if( FD_LIKELY( prev ) ) { - fd_quic_pkt_meta_treap_ele_remove( ds, prev, pool ); - fd_quic_pkt_meta_pool_ele_release( pool, prev ); - cnt_removed++; + } else { + if( pkt_meta_next == NULL ) { + /* we're removing the last, so move tail */ + list->tail = pkt_meta_prior; + } + + /* not head, make pkt_meta_prior point to next */ + pkt_meta_prior->next = pkt_meta_next; } - return cnt_removed; + + pkt_meta->next = NULL; } -fd_quic_pkt_meta_t * -fd_quic_pkt_meta_min( fd_quic_pkt_meta_ds_t * ds, - fd_quic_pkt_meta_t * pool ) { - fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( ds, pool ); - if( FD_UNLIKELY( fd_quic_pkt_meta_ds_fwd_iter_done( iter ) ) ) return NULL; - return fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool ); +/* allocate a pkt_meta + obtains a free pkt_meta from the free list, and returns it + returns NULL if none is available */ +fd_quic_pkt_meta_t * +fd_quic_pkt_meta_allocate( fd_quic_pkt_meta_pool_t * pool ) { + fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pop_front( &pool->free ); + if( FD_LIKELY( pkt_meta ) ) { + fd_memset( pkt_meta, 0, sizeof( *pkt_meta ) ); + } + return pkt_meta; } + +/* free a pkt_meta + returns a pkt_meta to the free list, ready to be allocated again */ void -fd_quic_pkt_meta_ds_clear( fd_quic_pkt_meta_tracker_t * tracker, - uint enc_level ) { - ulong ele_max = fd_quic_pkt_meta_treap_ele_max( &tracker->sent_pkt_metas[enc_level] ); - fd_quic_pkt_meta_treap_new( &tracker->sent_pkt_metas[enc_level], ele_max ); +fd_quic_pkt_meta_deallocate( fd_quic_pkt_meta_pool_t * pool, fd_quic_pkt_meta_t * pkt_meta ) { + /* pushing to the front should help cache usage */ + fd_quic_pkt_meta_push_front( &pool->free, pkt_meta ); } + + diff --git a/src/waltz/quic/fd_quic_pkt_meta.h b/src/waltz/quic/fd_quic_pkt_meta.h index 0f0ee92181..76af69ec4b 100644 --- a/src/waltz/quic/fd_quic_pkt_meta.h +++ b/src/waltz/quic/fd_quic_pkt_meta.h @@ -3,58 +3,60 @@ #include "fd_quic_common.h" -typedef struct fd_quic_pkt_meta fd_quic_pkt_meta_t; -typedef struct fd_quic_pkt_meta_list fd_quic_pkt_meta_list_t; -typedef struct fd_quic_pkt_meta_tracker fd_quic_pkt_meta_tracker_t; +typedef struct fd_quic_pkt_meta fd_quic_pkt_meta_t; +typedef struct fd_quic_pkt_meta_list fd_quic_pkt_meta_list_t; +typedef struct fd_quic_pkt_meta_pool fd_quic_pkt_meta_pool_t; -/* fd_quic_pkt_meta_key used as key for tracking sent frames +/* TODO convert to a union with various types of metadata overlaid */ + +/* fd_quic_pkt_meta_var used for tracking max_data, max_stream_data and + * max_streams * - * pkt_num: packet number that carried this data - * type: type of data for retx (~frame type) - * stream_id: if stream type, the stream id + * type: FD_QUIC_PKT_META_TYPE_STREAM_DATA + * FD_QUIC_PKT_META_TYPE_OTHER + * flags: FD_QUIC_PKT_META_FLAGS_* + * value: max_data number of bytes + * max_stream_data number of bytes + * max_streams number of streams */ - union fd_quic_pkt_meta_key { - struct { - /* which frame type is recorded: - FD_QUIC_PKT_META_TYPE_HS_DATA handshake data - FD_QUIC_PKT_META_TYPE_STREAM stream data - FD_QUIC_PKT_META_TYPE_HS_DONE handshake-done frame - FD_QUIC_PKT_META_TYPE_MAX_DATA max_data frame - FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR max_streams frame (unidir) - FD_QUIC_PKT_META_TYPE_CLOSE close frame - FD_QUIC_PKT_META_TYPE_PING set to send a PING frame - */ - # define FD_QUIC_PKT_META_TYPE_HS_DATA (0) - # define FD_QUIC_PKT_META_TYPE_STREAM (1) - # define FD_QUIC_PKT_META_TYPE_HS_DONE (2) - # define FD_QUIC_PKT_META_TYPE_MAX_DATA (3) - # define FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR (4) - # define FD_QUIC_PKT_META_TYPE_CLOSE (5) - # define FD_QUIC_PKT_META_TYPE_PING (6) - uchar type: 4; - - ulong pkt_num: 60; - #define FD_QUIC_PKT_META_SET_TYPE(PKT_META_PTR, TYPE) \ - (PKT_META_PTR)->key.type = (uchar)((TYPE)&0x0f) - - #define FD_QUIC_PKT_META_PKT_NUM_MASK ( (1UL<<60) - 1 ) - #define FD_QUIC_PKT_META_SET_PKT_NUM(PKT_META_PTR, PKT_NUM) \ - (PKT_META_PTR)->key.pkt_num = (PKT_NUM)&FD_QUIC_PKT_META_PKT_NUM_MASK - + union { +#define FD_QUIC_PKT_META_STREAM_MASK ((1UL<<62UL)-1UL) ulong stream_id; + struct { + ulong flags:62; + ulong type:2; +#define FD_QUIC_PKT_META_TYPE_OTHER 0UL +#define FD_QUIC_PKT_META_TYPE_STREAM_DATA 1UL + }; +#define FD_QUIC_PKT_META_KEY( TYPE, FLAGS, STREAM_ID ) \ + ((fd_quic_pkt_meta_key_t) \ + { .stream_id = ( ( (ulong)(STREAM_ID) ) | \ + ( (ulong)(TYPE) << 62UL ) | \ + ( (ulong)(FLAGS) ) ) } ) + /* FD_QUIC_PKT_META_STREAM_ID + * This is used to extract the stream_id, since some of the bits are used + * for "type". + * The more natural way "stream_id:62" caused compilation warnings and ugly + * work-arounds */ +#define FD_QUIC_PKT_META_STREAM_ID( KEY ) ( (KEY).stream_id & FD_QUIC_PKT_META_STREAM_MASK ) }; - ulong b[2]; }; typedef union fd_quic_pkt_meta_key fd_quic_pkt_meta_key_t; -FD_STATIC_ASSERT( sizeof(fd_quic_pkt_meta_key_t) == 16, fd_quic_pkt_meta_key_t ); -union fd_quic_pkt_meta_value { - ulong scalar; - fd_quic_range_t range; +struct fd_quic_pkt_meta_var { + fd_quic_pkt_meta_key_t key; + union { + ulong value; + fd_quic_range_t range; + }; }; -typedef union fd_quic_pkt_meta_value fd_quic_pkt_meta_value_t; +typedef struct fd_quic_pkt_meta_var fd_quic_pkt_meta_var_t; +/* the max number of pkt_meta_var entries in pkt_meta + this limits the number of max_data, max_stream_data and max_streams + allowed in a single quic packet */ +#define FD_QUIC_PKT_META_VAR_MAX 16 /* fd_quic_pkt_meta @@ -62,220 +64,103 @@ typedef union fd_quic_pkt_meta_value fd_quic_pkt_meta_value_t; used when acks arrive to determine what is being acked specifically */ struct fd_quic_pkt_meta { /* stores metadata about what was sent in the identified packet */ - fd_quic_pkt_meta_key_t key; - fd_quic_pkt_meta_value_t val; - uchar enc_level: 2; - uchar pn_space; /* packet number space (derived from enc_level) */ - ulong tx_time; /* transmit time */ - ulong expiry; /* time pkt_meta expires... this is the time the + ulong pkt_number; /* packet number (in pn_space) */ + uchar enc_level; /* encryption level of packet */ + uchar pn_space; /* packet number space (derived from enc_level) */ + uchar var_sz; /* number of populated entries in var */ + + /* does/should the referenced packet contain: + FD_QUIC_PKT_META_FLAGS_HS_DATA handshake data + FD_QUIC_PKT_META_FLAGS_STREAM stream data + FD_QUIC_PKT_META_FLAGS_HS_DONE handshake-done frame + FD_QUIC_PKT_META_FLAGS_MAX_DATA max_data frame + FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR max_streams frame (unidir) + FD_QUIC_PKT_META_FLAGS_CLOSE close frame + FD_QUIC_PKT_META_FLAGS_PING set to send a PING frame + + some of these flags are mutually exclusive */ + uint flags; /* flags */ +# define FD_QUIC_PKT_META_FLAGS_HS_DATA (1u<<0u) +# define FD_QUIC_PKT_META_FLAGS_STREAM (1u<<1u) +# define FD_QUIC_PKT_META_FLAGS_HS_DONE (1u<<2u) +# define FD_QUIC_PKT_META_FLAGS_MAX_DATA (1u<<3u) +# define FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR (1u<<4u) +# define FD_QUIC_PKT_META_FLAGS_CLOSE (1u<<5u) +# define FD_QUIC_PKT_META_FLAGS_PING (1u<<6u) + fd_quic_range_t range; /* CRYPTO data range; FIXME use pkt_meta var instead */ + ulong stream_id; /* if this contains stream data, + the stream id, else zero */ + + ulong tx_time; /* transmit time */ + ulong expiry; /* time pkt_meta expires... this is the time the ack is expected by */ - /* treap fields */ - ulong parent; - ulong left; - ulong right; - ulong prio; - ulong next; - ulong prev; + fd_quic_pkt_meta_var_t var[FD_QUIC_PKT_META_VAR_MAX]; + + fd_quic_pkt_meta_t * next; /* next in current list */ }; -typedef struct fd_quic_pkt_meta fd_quic_pkt_meta_t; -#define POOL_NAME fd_quic_pkt_meta_pool -#define POOL_T fd_quic_pkt_meta_t -#include "../../util/tmpl/fd_pool.c" -/* if are diff, returns sign of difference - * - * else, returns sign of difference in stream_id */ -static inline int -fd_quic_pkt_meta_cmp( const fd_quic_pkt_meta_key_t q, - const fd_quic_pkt_meta_t * e ) { - /* branchless implementation of: - diff = q.b[0] - e->key.b[0] - if( diff ) - return diff - return q.stream_id - e->key.stream_id */ - ulong q_b = q.b[0]; - ulong e_b = e->key.b[0]; - ulong q_s = q.stream_id; - ulong e_s = e->key.stream_id; - - int pkt_num_type_cmp = -2*(q_b < e_b) + ((q_b > e_b)<<1); - int stream_id_cmp = -1*(q_s < e_s) + (q_s > e_s); - return pkt_num_type_cmp + stream_id_cmp; -} - -static inline int -fd_quic_pkt_meta_lt( const fd_quic_pkt_meta_t * e1, - const fd_quic_pkt_meta_t * e2 ) { - ulong e1_b0 = e1->key.b[0]; - ulong e2_b0 = e2->key.b[0]; - return e1_b0 < e2_b0 || (e1_b0 == e2_b0 && e1->key.stream_id < e2->key.stream_id); -} - -#define TREAP_NAME fd_quic_pkt_meta_treap -#define TREAP_T fd_quic_pkt_meta_t -#define TREAP_QUERY_T fd_quic_pkt_meta_key_t -#define TREAP_CMP(q,e) fd_quic_pkt_meta_cmp( q, e ) -#define TREAP_LT(e0,e1) fd_quic_pkt_meta_lt( e0, e1 ) -#define TREAP_OPTIMIZE_ITERATION 1 -#include "../../util/tmpl/fd_treap.c" - -/* begin aliasing to abstract data structure */ -typedef fd_quic_pkt_meta_treap_t fd_quic_pkt_meta_ds_t; -typedef fd_quic_pkt_meta_treap_fwd_iter_t fd_quic_pkt_meta_ds_fwd_iter_t; - -/* fd_quic_pkt_meta_ds_fwd_iter_init is equivalent of ds.begin() - @arguments: - - ds: pointer to the ds - - pool: pointer to the backing pool - @returns: - - beginning iterator */ -static inline fd_quic_pkt_meta_ds_fwd_iter_t -fd_quic_pkt_meta_ds_fwd_iter_init( fd_quic_pkt_meta_ds_t * ds, - fd_quic_pkt_meta_t * pool ) { - return fd_quic_pkt_meta_treap_fwd_iter_init( ds, pool ); -} - -/* fd_quic_pkt_meta_ds_fwd_iter_ele returns pkt_meta* from iter - @arguments: - - iter: iterator - - pool: pointer to the backing pool - @returns: - - pointer to pkt_meta */ -static inline fd_quic_pkt_meta_t * -fd_quic_pkt_meta_ds_fwd_iter_ele( fd_quic_pkt_meta_ds_fwd_iter_t iter, - fd_quic_pkt_meta_t * pool ) { - return fd_quic_pkt_meta_treap_fwd_iter_ele( iter, pool ); -} - -/* fd_quic_pkt_meta_ds_fwd_iter_next is equivalent of iter++ - @arguments: - - iter: iterator - - pool: pointer to the backing pool - @returns: - - next iterator */ -static inline fd_quic_pkt_meta_ds_fwd_iter_t -fd_quic_pkt_meta_ds_fwd_iter_next( fd_quic_pkt_meta_ds_fwd_iter_t iter, - fd_quic_pkt_meta_t * pool ) { - return fd_quic_pkt_meta_treap_fwd_iter_next( iter, pool ); -} - -/* fd_quic_pkt_meta_ds_fwd_iter_done returns boolean - @arguments - - iter: iterator - @returns - - non-zero if iterator marks end, 0 otherwise */ -static inline int -fd_quic_pkt_meta_ds_fwd_iter_done( fd_quic_pkt_meta_ds_fwd_iter_t iter ) { - return fd_quic_pkt_meta_treap_fwd_iter_done( iter ); -} - -/* fd_quic_pkt_meta_ds_idx_ge returns iterator pointing to first pkt_meta - whose packet number is >= pkt_number - @arguments - - ds: pointer to the ds - - pkt_number: pkt_number to search for - - pool: pointer to the backing pool - @returns - - iterator to first pkt_meta with pkt number >= pkt_number */ -static inline fd_quic_pkt_meta_ds_fwd_iter_t -fd_quic_pkt_meta_ds_idx_ge( fd_quic_pkt_meta_ds_t * ds, - ulong pkt_number, - fd_quic_pkt_meta_t * pool ) { - return fd_quic_pkt_meta_treap_idx_ge( ds, - (fd_quic_pkt_meta_key_t){ - .pkt_num = pkt_number & FD_QUIC_PKT_META_PKT_NUM_MASK, - .type = 0, - .stream_id = 0}, - pool ); -} - -/* fd_quic_pkt_meta_ds_ele_cnt returns count of elements in ds */ -static inline ulong -fd_quic_pkt_meta_ds_ele_cnt( fd_quic_pkt_meta_ds_t * ds ) { - return fd_quic_pkt_meta_treap_ele_cnt( ds ); -} - -/* end aliasing to abstract data structure */ - -struct fd_quic_pkt_meta_tracker { - fd_quic_pkt_meta_ds_t sent_pkt_metas[4]; - fd_quic_pkt_meta_t * pool; +struct fd_quic_pkt_meta_list { + fd_quic_pkt_meta_t * head; + fd_quic_pkt_meta_t * tail; }; -typedef struct fd_quic_pkt_meta_tracker fd_quic_pkt_meta_tracker_t; -/* fd_quic_pkt_meta_ds_init_pool does any data structure-particular setup - on the entire pool at once. Useful for e.g. treap randomness - @arguments: - - pool: pointer pkt_meta pool - - total_meta_cnt: total pool size */ + +struct fd_quic_pkt_meta_pool { + fd_quic_pkt_meta_list_t free; /* free pkt_meta */ + + /* one of each of these for each enc_level */ + fd_quic_pkt_meta_list_t sent_pkt_meta[4]; /* sent pkt_meta */ +}; + + + +FD_PROTOTYPES_BEGIN + +/* initialize pool with existing array of pkt_meta */ void -fd_quic_pkt_meta_ds_init_pool( fd_quic_pkt_meta_t * pool, - ulong total_meta_cnt ); - -/* fd_quic_pkt_meta_tracker_init initializes the metadata tracker for each enc level - @arguments: - - tracker: pointer to the tracker - - total_meta_cnt: total number of max pkt_meta entries in this tracker - (shared across all encoding levels) - - pool: pointer to the backing pool - @returns: - - pointer to tracker if successful, NULL otherwise */ -void * -fd_quic_pkt_meta_tracker_init( fd_quic_pkt_meta_tracker_t * tracker, - ulong total_meta_cnt, - fd_quic_pkt_meta_t * pool ); - -/* fd_quic_pkt_meta_insert inserts a pkt_meta into the ds - @arguments: - - ds: pointer to the ds - - pkt_meta: pointer to the pkt_meta to insert. This pkt_meta - should have been acquired from the pool - - pool: pointer to the backing pool */ +fd_quic_pkt_meta_pool_init( fd_quic_pkt_meta_pool_t * pool, + fd_quic_pkt_meta_t * pkt_meta_array, + ulong pkt_meta_array_sz ); + +/* pop from front of list */ +fd_quic_pkt_meta_t * +fd_quic_pkt_meta_pop_front( fd_quic_pkt_meta_list_t * list ); + + +/* push onto front of list */ +void +fd_quic_pkt_meta_push_front( fd_quic_pkt_meta_list_t * list, + fd_quic_pkt_meta_t * pkt_meta ); + +/* push onto back of list */ void -fd_quic_pkt_meta_insert( fd_quic_pkt_meta_ds_t * ds, - fd_quic_pkt_meta_t * pkt_meta, - fd_quic_pkt_meta_t * pool ); - -/* - remove all pkt_meta in the range [pkt_number_lo, pkt_number_hi] - rm from treap and return to pool -*/ -/* fd_quic_pkt_meta_remove_range removes all pkt_meta in the range - [pkt_number_lo, pkt_number_hi] from the ds and returns them to the pool. - Any part of the range that's missing simply gets skipped - @arguments: - - ds: pointer to the ds - - pool: pointer to the backing pool - - pkt_number_lo: lower bound of the range - - pkt_number_hi: upper bound of the range - @returns: - - number of pkt_meta removed */ -ulong -fd_quic_pkt_meta_remove_range( fd_quic_pkt_meta_ds_t * ds, - fd_quic_pkt_meta_t * pool, - ulong pkt_number_lo, - ulong pkt_number_hi ); - -/* fd_quic_pkt_meta_min returns pointer to pkt_meta with smallest pkt_number in the ds - @arguments: - - ds: pointer to the ds - - pool: pointer to the backing pool - @returns: - - pointer to pkt_meta with smallest pkt_number in the ds */ +fd_quic_pkt_meta_push_back( fd_quic_pkt_meta_list_t * list, + fd_quic_pkt_meta_t * pkt_meta ); + +/* remove from list + requires the prior element */ +void +fd_quic_pkt_meta_remove( fd_quic_pkt_meta_list_t * list, + fd_quic_pkt_meta_t * pkt_meta_prior, + fd_quic_pkt_meta_t * pkt_meta ); + + +/* allocate a pkt_meta + obtains a free pkt_meta from the free list, and returns it + returns NULL if none is available */ fd_quic_pkt_meta_t * -fd_quic_pkt_meta_min( fd_quic_pkt_meta_ds_t * ds, - fd_quic_pkt_meta_t * pool ); +fd_quic_pkt_meta_allocate( fd_quic_pkt_meta_pool_t * pool ); + -/* fd_quic_pkt_meta_ds_clear clears all pkt_meta tracking for a given encoding level - @arguments: - - tracker: pointer to the pkt_meta tracker - - enc_level: encoding level to clear */ +/* free a pkt_meta + returns a pkt_meta to the free list, ready to be allocated again */ void -fd_quic_pkt_meta_ds_clear( fd_quic_pkt_meta_tracker_t * tracker, - uint enc_level ); +fd_quic_pkt_meta_deallocate( fd_quic_pkt_meta_pool_t * pool, + fd_quic_pkt_meta_t * pkt_meta ); FD_PROTOTYPES_END #endif // HEADER_fd_src_waltz_quic_fd_quic_pkt_meta_h + diff --git a/src/waltz/quic/fd_quic_private.h b/src/waltz/quic/fd_quic_private.h index ccbb5e5b44..9a00d6f439 100644 --- a/src/waltz/quic/fd_quic_private.h +++ b/src/waltz/quic/fd_quic_private.h @@ -67,8 +67,6 @@ struct __attribute__((aligned(16UL))) fd_quic_state_private { fd_quic_transport_params_t transport_params; - ulong max_inflight_frame_cnt_conn; /* per-conn max, computed from limits */ - /* Various internal state */ fd_quic_log_tx_t log_tx[1]; @@ -80,7 +78,6 @@ struct __attribute__((aligned(16UL))) fd_quic_state_private { fd_quic_tls_hs_cache_t hs_cache; /* dlist <> dlist_private */ fd_quic_stream_pool_t * stream_pool; /* stream pool, nullable */ - fd_quic_pkt_meta_t * pkt_meta_pool; fd_rng_t _rng[1]; /* random number generator */ /* need to be able to access connections by index */ @@ -450,21 +447,12 @@ fd_quic_calc_expiry( fd_quic_conn_t * conn, ulong now ) { } uchar * -fd_quic_gen_stream_frames( fd_quic_conn_t * conn, - uchar * payload_ptr, - uchar * payload_end, - fd_quic_pkt_meta_t * pkt_meta_tmpl, - fd_quic_pkt_meta_tracker_t * tracker ); - -void -fd_quic_process_ack_range( fd_quic_conn_t * conn, - fd_quic_frame_ctx_t * context, - uint enc_level, - ulong largest_ack, - ulong ack_range, - int is_largest, - ulong now, - ulong ack_delay ); +fd_quic_gen_stream_frames( fd_quic_conn_t * conn, + uchar * payload_ptr, + uchar * payload_end, + fd_quic_pkt_meta_t * pkt_meta, + ulong pkt_number, + ulong now ); FD_PROTOTYPES_END diff --git a/src/waltz/quic/tests/Local.mk b/src/waltz/quic/tests/Local.mk index 2856b735ae..95ccebd5b0 100644 --- a/src/waltz/quic/tests/Local.mk +++ b/src/waltz/quic/tests/Local.mk @@ -16,7 +16,6 @@ $(call make-unit-test,test_quic_conformance,test_quic_conformance,$(QUIC_TEST_LI $(call make-unit-test,test_quic_ack_tx, test_quic_ack_tx, $(QUIC_TEST_LIBS)) $(call make-unit-test,test_quic_concurrency,test_quic_concurrency,$(QUIC_TEST_LIBS)) $(call make-unit-test,test_quic_svc_q,test_quic_svc_q,$(QUIC_TEST_LIBS)) -$(call make-unit-test,test_quic_pkt_meta,test_quic_pkt_meta,$(QUIC_TEST_LIBS)) $(call run-unit-test,test_quic_proto) $(call run-unit-test,test_quic_hs) $(call run-unit-test,test_quic_streams) @@ -26,7 +25,6 @@ $(call run-unit-test,test_quic_layout) $(call run-unit-test,test_quic_ack_tx) $(call run-unit-test,test_quic_concurrency) $(call run-unit-test,test_quic_svc_q) -$(call run-unit-test,test_quic_pkt_meta) # fd_quic_tls unit tests $(call make-unit-test,test_quic_tls_hs,test_quic_tls_hs,$(QUIC_TEST_LIBS)) diff --git a/src/waltz/quic/tests/fd_quic_test_helpers.c b/src/waltz/quic/tests/fd_quic_test_helpers.c index ae2aea84a1..faa66553de 100644 --- a/src/waltz/quic/tests/fd_quic_test_helpers.c +++ b/src/waltz/quic/tests/fd_quic_test_helpers.c @@ -166,7 +166,7 @@ fd_quic_new_anonymous_small( fd_wksp_t * wksp, .conn_cnt = 1UL, .handshake_cnt = 1UL, .conn_id_cnt = 4UL, - .inflight_frame_cnt = 64UL, + .inflight_pkt_cnt = 64UL, .tx_buf_sz = 1UL<<15UL, .stream_pool_cnt = 1024 }; diff --git a/src/waltz/quic/tests/fuzz_quic.c b/src/waltz/quic/tests/fuzz_quic.c index 7a2a44b542..3c415c90bb 100644 --- a/src/waltz/quic/tests/fuzz_quic.c +++ b/src/waltz/quic/tests/fuzz_quic.c @@ -137,7 +137,7 @@ int LLVMFuzzerInitialize(int *argc, char ***argv) { .conn_id_cnt = 10, .handshake_cnt = 10, .stream_pool_cnt = 640, - .inflight_frame_cnt = 1024, + .inflight_pkt_cnt = 1024, .tx_buf_sz = 1 << 14}; ulong quic_footprint = fd_quic_footprint(&quic_limits); diff --git a/src/waltz/quic/tests/fuzz_quic_wire.c b/src/waltz/quic/tests/fuzz_quic_wire.c index fdb7cc9574..bd9cf363a3 100644 --- a/src/waltz/quic/tests/fuzz_quic_wire.c +++ b/src/waltz/quic/tests/fuzz_quic_wire.c @@ -108,12 +108,12 @@ LLVMFuzzerTestOneInput( uchar const * data, /* Create ultra low limits for QUIC instance for maximum performance */ fd_quic_limits_t const quic_limits = { - .conn_cnt = 2, - .handshake_cnt = 2, - .conn_id_cnt = 4, - .inflight_frame_cnt = 16UL, - .stream_pool_cnt = 8UL, - .tx_buf_sz = 1UL<<8UL + .conn_cnt = 2, + .handshake_cnt = 2, + .conn_id_cnt = 4, + .inflight_pkt_cnt = 8UL, + .stream_pool_cnt = 8UL, + .tx_buf_sz = 1UL<<8UL }; /* Enable features depending on the last few bits. The last bits are diff --git a/src/waltz/quic/tests/test_quic_bw.c b/src/waltz/quic/tests/test_quic_bw.c index e1c770c406..f347af302e 100644 --- a/src/waltz/quic/tests/test_quic_bw.c +++ b/src/waltz/quic/tests/test_quic_bw.c @@ -114,7 +114,7 @@ main( int argc, .conn_id_cnt = 4, .handshake_cnt = 1, .stream_pool_cnt = 1, - .inflight_frame_cnt = 128, + .inflight_pkt_cnt = 128, }; FD_LOG_NOTICE(( "Creating server QUIC (%lu bytes)", fd_quic_footprint( &server_limits ) )); fd_quic_t * server_quic = fd_quic_new_anonymous( wksp, &server_limits, FD_QUIC_ROLE_SERVER, rng ); @@ -126,7 +126,7 @@ main( int argc, .handshake_cnt = 1, .stream_id_cnt = client_burst, .stream_pool_cnt = client_burst, - .inflight_frame_cnt = client_burst+16, + .inflight_pkt_cnt = client_burst+16, .tx_buf_sz = sz }; FD_LOG_NOTICE(( "Creating client QUIC (%lu bytes)", fd_quic_footprint( &client_limits ) )); @@ -214,11 +214,7 @@ main( int argc, service_server( server_quic ); client_stream = fd_quic_conn_new_stream( client_conn ); - if( client_conn->state != FD_QUIC_CONN_STATE_ACTIVE ) { - FD_LOG_NOTICE(( "Early break due to inactive connection")); - break; - } - else if( !client_stream ) continue; + if( !client_stream ) continue; fd_quic_stream_send( client_stream, buf, sz, 1 ); long t = fd_log_wallclock(); @@ -226,12 +222,12 @@ main( int argc, FD_TEST( client_quic->metrics.conn_closed_cnt==0 ); FD_TEST( server_quic->metrics.conn_closed_cnt==0 ); - long dt = t - last_ts; + long dt = t - last_ts; float net_rx_gbps = (float)(8UL*server_quic->metrics.net_rx_byte_cnt) / (float)dt; float net_rx_gpps = (float)server_quic->metrics.net_rx_pkt_cnt / (float)dt; float net_tx_gbps = (float)(8UL*server_quic->metrics.net_tx_byte_cnt) / (float)dt; float net_tx_gpps = (float)server_quic->metrics.net_tx_pkt_cnt / (float)dt; - float data_rate = (8 * (float)rx_tot_sz) / (float)dt; + float data_rate = (8 * (float)rx_tot_sz) / (float)dt; FD_LOG_NOTICE(( "data=%6.4g Gbps net_rx=(%6.4g Gbps %6.4g Mpps) net_tx=(%6.4g Gbps %6.4g Mpps) bytes=%g", (double)data_rate, (double)net_rx_gbps, (double)net_rx_gpps * 1e3, diff --git a/src/waltz/quic/tests/test_quic_client_flood.c b/src/waltz/quic/tests/test_quic_client_flood.c index 6c5122c365..72802869c2 100644 --- a/src/waltz/quic/tests/test_quic_client_flood.c +++ b/src/waltz/quic/tests/test_quic_client_flood.c @@ -188,30 +188,16 @@ run_quic_client( } /* Reclaim packet metas */ - if ( g_unreliable ) { - /* treat all packets as ACKed (freeing handshake data, etc.) */ - static const uint enc_level = fd_quic_enc_level_appdata_id; - fd_quic_pkt_meta_tracker_t * tracker = &client_conn->pkt_meta_tracker; - fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level]; - fd_quic_pkt_meta_t * pool = fd_quic_get_state( quic )->pkt_meta_pool; - - fd_quic_pkt_meta_t* prev = NULL; - - for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool ); - !fd_quic_pkt_meta_ds_fwd_iter_done( iter ); - iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) { - if( FD_LIKELY( prev ) ) { - fd_quic_pkt_meta_pool_ele_release( pool, prev ); - } - fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool ); - fd_quic_reclaim_pkt_meta( client_conn, e, enc_level ); - prev = e; + if( g_unreliable ) { + fd_quic_pkt_meta_pool_t * pool = &client_conn->pkt_meta_pool; + fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[ fd_quic_enc_level_appdata_id ]; + for(;;) { + fd_quic_pkt_meta_t * pkt = sent->head; + if( !pkt ) break; + fd_quic_reclaim_pkt_meta( client_conn, pkt, fd_quic_enc_level_appdata_id ); + fd_quic_pkt_meta_remove( sent, NULL, pkt ); + fd_quic_pkt_meta_deallocate( pool, pkt ); } - if( FD_LIKELY( prev ) ) { - fd_quic_pkt_meta_pool_ele_release( pool, prev ); - } - - fd_quic_pkt_meta_ds_clear( tracker, enc_level ); } } diff --git a/src/waltz/quic/tests/test_quic_concurrency.c b/src/waltz/quic/tests/test_quic_concurrency.c index 71ea81c7fc..28b3a0f290 100644 --- a/src/waltz/quic/tests/test_quic_concurrency.c +++ b/src/waltz/quic/tests/test_quic_concurrency.c @@ -44,11 +44,11 @@ main( int argc, traffic instead of running real QUIC clients. */ fd_quic_limits_t quic_limits = { - .conn_cnt = conn_cnt, - .handshake_cnt = 1, - .conn_id_cnt = 4, - .inflight_frame_cnt = 4 * conn_cnt, - .tx_buf_sz = 0 + .conn_cnt = conn_cnt, + .handshake_cnt = 1, + .conn_id_cnt = 4, + .inflight_pkt_cnt = 4, + .tx_buf_sz = 0 }; quic_limits.stream_pool_cnt = quic_limits.conn_cnt; FD_LOG_INFO(( "fd_quic limits: conn_cnt=%lu conn_id_cnt=%lu stream_pool_cnt=%lu", diff --git a/src/waltz/quic/tests/test_quic_conformance.c b/src/waltz/quic/tests/test_quic_conformance.c index 6f71efa23d..24cd4e9925 100644 --- a/src/waltz/quic/tests/test_quic_conformance.c +++ b/src/waltz/quic/tests/test_quic_conformance.c @@ -462,9 +462,9 @@ test_quic_send_streams( fd_quic_sandbox_t * sandbox, FD_TEST( in_stream_list( empty_stream, conn->send_streams ) ); uchar buf[10]; - fd_quic_pkt_meta_t pkt_meta_tmpl = {.enc_level = fd_quic_enc_level_appdata_id}; + fd_quic_pkt_meta_t pkt_meta = {0}; - fd_quic_gen_stream_frames( conn, buf, buf+sizeof(buf), &pkt_meta_tmpl, &conn->pkt_meta_tracker ); + fd_quic_gen_stream_frames( conn, buf, buf+sizeof(buf), &pkt_meta, 10, 0 ); FD_TEST( !in_stream_list( empty_stream, conn->send_streams ) ); FD_TEST( in_stream_list( empty_stream, conn->used_streams ) ); @@ -484,7 +484,7 @@ test_quic_send_streams( fd_quic_sandbox_t * sandbox, uchar buf[FD_QUIC_MAX_FOOTPRINT( stream_e_frame ) + 2]; /* only 2 bytes for actual data*/ fd_quic_pkt_meta_t pkt_meta = {0}; - fd_quic_gen_stream_frames( conn, buf, buf+sizeof(buf), &pkt_meta, &conn->pkt_meta_tracker ); + fd_quic_gen_stream_frames( conn, buf, buf+sizeof(buf), &pkt_meta, 10, 0 ); FD_TEST( in_stream_list( big_stream, conn->send_streams ) ); conn->send_streams->next->sentinel = 1; /* rm this one to reset for next test */ @@ -492,69 +492,6 @@ test_quic_send_streams( fd_quic_sandbox_t * sandbox, } -static void pretend_stream( fd_quic_stream_t * stream ) { - stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; - stream->tx_buf.head = 1; - stream->tx_sent = 0; -} - -static __attribute__ ((noinline)) void -test_quic_inflight_pkt_limit( fd_quic_sandbox_t * sandbox, - fd_rng_t * rng ) { - - /* min_inflight reserved AND max enforced */ - fd_quic_t * quic = sandbox->quic; - do { - fd_quic_conn_t * conn = fd_quic_sandbox_new_conn_established( sandbox, rng ); - FD_TEST( conn ); - FD_TEST( conn->state == FD_QUIC_CONN_STATE_ACTIVE ); - FD_TEST( conn->used_pkt_meta == 0UL ); - - conn->tx_sup_stream_id = 4; /* we'll just use one stream */ - fd_quic_stream_t * empty_stream = fd_quic_conn_new_stream( conn ); - - for( int i=0; i<12; i++ ) { - empty_stream->upd_pkt_number = ~0UL; - - /* manually add to send_streams */ - FD_QUIC_STREAM_LIST_REMOVE( empty_stream ); - pretend_stream( empty_stream ); - FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, empty_stream ); - FD_TEST( in_stream_list( empty_stream, conn->send_streams ) ); - - ulong metrics_before = quic->metrics.pkt_tx_alloc_fail_cnt; - fd_quic_conn_service( quic, conn, 0 ); - ulong metrics_after = quic->metrics.pkt_tx_alloc_fail_cnt; - if( i==11 ) { - /* 12th packet should fail */ - FD_TEST( metrics_after == metrics_before + 1 ); - } else { - /* 11th packet should succeed */ - FD_TEST( metrics_after == metrics_before ); - } - } - - } while(0); - - - /* test conn_cnt*min_inflight_frame_cnt_conn <= inflight_frame_cnt */ - do { - fd_quic_limits_t quic_limits = { - .conn_cnt = 2UL, - .inflight_frame_cnt = 8UL, - .handshake_cnt = 1UL, - .conn_id_cnt = 4UL, - .stream_id_cnt = 8UL, - .tx_buf_sz = 512UL, - .stream_pool_cnt = 32UL, - }; - - quic_limits.min_inflight_frame_cnt_conn = 5UL; - FD_TEST( !fd_quic_footprint( &quic_limits ) ); - } while(0); - -} - int main( int argc, char ** argv ) { @@ -573,14 +510,13 @@ main( int argc, if( FD_UNLIKELY( !page_sz ) ) FD_LOG_ERR(( "unsupported --page-sz" )); fd_quic_limits_t quic_limits = { - .conn_cnt = 4UL, - .handshake_cnt = 1UL, - .conn_id_cnt = 4UL, - .stream_id_cnt = 8UL, - .inflight_frame_cnt = 8UL * 4, - .min_inflight_frame_cnt_conn = 7UL, /* for test_quic_inflight_pkt_limit */ - .tx_buf_sz = 512UL, - .stream_pool_cnt = 32UL, + .conn_cnt = 4UL, + .handshake_cnt = 1UL, + .conn_id_cnt = 4UL, + .stream_id_cnt = 8UL, + .inflight_pkt_cnt = 8UL, + .tx_buf_sz = 512UL, + .stream_pool_cnt = 32UL, }; ulong const pkt_cnt = 128UL; @@ -613,7 +549,6 @@ main( int argc, test_quic_rx_max_streams_frame ( sandbox, rng ); test_quic_small_pkt_ping ( sandbox, rng ); test_quic_send_streams ( sandbox, rng ); - test_quic_inflight_pkt_limit ( sandbox, rng ); test_quic_parse_path_challenge(); /* Wind down */ diff --git a/src/waltz/quic/tests/test_quic_conn.c b/src/waltz/quic/tests/test_quic_conn.c index 6f7b806be2..75c5a98426 100644 --- a/src/waltz/quic/tests/test_quic_conn.c +++ b/src/waltz/quic/tests/test_quic_conn.c @@ -132,7 +132,7 @@ main( int argc, char ** argv ) { .handshake_cnt = 10, .stream_id_cnt = 10, .stream_pool_cnt = 400, - .inflight_frame_cnt = 1024 * 10, + .inflight_pkt_cnt = 1024, .tx_buf_sz = 1<<14 }; diff --git a/src/waltz/quic/tests/test_quic_drops.c b/src/waltz/quic/tests/test_quic_drops.c index cc15a1bf7e..b729a6381a 100644 --- a/src/waltz/quic/tests/test_quic_drops.c +++ b/src/waltz/quic/tests/test_quic_drops.c @@ -340,7 +340,7 @@ main( int argc, char ** argv ) { .handshake_cnt = 10, .stream_id_cnt = 10, .stream_pool_cnt = 512, - .inflight_frame_cnt = 1024 * 10, + .inflight_pkt_cnt = 1024, .tx_buf_sz = 1<<14 }; diff --git a/src/waltz/quic/tests/test_quic_hs.c b/src/waltz/quic/tests/test_quic_hs.c index 4ee5d33afa..d6fd6748b8 100644 --- a/src/waltz/quic/tests/test_quic_hs.c +++ b/src/waltz/quic/tests/test_quic_hs.c @@ -109,7 +109,7 @@ main( int argc, char ** argv ) { .handshake_cnt = 10, .stream_id_cnt = 10, .stream_pool_cnt = 400, - .inflight_frame_cnt = 1024 * 10, + .inflight_pkt_cnt = 1024, .tx_buf_sz = 1<<14 }; diff --git a/src/waltz/quic/tests/test_quic_idle_conns.c b/src/waltz/quic/tests/test_quic_idle_conns.c index 3398ac0a52..108602e204 100644 --- a/src/waltz/quic/tests/test_quic_idle_conns.c +++ b/src/waltz/quic/tests/test_quic_idle_conns.c @@ -180,7 +180,7 @@ main( int argc, .handshake_cnt = num_conns, .conn_id_cnt = 16UL, .stream_pool_cnt = num_conns * 2, - .inflight_frame_cnt = num_conns * 64UL, + .inflight_pkt_cnt = 64UL, .tx_buf_sz = 0 }; ulong quic_footprint = fd_quic_footprint( &quic_limits ); diff --git a/src/waltz/quic/tests/test_quic_key_phase.c b/src/waltz/quic/tests/test_quic_key_phase.c index 698ae4241a..a531eedd6c 100644 --- a/src/waltz/quic/tests/test_quic_key_phase.c +++ b/src/waltz/quic/tests/test_quic_key_phase.c @@ -294,20 +294,20 @@ main( int argc, char ** argv ) { .conn_cnt = 2, .conn_id_cnt = 4, .handshake_cnt = 2, - .inflight_frame_cnt = 16 * 2 + .inflight_pkt_cnt = 16 }; fd_quic_t * server_quic = fd_quic_new_anonymous( wksp, &server_limits, FD_QUIC_ROLE_SERVER, rng ); FD_TEST( server_quic ); FD_LOG_INFO(( "Creating client QUIC" )); fd_quic_limits_t const client_limits = { - .conn_cnt = 2, - .conn_id_cnt = 4, - .handshake_cnt = 2, - .inflight_frame_cnt = 530 * 2, - .stream_id_cnt = 512, - .stream_pool_cnt = 512, - .tx_buf_sz = 32, + .conn_cnt = 2, + .conn_id_cnt = 4, + .handshake_cnt = 2, + .inflight_pkt_cnt = 530, + .stream_id_cnt = 512, + .stream_pool_cnt = 512, + .tx_buf_sz = 32, }; fd_quic_t * client_quic = fd_quic_new_anonymous( wksp, &client_limits, FD_QUIC_ROLE_CLIENT, rng ); FD_TEST( client_quic ); diff --git a/src/waltz/quic/tests/test_quic_pkt_meta.c b/src/waltz/quic/tests/test_quic_pkt_meta.c deleted file mode 100644 index f8d0005341..0000000000 --- a/src/waltz/quic/tests/test_quic_pkt_meta.c +++ /dev/null @@ -1,249 +0,0 @@ -#include "../fd_quic.h" -#include "../fd_quic_private.h" -#include "fd_quic_test_helpers.h" - - -static fd_quic_conn_t conn; -static fd_quic_pkt_meta_t * pkt_meta_mem; -static fd_quic_t * quic; - -static void -init_tracker( ulong max_inflight ) { - - fd_quic_pkt_meta_t * pool = fd_quic_get_state( quic )->pkt_meta_pool; - fd_quic_pkt_meta_ds_init_pool( pool, max_inflight ); - - if( !fd_quic_pkt_meta_tracker_init( &conn.pkt_meta_tracker, max_inflight, pool ) ) { - FD_LOG_ERR(( "Failed to initialize tracker" )); - return; - } - - fd_quic_pkt_meta_t * pkt_meta = NULL; - for( ulong i = 0; i < max_inflight; i++ ) { - pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( pool ); - memset( pkt_meta, 0, sizeof(fd_quic_pkt_meta_t) ); - FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta, i ); - fd_quic_pkt_meta_insert( &conn.pkt_meta_tracker.sent_pkt_metas[fd_quic_enc_level_appdata_id], - pkt_meta, pool ); - } - FD_TEST( fd_quic_pkt_meta_ds_ele_cnt( &conn.pkt_meta_tracker.sent_pkt_metas[fd_quic_enc_level_appdata_id] ) == max_inflight ); -} - - -/* Compile-time test of fd_quic_pkt_meta_cmp */ -static void -fd_quic_pkt_meta_cmp_test(void) { - FD_LOG_INFO(("testing pkt_meta_cmp")); - fd_quic_pkt_meta_key_t pkt_1_big_type = {.type = 3, .pkt_num = 1, .stream_id = 1<<30UL}; - fd_quic_pkt_meta_key_t pkt_2_small_type = {.type = 1, .pkt_num = 2, .stream_id = 2}; - - fd_quic_pkt_meta_t pkt_1_big_type_e = { .key = pkt_1_big_type }; - - /* Equal keys should return 0 */ - FD_TEST( fd_quic_pkt_meta_cmp(pkt_1_big_type, &pkt_1_big_type_e) == 0 ); - - /* pkt_num takes priority over type and stream id */ - FD_TEST( fd_quic_pkt_meta_cmp( pkt_2_small_type, &pkt_1_big_type_e ) > 0 ); - - /* same pkt_num, same type, stream_id differentiates */ - fd_quic_pkt_meta_key_t pkt_1_big_type_small_stream_id = pkt_1_big_type; - pkt_1_big_type_small_stream_id.stream_id = 2; - FD_TEST( fd_quic_pkt_meta_cmp( pkt_1_big_type_small_stream_id, &pkt_1_big_type_e ) < 0 ); -} - -static void -test_adversarial_ack( ulong max_inflight, - ulong range_sz ) { - fd_quic_frame_ctx_t context; - fd_quic_pkt_t pkt; - context.pkt = &pkt; - - /* Very adversarial */ - do { - init_tracker( max_inflight ); - fd_quic_pkt_meta_ds_t * sent_pkt_metas = &conn.pkt_meta_tracker.sent_pkt_metas[fd_quic_enc_level_appdata_id]; - - ulong highest_known = max_inflight - 1; - long start = fd_tickcount(); - ulong cnt; - while( ( cnt = fd_quic_pkt_meta_ds_ele_cnt( sent_pkt_metas ) ) > 0) { - /* let's send the largest range_sz values */ - fd_quic_process_ack_range( - &conn, - &context, - fd_quic_enc_level_appdata_id, - highest_known, - range_sz-1, - 1, - 0, - 0 - ); - highest_known -= fd_ulong_min( range_sz, highest_known ); - - /* then middle-ish range_sz values (reduce locality) */ - fd_quic_process_ack_range( - &conn, - &context, - fd_quic_enc_level_appdata_id, - highest_known>>1, - range_sz-1, - 1, - 0, - 0 - ); - - /* then range_sz higher than max */ - fd_quic_process_ack_range( - &conn, - &context, - fd_quic_enc_level_appdata_id, - highest_known + range_sz - 1, - range_sz-1, - 1, - 0, - 0 - ); - } - long end = fd_tickcount(); - FD_LOG_NOTICE(( "Very adversarial: Time taken: %ld us", (end - start) / 1000 )); - } while(0); - - /* 'Reasonable reordering', alternating between second range and first range */ - do { - init_tracker( max_inflight ); - fd_quic_pkt_meta_ds_t * sent_pkt_metas = &conn.pkt_meta_tracker.sent_pkt_metas[fd_quic_enc_level_appdata_id]; - fd_quic_pkt_meta_t * pool = fd_quic_get_state( quic )->pkt_meta_pool; - - long start = fd_tickcount(); - ulong cnt; - while( ( cnt = fd_quic_pkt_meta_ds_ele_cnt( sent_pkt_metas ) ) > 0 ) { - fd_quic_pkt_meta_ds_fwd_iter_t start = fd_quic_pkt_meta_ds_fwd_iter_init( sent_pkt_metas, pool ); - fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( start, pool ); - ulong min_pkt_number = e->key.pkt_num; - - /* send second range first */ - fd_quic_process_ack_range( - &conn, - &context, - fd_quic_enc_level_appdata_id, - min_pkt_number + 2*range_sz - 1, - range_sz-1, - 1, - 0, - 0 - ); - - /* then first range */ - fd_quic_process_ack_range( - &conn, - &context, - fd_quic_enc_level_appdata_id, - min_pkt_number+range_sz - 1, - range_sz-1, - 1, - 0, - 0 - ); - - } - long end = fd_tickcount(); - FD_LOG_NOTICE(( "Reasonable reordering: Time taken: %ld us", (end - start) / 1000 )); - } while(0); - - - /* 'Happy case', no reordering */ - do { - init_tracker( max_inflight ); - fd_quic_pkt_meta_ds_t * sent_pkt_metas = &conn.pkt_meta_tracker.sent_pkt_metas[fd_quic_enc_level_appdata_id]; - fd_quic_pkt_meta_t * pool = fd_quic_get_state( conn.quic )->pkt_meta_pool; - - long start = fd_tickcount(); - ulong cnt; - while( ( cnt = fd_quic_pkt_meta_ds_ele_cnt( sent_pkt_metas ) ) > 0 ) { - fd_quic_pkt_meta_ds_fwd_iter_t start = fd_quic_pkt_meta_ds_fwd_iter_init( sent_pkt_metas, pool ); - fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( start, pool ); - ulong min_pkt_number = e->key.pkt_num; - - /* send first range */ - fd_quic_process_ack_range( - &conn, - &context, - fd_quic_enc_level_appdata_id, - min_pkt_number+range_sz - 1, - range_sz-1, - 1, - 0, - 0 - ); - - } - long end = fd_tickcount(); - FD_LOG_NOTICE(( "Happy case: Time taken: %ld us", (end - start) / 1000 )); - } while(0); -} - -int -main( int argc, char ** argv ) { - fd_boot ( &argc, &argv ); - fd_quic_test_boot( &argc, &argv ); - - ulong max_inflight = fd_env_strip_cmdline_ulong ( &argc, &argv, "--max-inflight", NULL, 100UL ); - ulong range_sz = fd_env_strip_cmdline_ulong ( &argc, &argv, "--range-sz", NULL, 10UL ); - - FD_LOG_INFO(("booted")); - - ulong cpu_idx = fd_tile_cpu_id( fd_tile_idx() ); - if( cpu_idx>fd_shmem_cpu_cnt() ) cpu_idx = 0UL; - - fd_quic_pkt_meta_cmp_test(); - - fd_quic_limits_t limits = { - .inflight_frame_cnt = max_inflight, - .conn_id_cnt = 4, - .conn_cnt = 1, - .handshake_cnt = 1, - .log_depth = 256, - .stream_pool_cnt = 100, - .stream_id_cnt = 10 - }; - - fd_wksp_t * wksp = fd_wksp_join( - fd_wksp_new_anonymous( - fd_cstr_to_shmem_page_sz( "gigantic" ), 1, fd_shmem_cpu_idx( 0 ), "wksp", 0UL - ) - ); - FD_TEST( wksp ); - - uchar * laddr = (uchar*)wksp; - ulong footprint = fd_quic_footprint( &limits ); - FD_TEST( footprint ); - quic = (fd_quic_t*)laddr; - laddr += footprint; - FD_TEST( quic ); - fd_quic_state_t * state = fd_quic_get_state( quic ); - - /* Allocate pkt_meta space */ - /* pool alloc is max(128, alignof(pkt_meta_t)) so we may need extra space */ - uchar * pkt_meta_alloc = laddr; - ulong extra = fd_quic_pkt_meta_pool_align() / alignof(fd_quic_pkt_meta_t) + 1; - ulong pkt_meta_footprint = sizeof(fd_quic_pkt_meta_t) * (max_inflight + extra); - laddr += pkt_meta_footprint; - - pkt_meta_mem = (fd_quic_pkt_meta_t*)fd_ulong_align_up( (ulong)pkt_meta_alloc, fd_quic_pkt_meta_pool_align() ); - FD_TEST( pkt_meta_mem ); - - FD_LOG_INFO(("allocated space")); - - fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_mem, max_inflight ); - state->pkt_meta_pool = fd_quic_pkt_meta_pool_join( pkt_meta_pool ); - FD_TEST( state->pkt_meta_pool ); - - FD_LOG_INFO(("joined pool")); - - conn.quic = quic; - - test_adversarial_ack( max_inflight, range_sz ); - - FD_LOG_NOTICE(( "pass" )); - return 0; -} diff --git a/src/waltz/quic/tests/test_quic_retry_integration.c b/src/waltz/quic/tests/test_quic_retry_integration.c index 642833c236..5b7ac56fbe 100644 --- a/src/waltz/quic/tests/test_quic_retry_integration.c +++ b/src/waltz/quic/tests/test_quic_retry_integration.c @@ -88,7 +88,7 @@ main( int argc, char ** argv ) { .handshake_cnt = 10, .stream_id_cnt = 10, .stream_pool_cnt = 400, - .inflight_frame_cnt = 1024 * 10, + .inflight_pkt_cnt = 1024, .tx_buf_sz = 1<<14 }; diff --git a/src/waltz/quic/tests/test_quic_streams.c b/src/waltz/quic/tests/test_quic_streams.c index 2352992a42..b8119f0ff5 100644 --- a/src/waltz/quic/tests/test_quic_streams.c +++ b/src/waltz/quic/tests/test_quic_streams.c @@ -106,7 +106,7 @@ main( int argc, .conn_cnt = 2, .conn_id_cnt = 4, .handshake_cnt = 10, - .inflight_frame_cnt = 100 * 2, + .inflight_pkt_cnt = 100, .tx_buf_sz = 1<<15, .stream_pool_cnt = 512 }; @@ -120,7 +120,7 @@ main( int argc, .conn_id_cnt = 4, .handshake_cnt = 10, .stream_id_cnt = 20, - .inflight_frame_cnt = 100 * 2, + .inflight_pkt_cnt = 100, .tx_buf_sz = 1<<15, .stream_pool_cnt = 512 }; diff --git a/src/waltz/quic/tests/test_quic_svc_q.c b/src/waltz/quic/tests/test_quic_svc_q.c index f810e5f483..438e7ed7be 100644 --- a/src/waltz/quic/tests/test_quic_svc_q.c +++ b/src/waltz/quic/tests/test_quic_svc_q.c @@ -175,14 +175,14 @@ main( int argc, char ** argv ) { /* Allocate a large buffer upfront */ fd_quic_limits_t limits = { - .inflight_frame_cnt = 10*max_conn, - .conn_cnt = max_conn, - .conn_id_cnt = max_conn, - .handshake_cnt = 10, - .log_depth = 1, - .tx_buf_sz = 256, - .stream_pool_cnt = 10, - .stream_id_cnt = 10 + .inflight_pkt_cnt = 10*max_conn, + .conn_cnt = max_conn, + .conn_id_cnt = max_conn, + .handshake_cnt = 10, + .log_depth = 1, + .tx_buf_sz = 256, + .stream_pool_cnt = 10, + .stream_id_cnt = 10 }; FD_LOG_NOTICE(( "Starting fd_quic_svc_q tests" )); diff --git a/src/waltz/quic/tests/test_quic_txns.c b/src/waltz/quic/tests/test_quic_txns.c index c3585ccf4b..a29a310405 100644 --- a/src/waltz/quic/tests/test_quic_txns.c +++ b/src/waltz/quic/tests/test_quic_txns.c @@ -219,7 +219,7 @@ main( int argc, .handshake_cnt = 256UL, .conn_id_cnt = 16UL, .stream_pool_cnt = 2048UL, - .inflight_frame_cnt = 64UL * 1024, + .inflight_pkt_cnt = 64UL, .tx_buf_sz = 1UL<<15UL }; ulong quic_footprint = fd_quic_footprint( &quic_limits );