diff --git a/src/include/zf_internal/private/zf_stack_def.h b/src/include/zf_internal/private/zf_stack_def.h index cc40a64..5e19b18 100644 --- a/src/include/zf_internal/private/zf_stack_def.h +++ b/src/include/zf_internal/private/zf_stack_def.h @@ -181,6 +181,9 @@ struct zf_stack { #define ZF_RES_NIC_FLAG_VLAN_FILTERS 0x1 #define ZF_RES_NIC_FLAG_RX_LL 0x2 #define ZF_RES_NIC_FLAG_TX_LL 0x4 +#define ZF_RES_NIC_FLAG_RX_REF 0x8 +#define ZF_RES_NIC_FLAG_CTPIO_ONLY 0x10 +#define ZF_RES_NIC_FLAG_PIO 0x20 #include #define ZF_VERSION_LENGTH_MAX OO_VER_STR_LEN @@ -341,4 +344,9 @@ static inline const ef_vi* zf_stack_nic_tx_vi(const zf_stack* st, int nicno) { static inline bool zf_stack_nic_has_tx_vi(const zf_stack* st, int nicno) { return st->nic[nicno].tx_vi.inited; } +static inline unsigned* zf_stack_res_nic_flags(zf_stack* st, int nicno) { + struct zf_stack_impl* sti = ZF_CONTAINER(struct zf_stack_impl, st, st); + return &sti->nic[nicno].flags; +} + #endif /* __ZF_INTERNAL_STACK_DEF_H__ */ diff --git a/src/include/zf_internal/tx_send.h b/src/include/zf_internal/tx_send.h index d8d9049..8314dc0 100644 --- a/src/include/zf_internal/tx_send.h +++ b/src/include/zf_internal/tx_send.h @@ -164,7 +164,7 @@ zf_send(struct zf_tx* restrict tx, return -EAGAIN; #endif - if( vi->nic_type.arch == EF_VI_ARCH_EFCT ) { + if( *zf_stack_res_nic_flags(st, nicno) & ZF_RES_NIC_FLAG_CTPIO_ONLY ) { /* FIXME get X3 overhead sorted properly */ if(ZF_UNLIKELY( ef_vi_transmit_space_bytes(vi) < (int)tot_len + 128 )) { if( st->flags & ZF_STACK_FLAG_TRANSMIT_WARM_ENABLED ) @@ -207,7 +207,7 @@ zf_send(struct zf_tx* restrict tx, !ctpio_is_allowed(st_nic, tot_len) && pio_is_available(st_nic) && tot_len <= max_pio_len(st_nic) && iov_cnt <= 2 ) { - zf_assert_nequal(vi->nic_type.arch, EF_VI_ARCH_EFCT); + zf_assert(*zf_stack_res_nic_flags(st, nicno) & ZF_RES_NIC_FLAG_PIO); int pio_buf_no = st_nic->pio.busy & 1; int orig_pio_ofs = pio_buf_no ? max_pio_len(st_nic) : 0; rc = ef10_ef_vi_transmitv_copy_pio(vi, orig_pio_ofs, iov, iov_cnt, 1); @@ -288,6 +288,7 @@ zf_send(struct zf_tx* restrict tx, } } zf_assert_nequal(vi->nic_type.arch, EF_VI_ARCH_EFCT); + zf_assert_nequal(vi->nic_type.arch, EF_VI_ARCH_EF10CT); /* Strictly speaking, we should call ef_vi_transmit_ctpio_fallback() * to post a CTPIO fallback descriptor. However, that entry point is * actually identical to this one, and this code path is also used diff --git a/src/include/zf_internal/zf_stack.h b/src/include/zf_internal/zf_stack.h index 5f9d611..22062ec 100644 --- a/src/include/zf_internal/zf_stack.h +++ b/src/include/zf_internal/zf_stack.h @@ -71,7 +71,7 @@ ZF_HOT static inline void zf_stack_refill_rx_ring(struct zf_stack* st, int nic, unsigned max_count) { ef_vi* vi = &st->nic[nic].vi; - if( vi->nic_type.arch == EF_VI_ARCH_EFCT ) + if( *zf_stack_res_nic_flags(st, nic) & ZF_RES_NIC_FLAG_RX_REF ) return; unsigned space = ef_vi_receive_space(vi); if( space < st->nic[nic].rx_ring_refill_batch_size ) diff --git a/src/lib/zf/private/reactor.c b/src/lib/zf/private/reactor.c index bcb8de2..afa3d27 100644 --- a/src/lib/zf/private/reactor.c +++ b/src/lib/zf/private/reactor.c @@ -508,10 +508,11 @@ __zf_reactor_perform(struct zf_stack* st, unsigned spin_cnt) * from the future in the next RX buffer. */ pkt_id next_packet_id = 0; char* next_packet; - bool is_efct = vi->nic_type.arch == EF_VI_ARCH_EFCT; + bool is_rx_ref = *zf_stack_res_nic_flags(st, nic) & + ZF_RES_NIC_FLAG_RX_REF; bool packet_present; - if( is_efct ) { + if( is_rx_ref ) { char* vpkt = (char*)efct_vi_rx_future_peek(vi); packet_present = vpkt != NULL; if( packet_present ) { @@ -643,7 +644,7 @@ zf_reactor_purge_event(struct zf_stack* st, int nic, ef_vi* vi, ef_event* ev) case EF_EVENT_TYPE_RX: zf_assert_nequal(EF_EVENT_RX_SOP(*ev), 0); zf_assert_equal(EF_EVENT_RX_CONT(*ev), 0); - zf_assert_nequal(vi->nic_type.arch, EF_VI_ARCH_EFCT); + zf_assert(!(*zf_stack_res_nic_flags(st, nic) & ZF_RES_NIC_FLAG_RX_REF)); zf_pool_free_pkt(&st->pool, EF_EVENT_RX_RQ_ID(*ev)); zf_log_event_trace(st, "%s: purged event %x\n", __FUNCTION__, EF_EVENT_RX_RQ_ID(*ev)); @@ -652,7 +653,7 @@ zf_reactor_purge_event(struct zf_stack* st, int nic, ef_vi* vi, ef_event* ev) zf_reactor_handle_tx_event(st, nic, vi, ev); return ZF_REACTOR_PURGE_STATUS_TX; case EF_EVENT_TYPE_RX_REF: - zf_assert_equal(vi->nic_type.arch, EF_VI_ARCH_EFCT); + zf_assert(*zf_stack_res_nic_flags(st, nic) & ZF_RES_NIC_FLAG_RX_REF); efct_vi_rxpkt_release(vi, ev->rx_ref.pkt_id); zf_log_event_trace(st, "%s: purged event %x\n", __FUNCTION__, ev->rx_ref.pkt_id); return ZF_REACTOR_PURGE_STATUS_RX; diff --git a/src/lib/zf/private/stack_alloc.c b/src/lib/zf/private/stack_alloc.c index 21ea952..a96e363 100644 --- a/src/lib/zf/private/stack_alloc.c +++ b/src/lib/zf/private/stack_alloc.c @@ -243,7 +243,7 @@ static int zf_stack_init_pio(struct zf_stack_impl* sti, struct zf_attr* attr, struct zf_stack_res_nic* sti_nic = &sti->nic[nicno]; ef_vi* vi = zf_stack_nic_tx_vi(st_nic); - if( st_nic->vi.nic_type.arch == EF_VI_ARCH_EFCT && + if( !(*zf_stack_res_nic_flags(st, nicno) & ZF_RES_NIC_FLAG_PIO) && attr->pio >= PIO_MUST_USE ) { zf_log_stack_warn(st, "PIO not supported by efct interface but pio=%d. " @@ -491,13 +491,16 @@ static void zf_stack_init_state(struct zf_stack_impl* sti, static int zf_stack_init_nic_capabilities(struct zf_stack* st, int nicno) { struct zf_stack_impl* sti = ZF_CONTAINER(struct zf_stack_impl, st, st); + unsigned* res_nic_flags = zf_stack_res_nic_flags(st, nicno); unsigned long vlan_filters; unsigned long variant; + unsigned long is_rx_ref; + unsigned long is_ctpio_only; ef_driver_handle dh = sti->nic[nicno].dh; ef_pd* pd = &sti->nic[nicno].pd; int rc; - sti->nic[nicno].flags = 0; + *res_nic_flags = 0; rc = ef_pd_capabilities_get(dh, pd, dh, EF_VI_CAP_RX_FW_VARIANT, &variant); if( rc != 0 ) { @@ -512,7 +515,7 @@ static int zf_stack_init_nic_capabilities(struct zf_stack* st, int nicno) return -EOPNOTSUPP; } if( variant == MC_CMD_GET_CAPABILITIES_OUT_RXDP_LOW_LATENCY ) - sti->nic[nicno].flags |= ZF_RES_NIC_FLAG_RX_LL; + *res_nic_flags |= ZF_RES_NIC_FLAG_RX_LL; rc = ef_pd_capabilities_get(dh, pd, dh, EF_VI_CAP_TX_FW_VARIANT, &variant); if( rc != 0 ) { @@ -527,12 +530,27 @@ static int zf_stack_init_nic_capabilities(struct zf_stack* st, int nicno) return -EOPNOTSUPP; } if( variant == MC_CMD_GET_CAPABILITIES_OUT_TXDP_LOW_LATENCY ) - sti->nic[nicno].flags |= ZF_RES_NIC_FLAG_TX_LL; + *res_nic_flags |= ZF_RES_NIC_FLAG_TX_LL; rc = ef_pd_capabilities_get(dh, pd, dh, EF_VI_CAP_RX_FILTER_TYPE_IP_VLAN, &vlan_filters); if( rc == 0 && vlan_filters != 0 ) - sti->nic[nicno].flags |= ZF_RES_NIC_FLAG_VLAN_FILTERS; + *res_nic_flags |= ZF_RES_NIC_FLAG_VLAN_FILTERS; + + rc = ef_pd_capabilities_get(dh, pd, dh, EF_VI_CAP_RX_REF, + &is_rx_ref); + if( rc == 0 && is_rx_ref != 0 ) + *res_nic_flags |= ZF_RES_NIC_FLAG_RX_REF; + + rc = ef_pd_capabilities_get(dh, pd, dh, EF_VI_CAP_CTPIO_ONLY, + &is_ctpio_only); + if( rc == 0 && is_ctpio_only != 0 ) + *res_nic_flags |= ZF_RES_NIC_FLAG_CTPIO_ONLY; + + rc = ef_pd_capabilities_get(dh, pd, dh, EF_VI_CAP_PIO, + &is_ctpio_only); + if( rc == 0 && is_ctpio_only != 0 ) + *res_nic_flags |= ZF_RES_NIC_FLAG_PIO; return 0; } @@ -546,6 +564,7 @@ int zf_stack_init_nic_resources(struct zf_stack_impl* sti, zf_stack* st = &sti->st; struct zf_stack_nic* st_nic = &st->nic[nicno]; struct zf_stack_res_nic* sti_nic = &sti->nic[nicno]; + unsigned* res_nic_flags = zf_stack_res_nic_flags(st, nicno); int rc; /* Open driver. */ @@ -574,8 +593,8 @@ int zf_stack_init_nic_resources(struct zf_stack_impl* sti, if( rc < 0 ) goto fail1; - if( !(sti->nic[nicno].flags & ZF_RES_NIC_FLAG_RX_LL) || - !(sti->nic[nicno].flags & ZF_RES_NIC_FLAG_TX_LL) ) { + if( !(*res_nic_flags & ZF_RES_NIC_FLAG_RX_LL) || + !(*res_nic_flags & ZF_RES_NIC_FLAG_TX_LL) ) { zf_log_stack_warn(st, "Interface %s is not in low latency mode.\n", sti->nic[nicno].if_name); zf_log_stack_warn(st, "Low latency mode is recommended for best " @@ -636,9 +655,9 @@ int zf_stack_init_nic_resources(struct zf_stack_impl* sti, ); if ( attr->rx_ring_max != 0 ) { - /* For EFCT, we store the timestamp in a fake prefix when copying from - * the shared rx buffer into our own packet buffer. */ - if( st_nic->vi.nic_type.arch == EF_VI_ARCH_EFCT ) + /* For RX_REF nics, we store the timestamp in a fake prefix when copying + * from the shared rx buffer into our own packet buffer. */ + if( *res_nic_flags & ZF_RES_NIC_FLAG_RX_REF ) st_nic->rx_prefix_len = ES_DZ_RX_PREFIX_SIZE; else st_nic->rx_prefix_len = ef_vi_receive_prefix_len(&st_nic->vi); diff --git a/src/lib/zf/private/stack_fast.c b/src/lib/zf/private/stack_fast.c index 2e9b885..b5b85f1 100644 --- a/src/lib/zf/private/stack_fast.c +++ b/src/lib/zf/private/stack_fast.c @@ -90,7 +90,7 @@ zf_stack_handle_rx(struct zf_stack* st, int nic, const char* iov_base, if( rx_prefix_len ){ ef_vi* vi = &(st->nic[nic].vi); - if( vi->nic_type.arch != EF_VI_ARCH_EFCT ) { + if( *zf_stack_res_nic_flags(st, nic) & ZF_RES_NIC_FLAG_RX_REF ) { ef_eventq_state* evqs = &(vi->ep_state->evq); /* To overcome the restrictions that come with using ef_vi's timestamping diff --git a/src/lib/zf/rx.c b/src/lib/zf/rx.c index f011684..569a854 100644 --- a/src/lib/zf/rx.c +++ b/src/lib/zf/rx.c @@ -269,7 +269,7 @@ zfrr_hw_filter_init_vlan(struct zf_stack* st, int nicno, int proto, struct zf_stack_impl* sti = ZF_CONTAINER(struct zf_stack_impl, st, st); if( proto == IPPROTO_UDP && is_multicast(laddr) && - sti->nic[nicno].flags & ZF_RES_NIC_FLAG_VLAN_FILTERS && + *zf_stack_res_nic_flags(st, nicno) & ZF_RES_NIC_FLAG_VLAN_FILTERS && sti->sti_vlan_id != ZF_NO_VLAN ) { ef_filter_spec_set_vlan(spec, sti->sti_vlan_id); } diff --git a/src/lib/zf/stack.c b/src/lib/zf/stack.c index 7592f3a..0318085 100644 --- a/src/lib/zf/stack.c +++ b/src/lib/zf/stack.c @@ -242,12 +242,13 @@ int zf_stack_free(struct zf_stack* stack) unsigned tx_packets = ef_vi_transmit_fill_level(zf_stack_nic_tx_vi(stack, nicno)) - pkts_in_pio; unsigned rx_packets = ef_vi_receive_fill_level(&stack->nic[nicno].vi); - /* For X3 ef_vi_transmit_fill_level and PIO states are bogus */ - if( zf_stack_nic_tx_vi(stack, nicno)->nic_type.arch != EF_VI_ARCH_EFCT ) + /* For CTPIO only boards, ef_vi_transmit_fill_level and PIO states are + * bogus */ + if( !(*zf_stack_res_nic_flags(stack, nicno) & + ZF_RES_NIC_FLAG_CTPIO_ONLY) ) { pkts_accounted += tx_packets; - - if( stack->nic[nicno].vi.nic_type.arch != EF_VI_ARCH_EFCT ) pkts_accounted += rx_packets; + } zf_log_stack_trace(stack, "%s: VI %d: busy pkts=%d vs %d: rx_hw=%d tx_hw=%d " diff --git a/src/lib/zf/tx_warm.c b/src/lib/zf/tx_warm.c index cfa6874..7cf549d 100644 --- a/src/lib/zf/tx_warm.c +++ b/src/lib/zf/tx_warm.c @@ -16,7 +16,8 @@ int enable_tx_warm(struct zf_tx* tx, zf_tx_warm_state* state) zf_log_stack_trace(st, "%s: TX warm enabled\n", __func__); char* ctpio_warm_buf = NULL; state->ctpio_warm_buf_id = PKT_INVALID; - if( vi->vi_flags & EF_VI_TX_CTPIO && vi->nic_type.arch != EF_VI_ARCH_EFCT ) { + if( vi->vi_flags & EF_VI_TX_CTPIO && vi->nic_type.arch != EF_VI_ARCH_EFCT + && vi->nic_type.arch != EF_VI_ARCH_EF10CT ) { int rc = zft_alloc_pkt(&st->pool, &state->ctpio_warm_buf_id); if( rc < 0 ) return rc; diff --git a/src/lib/zf/udp_tx.c b/src/lib/zf/udp_tx.c index 9562be3..5e0074f 100644 --- a/src/lib/zf/udp_tx.c +++ b/src/lib/zf/udp_tx.c @@ -194,7 +194,7 @@ try_clean_tx_completions(struct zf_udp_tx* udp_tx) int iterations = 0; #endif - if( vi->nic_type.arch != EF_VI_ARCH_EFCT ) + if( *zf_stack_res_nic_flags(st, nicno) & ZF_RES_NIC_FLAG_CTPIO_ONLY ) return false; for( ; ; ) { ef_event ev; diff --git a/src/lib/zf/zf_stackdump.c b/src/lib/zf/zf_stackdump.c index 433d921..5c7bc12 100644 --- a/src/lib/zf/zf_stackdump.c +++ b/src/lib/zf/zf_stackdump.c @@ -96,7 +96,8 @@ void dump_nic(SkewPointer stimpl, int index) return; zf_dump("nic%d: vi=%d vi_flags=%x nic_flags=%x intf=%s index=%d hw=%u%c%u\n", - index, ef_vi_instance(vi), ef_vi_flags(vi), stimpl->nic[index].flags, + index, ef_vi_instance(vi), ef_vi_flags(vi), + *zf_stack_res_nic_flags(&stimpl->st, index), stimpl->nic[index].if_name, stimpl->nic[index].ifindex, vi->nic_type.arch, vi->nic_type.variant, vi->nic_type.revision); #if 0