Skip to content

Commit ae4eb4b

Browse files
ztefuyanztefuyan
committed
libzrdma: Add support for query interface and some other optimization
Thia patch add the following verbs: query_device_ex query_port attach_mcast detach_mcast This patch also add the following function: add support for new vendor id and device id add support for private verbs interface use ZXDH_DEBUG_MASK to control debug level log use ZXDH_WRITE_IMM_SPILIT_RNABLE to fix write with immediate bug on-behalf-of: @ZTE [email protected] Signed-off-by: zte_lifuyan <[email protected]>
1 parent 92b572c commit ae4eb4b

16 files changed

+215
-323
lines changed

‎debian/ibverbs-providers.install

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ usr/lib/*/libibverbs/lib*-rdmav*.so
55
usr/lib/*/libmana.so.*
66
usr/lib/*/libmlx4.so.*
77
usr/lib/*/libmlx5.so.*
8+
usr/lib/*/libzrdma.so.*

‎debian/ibverbs-providers.symbols

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,3 +188,13 @@ libmana.so.1 ibverbs-providers #MINVER#
188188
MANA_1.0@MANA_1.0 41
189189
manadv_init_obj@MANA_1.0 41
190190
manadv_set_context_attr@MANA_1.0 41
191+
* Build-Depends-Package: libibverbs-dev
192+
ZRDMA_1.0@ZRDMA_1.0 57
193+
zxdh_get_log_trace_switch@ZRDMA_1.0 57
194+
ZRDMA_1.1@ZRDMA_1.1 57
195+
zxdh_set_log_trace_switch@ZRDMA_1.1 57
196+
zxdh_modify_qp_udp_sport@ZRDMA_1.1 57
197+
zxdh_query_qpc@ZRDMA_1.1 57
198+
zxdh_modify_qpc@ZRDMA_1.1 57
199+
zxdh_reset_qp@ZRDMA_1.1 57
200+

‎debian/libibverbs-dev.install

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ usr/include/infiniband/sa.h
1313
usr/include/infiniband/tm_types.h
1414
usr/include/infiniband/verbs.h
1515
usr/include/infiniband/verbs_api.h
16+
usr/include/infiniband/zxdh_devids.h
17+
usr/include/infiniband/zxdh_dv.h
1618
usr/lib/*/lib*-rdmav*.a
1719
usr/lib/*/libefa.a
1820
usr/lib/*/libefa.so
@@ -26,12 +28,15 @@ usr/lib/*/libmlx4.a
2628
usr/lib/*/libmlx4.so
2729
usr/lib/*/libmlx5.a
2830
usr/lib/*/libmlx5.so
31+
usr/lib/*/libzrdma.a
32+
usr/lib/*/libzrdma.so
2933
usr/lib/*/pkgconfig/libefa.pc
3034
usr/lib/*/pkgconfig/libhns.pc
3135
usr/lib/*/pkgconfig/libibverbs.pc
3236
usr/lib/*/pkgconfig/libmana.pc
3337
usr/lib/*/pkgconfig/libmlx4.pc
3438
usr/lib/*/pkgconfig/libmlx5.pc
39+
usr/lib/*//pkgconfig/libzrdma.pc
3540
usr/share/man/man3/efadv_*.3
3641
usr/share/man/man3/hnsdv_*.3
3742
usr/share/man/man3/ibv_*

‎debian/libibverbs1.install

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
usr/lib/*/libibverbs*.so.*
2+
usr/lib/*/libzrdma.so.*
3+
usr/lib/*/libzrdma.a
24
usr/share/doc/rdma-core/libibverbs.md usr/share/doc/libibverbs1/

‎kernel-headers/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ publish_internal_headers(rdma
2727
rdma/siw-abi.h
2828
rdma/vmw_pvrdma-abi.h
2929
rdma/zxdh-abi.h
30+
rdma/zxdh_user_ioctl_cmds.h
3031
rdma/zxdh_user_ioctl_verbs.h
3132
)
3233

‎providers/zrdma/CMakeLists.txt

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
# Copyright (c) 2024 ZTE Corporation.
33

44
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
5-
rdma_provider(zrdma
5+
rdma_shared_provider(zrdma libzrdma.map
6+
1 1.1.${PACKAGE_VERSION}
67
zxdh_hw.c
78
zxdh_zrdma.c
89
zxdh_verbs.c
@@ -16,12 +17,3 @@ publish_headers(infiniband
1617
zxdh_devids.h
1718
)
1819

19-
if(TARGET zrdma)
20-
target_link_libraries(zrdma PRIVATE nl-3)
21-
rdma_pkg_config("zrdma" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")
22-
endif()
23-
if(TARGET zrdma-rdmav${IBVERBS_PABI_VERSION})
24-
target_link_libraries(zrdma-rdmav${IBVERBS_PABI_VERSION} PRIVATE nl-3)
25-
rdma_pkg_config("zrdma-rdmav${IBVERBS_PABI_VERSION}" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")
26-
endif()
27-

‎providers/zrdma/libzrdma.map

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,16 @@
11
/* Export symbols should be added below according to
22
Documentation/versioning.md document. */
33
ZRDMA_1.0 {
4+
global:
5+
zxdh_get_log_trace_switch;
46
local: *;
57
};
8+
9+
ZRDMA_1.1 {
10+
global:
11+
zxdh_set_log_trace_switch;
12+
zxdh_modify_qp_udp_sport;
13+
zxdh_query_qpc;
14+
zxdh_modify_qpc;
15+
zxdh_reset_qp;
16+
} ZRDMA_1.0;

‎providers/zrdma/private_verbs_cmd.c

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,92 @@ static int _zxdh_query_qpc(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc)
6969
return 0;
7070
}
7171

72+
static void copy_modify_qpc_fields(struct zxdh_modify_qpc_req *req_cmd,
73+
uint64_t attr_mask,
74+
struct zxdh_rdma_qpc *qpc)
75+
{
76+
if (attr_mask & ZXDH_TX_READ_RETRY_FLAG_SET) {
77+
req_cmd->retry_flag = qpc->retry_flag;
78+
req_cmd->rnr_retry_flag = qpc->rnr_retry_flag;
79+
req_cmd->read_retry_flag = qpc->read_retry_flag;
80+
req_cmd->cur_retry_count = qpc->cur_retry_count;
81+
}
82+
if (attr_mask & ZXDH_RETRY_CQE_SQ_OPCODE)
83+
req_cmd->retry_cqe_sq_opcode = qpc->retry_cqe_sq_opcode;
84+
85+
if (attr_mask & ZXDH_ERR_FLAG_SET) {
86+
req_cmd->err_flag = qpc->err_flag;
87+
req_cmd->ack_err_flag = qpc->ack_err_flag;
88+
}
89+
if (attr_mask & ZXDH_PACKAGE_ERR_FLAG)
90+
req_cmd->package_err_flag = qpc->package_err_flag;
91+
}
92+
93+
static int _zxdh_reset_qp(struct ibv_qp *qp, uint64_t opcode)
94+
{
95+
DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ,
96+
ZXDH_IB_METHOD_QP_RESET_QP, 2);
97+
fill_attr_in_obj(cmd, ZXDH_IB_ATTR_QP_RESET_QP_HANDLE, qp->handle);
98+
fill_attr_in_uint64(cmd, ZXDH_IB_ATTR_QP_RESET_OP_CODE, opcode);
99+
return execute_ioctl(qp->context, cmd);
100+
}
101+
102+
static int _zxdh_modify_qpc(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc,
103+
uint64_t qpc_mask)
104+
{
105+
DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ,
106+
ZXDH_IB_METHOD_QP_MODIFY_QPC, 3);
107+
struct zxdh_modify_qpc_req req = { 0 };
108+
109+
copy_modify_qpc_fields(&req, qpc_mask, qpc);
110+
fill_attr_in_obj(cmd, ZXDH_IB_ATTR_QP_QUERY_HANDLE, qp->handle);
111+
fill_attr_in_uint64(cmd, ZXDH_IB_ATTR_QP_MODIFY_QPC_MASK, qpc_mask);
112+
fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_QP_MODIFY_QPC_REQ, &req);
113+
return execute_ioctl(qp->context, cmd);
114+
}
115+
116+
static int _zxdh_modify_qp_udp_sport(struct ibv_context *ibctx,
117+
uint16_t udp_sport, uint32_t qpn)
118+
{
119+
if (udp_sport <= MIN_UDP_SPORT || qpn <= MIN_QP_QPN)
120+
return -EINVAL;
121+
122+
DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ,
123+
ZXDH_IB_METHOD_QP_MODIFY_UDP_SPORT, 2);
124+
fill_attr_in(cmd, ZXDH_IB_ATTR_QP_UDP_PORT, &udp_sport,
125+
sizeof(udp_sport));
126+
fill_attr_in_uint32(cmd, ZXDH_IB_ATTR_QP_QPN, qpn);
127+
return execute_ioctl(ibctx, cmd);
128+
}
129+
130+
static int _zxdh_get_log_trace_switch(struct ibv_context *ibctx,
131+
uint8_t *switch_status)
132+
{
133+
DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV,
134+
ZXDH_IB_METHOD_DEV_GET_LOG_TRACE, 1);
135+
136+
fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_LOG_TARCE_SWITCH,
137+
switch_status);
138+
return execute_ioctl(ibctx, cmd);
139+
}
140+
141+
static int _zxdh_set_log_trace_switch(struct ibv_context *ibctx,
142+
uint8_t switch_status)
143+
{
144+
DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV,
145+
ZXDH_IB_METHOD_DEV_SET_LOG_TRACE, 1);
146+
fill_attr_in(cmd, ZXDH_IB_ATTR_DEV_SET_LOG_TARCE_SWITCH, &switch_status,
147+
sizeof(switch_status));
148+
return execute_ioctl(ibctx, cmd);
149+
}
150+
72151
static struct zxdh_uvcontext_ops zxdh_ctx_ops = {
152+
.modify_qp_udp_sport = _zxdh_modify_qp_udp_sport,
153+
.get_log_trace_switch = _zxdh_get_log_trace_switch,
154+
.set_log_trace_switch = _zxdh_set_log_trace_switch,
73155
.query_qpc = _zxdh_query_qpc,
156+
.modify_qpc = _zxdh_modify_qpc,
157+
.reset_qp = _zxdh_reset_qp,
74158
};
75159

76160
static inline struct zxdh_uvcontext *to_zxdhtx(struct ibv_context *ibctx)

‎providers/zrdma/private_verbs_cmd.h

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -36,16 +36,6 @@
3636
#include "zxdh_zrdma.h"
3737
#include "zxdh_dv.h"
3838

39-
enum zxdh_rdma_tool_flags {
40-
ZXDH_QP_EXTEND_OP = 1 << 0,
41-
ZXDH_CAPTURE = 1 << 1,
42-
ZXDH_GET_HW_DATA = 1 << 2,
43-
ZXDH_GET_HW_OBJECT_DATA = 1 << 3,
44-
ZXDH_CHECK_HW_HEALTH = 1 << 4,
45-
ZXDH_RDMA_TOOL_CFG_DEV_PARAM = 1 << 5,
46-
ZXDH_RDMA_TOOL_SHOW_RES_MAP = 1 << 5,
47-
};
48-
4939
struct zxdh_uvcontext_ops {
5040
int (*modify_qp_udp_sport)(struct ibv_context *ibctx,
5141
uint16_t udp_sport, uint32_t qpn);

‎providers/zrdma/zxdh_devids.h

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,20 @@
3333
#ifndef __ZXDH_DEVIDS_H__
3434
#define __ZXDH_DEVIDS_H__
3535

36+
#ifdef __cplusplus
37+
extern "C" {
38+
#endif
39+
3640
/* ZXDH VENDOR ID */
3741
#define PCI_VENDOR_ID_ZXDH_EVB 0x16c3
3842
#define PCI_VENDOR_ID_ZXDH_E312 0x1cf2
43+
#define PCI_VENDOR_ID_ZXDH_E312S_D 0x1cf2
3944
#define PCI_VENDOR_ID_ZXDH_E310 0x1cf2
4045
#define PCI_VENDOR_ID_ZXDH_E310_RDMA 0x1cf2
4146
#define PCI_VENDOR_ID_ZXDH_E316 0x1cf2
4247
#define PCI_VENDOR_ID_ZXDH_X512 0x1cf2
48+
#define PCI_VENDOR_ID_ZXDH_E312_TY_CLOUD 0x1b18
49+
#define PCI_VENDOR_ID_ZXDH_E310_TY_CLOUD 0x1b18
4350
/* ZXDH Devices ID */
4451
#define ZXDH_DEV_ID_ADAPTIVE_EVB_PF 0x8040 /* ZXDH EVB PF DEVICE ID*/
4552
#define ZXDH_DEV_ID_ADAPTIVE_EVB_VF 0x8041 /* ZXDH EVB VF DEVICE ID*/
@@ -53,4 +60,15 @@
5360
#define ZXDH_DEV_ID_ADAPTIVE_E316_VF 0x807f /* ZXDH E316 VF DEVICE ID*/
5461
#define ZXDH_DEV_ID_ADAPTIVE_X512_PF 0x806B /* ZXDH X512 PF DEVICE ID*/
5562
#define ZXDH_DEV_ID_ADAPTIVE_X512_VF 0x806C /* ZXDH X512 VF DEVICE ID*/
56-
#endif /* __ZXDH_DEVIDS_H__ */
63+
#define ZXDH_DEV_ID_ADAPTIVE_E312_TY_CLOUD_PF 0x1110 /* ZXDH E312 TY CLOUD PF DEVICE ID*/
64+
#define ZXDH_DEV_ID_ADAPTIVE_E312_TY_CLOUD_VF 0x1111 /* ZXDH E312 TY CLOUD VF DEVICE ID*/
65+
#define ZXDH_DEV_ID_ADAPTIVE_E310_TY_CLOUD_PF 0x1100 /* ZXDH E310 TY CLOUD PF DEVICE ID*/
66+
#define ZXDH_DEV_ID_ADAPTIVE_E310_TY_CLOUD_VF 0x1101 /* ZXDH E310 TY CLOUD VF DEVICE ID*/
67+
#define ZXDH_DEV_ID_ADAPTIVE_E312S_D_PF 0x80a2 /* ZXDH E310 TY CLOUD PF DEVICE ID*/
68+
#define ZXDH_DEV_ID_ADAPTIVE_E312S_D_VF 0x80a3 /* ZXDH E310 TY CLOUD VF DEVICE ID*/
69+
70+
#ifdef __cplusplus
71+
}
72+
#endif
73+
74+
#endif /* ZXDH_DEVIDS_H */

0 commit comments

Comments
 (0)