CONTRIB/IBMOCK: Inital add with EFA support

buildlib/pr/efa.yml

@@ -0,0 +1,31 @@
+parameters:
+  demands: []
+  container: rhel90_ib
+
+jobs:
+  - job: tests_${{ parameters.name }}
+    pool:
+      name: MLNX
+      demands: ${{ parameters.demands }}
+    displayName: ${{ parameters.name }} on ${{ parameters.container }}
+    container: ${{ parameters.container }}
+    timeoutInMinutes: 90
+    workspace:
+      clean: outputs
+    steps:
+      - checkout: self
+        clean: true
+        fetchDepth: 100
+        retryCountOnTaskFailure: 5
+      - bash: |
+          ./contrib/test_efa.sh build_efa
+        displayName: Build rdma-core and ibmock
+      - bash: |
+          ./contrib/test_efa.sh gtest_efa
+        displayName: Run gtests
+      - bash: |
+          ./contrib/test_efa.sh install_rdma_core_efa
+        displayName: Setup rmda-core
+      - bash: |
+          ./contrib/test_efa.sh test_rpm_efa
+        displayName: RPM test

buildlib/pr/main.yml

@@ -33,6 +33,9 @@ resources:
     - container: rhel90
       image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/rhel9.0/builder:mofed-5.6-0.5.0.0
       options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
+    - container: rhel90_ib
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/rhel9.0/builder:mofed-5.6-0.5.0.0
+      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(DOCKER_OPT_IB)
     - container: ubuntu2004
       image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu20.04/builder:mofed-5.0-1.0.0.0
       options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
@@ -343,6 +346,14 @@ stages:
         run_tests: yes
         test_perf: 0
 
+  - stage: EFA_Tests
+    dependsOn: [Static_check]
+    jobs:
+    - template: efa.yml
+      parameters:
+        name: EFA
+        demands: ucx_new -equals yes
+
   - stage: Namespace_Tests
     dependsOn: [Static_check]
     jobs:

contrib/ibmock/Makefile

@@ -0,0 +1,43 @@
+#
+# Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2025. ALL RIGHTS RESERVED.
+# See file LICENSE for terms.
+#
+
+CC ?= gcc
+CFLAGS = -ggdb3 -fPIC -Wall -Wextra -Werror
+
+INCLUDES ?=
+BUILD ?= ./build
+
+MAJOR = 1
+MINOR = 1
+VERSION = $(MAJOR).$(MINOR)
+
+OBJS_IBVERBS = $(addprefix $(BUILD)/, verbs.o config.o stubs.o)
+OBJS_EFA = $(addprefix $(BUILD)/, efa.o config.o)
+
+.PHONY: all clean make_dir
+
+all: make_dir $(BUILD)/libibverbs.so.1 $(BUILD)/libefa.so.1
+
+make_dir: $(BUILD)/
+$(BUILD)/:
+	mkdir -p $(BUILD)
+
+$(OBJS_EFA) $(OBJS_IBVERBS): *.h
+
+$(BUILD)/libibverbs.so: $(OBJS_IBVERBS)
+$(BUILD)/libefa.so: $(OBJS_EFA)
+$(BUILD)/%.so.$(MAJOR): $(BUILD)/%.so
+
+$(BUILD)/%.o: %.c
+	$(CC) -c $(INCLUDES) $(CFLAGS) -o $@ $<
+
+$(BUILD)/%.so:
+	$(CC) -shared -o $@ -Wl,--soname='$@.$(VERSION)' $^
+
+$(BUILD)/%.so.$(MAJOR): $(BUILD)/%.so
+	ln -sf $(shell basename $<) $@
+
+clean:
+	rm -f $(addprefix $(BUILD)/, *.o *.so.* *.so)

contrib/ibmock/README.md

@@ -0,0 +1,33 @@
+## What
+IB devices mock implementation. Only EFA devices from same process are supported for now, UD and SRD.
+
+## How
+
+### Build
+rdma-core (optional)
+```
+cd rdma-core
+./build.sh
+```
+
+UCX
+```
+cd ucx
+./autogen.sh
+./contrib/configure-devel --with-verbs=$(pwd)/../rdma-core/build --with-efa
+make -j && make install
+```
+
+IB mock
+```
+cd ibmock
+make INCLUDES=-I$(pwd)/../rdma-core/build/include
+```
+
+### Run
+```
+export LD_LIBRARY_PATH=$(pwd)/../efa_mock/build:$LD_LIBRARY_PATH
+UCX_TLS=srd ucx_perftest -t tag_bw -l
+UCX_TLS=ud ucx_perftest -t tag_bw -l
+ucx_info -d
+```

contrib/ibmock/config.c

@@ -0,0 +1,109 @@
+/**
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See file LICENSE for terms.
+ */
+
+#include "config.h"
+
+struct efadv_device_attr efa_dev_attr = {
+    .comp_mask       = 0,
+    .max_sq_wr       = 4096,
+    .max_rq_wr       = 32768,
+    .max_sq_sge      = 2,
+    .max_rq_sge      = 3,
+    .inline_buf_size = 32,
+    .device_caps     = 15,
+    .max_rdma_size   = 1073741824
+};
+
+struct ibv_device_attr efa_ibv_dev_attr = {
+    .fw_ver                    = "0.0.0.0",
+    .node_guid                 = 0,
+    .sys_image_guid            = 0,
+    .max_mr_size               = 103079215104,
+    .page_size_cap             = 4294963200,
+    .vendor_id                 = 7439,
+    .vendor_part_id            = 61345,
+    .hw_ver                    = 61345,
+    .max_qp                    = 256,
+    .max_qp_wr                 = 4096,
+    .device_cap_flags          = 0,
+    .max_sge                   = 2,
+    .max_sge_rd                = 1,
+    .max_cq                    = 512,
+    .max_cqe                   = 32768,
+    .max_mr                    = 262144,
+    .max_pd                    = 256,
+    .max_qp_rd_atom            = 0,
+    .max_ee_rd_atom            = 0,
+    .max_res_rd_atom           = 0,
+    .max_qp_init_rd_atom       = 0,
+    .max_ee_init_rd_atom       = 0,
+    .atomic_cap                = IBV_ATOMIC_NONE,
+    .max_ee                    = 0,
+    .max_rdd                   = 0,
+    .max_mw                    = 0,
+    .max_raw_ipv6_qp           = 0,
+    .max_raw_ethy_qp           = 0,
+    .max_mcast_grp             = 0,
+    .max_mcast_qp_attach       = 0,
+    .max_total_mcast_qp_attach = 0,
+    .max_ah                    = 1024,
+    .max_fmr                   = 0,
+    .max_map_per_fmr           = 0,
+    .max_srq                   = 0,
+    .max_srq_wr                = 0,
+    .max_srq_sge               = 0,
+    .max_pkeys                 = 1,
+    .local_ca_ack_delay        = 0,
+    .phys_port_cnt             = 1
+};
+
+struct ibv_port_attr efa_ib_port_attr = {
+    .state           = IBV_PORT_ACTIVE,
+    .max_mtu         = IBV_MTU_4096,
+    .active_mtu      = IBV_MTU_4096,
+    .gid_tbl_len     = 1,
+    .port_cap_flags  = 0,
+    .max_msg_sz      = 8928,
+    .bad_pkey_cntr   = 0,
+    .qkey_viol_cntr  = 0,
+    .pkey_tbl_len    = 1,
+    .lid             = 0,
+    .sm_lid          = 0,
+    .lmc             = 1,
+    .max_vl_num      = 1,
+    .sm_sl           = 0,
+    .subnet_timeout  = 0,
+    .init_type_reply = 0,
+    .active_width    = 2,
+    .active_speed    = 32,
+    .phys_state      = 5,
+    .link_layer      = 0,
+    .flags           = 0,
+    .port_cap_flags2 = 0,
+    /* .active_speed_ex = 0 */
+};
+
+struct ibv_qp_attr efa_ib_qp_attr = {
+    .cap = {
+        .max_send_wr     = 256,
+        .max_recv_wr     = 4096,
+        .max_send_sge    = 2,
+        .max_recv_sge    = 1,
+        .max_inline_data = 32
+    }
+};
+
+struct ibv_qp_init_attr efa_ib_qp_init_attr = {
+    .srq        = 0x0,
+    .cap        = {
+        .max_send_wr     = 256,
+        .max_recv_wr     = 4096,
+        .max_send_sge    = 2,
+        .max_recv_sge    = 1,
+        .max_inline_data = 32
+    },
+    .sq_sig_all = 1
+};

contrib/ibmock/config.h

@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See file LICENSE for terms.
+ */
+
+#ifndef __CONFIG_H
+#define __CONFIG_H
+
+#include <infiniband/efadv.h>
+#include <infiniband/verbs.h>
+
+
+extern struct efadv_device_attr efa_dev_attr;
+extern struct ibv_device_attr efa_ibv_dev_attr;
+extern struct ibv_port_attr efa_ib_port_attr;
+extern struct ibv_qp_attr efa_ib_qp_attr;
+extern struct ibv_qp_init_attr efa_ib_qp_init_attr;
+
+#endif

contrib/ibmock/efa.c

@@ -0,0 +1,102 @@
+/**
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See file LICENSE for terms.
+ */
+
+#include <infiniband/efadv.h>
+#include <infiniband/verbs.h>
+
+#include <errno.h>
+#include <stdlib.h>
+
+#include "fake.h"
+#include "verbs.h"
+#include "config.h"
+
+int efadv_query_device(struct ibv_context *context,
+                       struct efadv_device_attr *attr, uint32_t inlen)
+{
+    if ((context == NULL) || (inlen != sizeof(efa_dev_attr))) {
+        return EINVAL;
+    }
+
+    if ((context->device->node_type != IBV_NODE_UNSPECIFIED) ||
+        (context->device->transport_type != IBV_TRANSPORT_UNSPECIFIED)) {
+        return ENOTSUP;
+    }
+
+    memcpy(attr, &efa_dev_attr, sizeof(efa_dev_attr));
+    return 0;
+}
+
+struct ibv_qp *efadv_create_driver_qp_impl(struct ibv_pd           *pd,
+                                           struct ibv_qp_init_attr *attr,
+                                           uint32_t driver_qp_type)
+{
+    struct fake_qp *fqp;
+    struct ibv_qp *qp;
+
+
+    fqp = calloc(1, sizeof(*fqp));
+    if (fqp == NULL) {
+        return NULL;
+    }
+
+    fqp->fpd = (struct fake_pd*)pd;
+
+    fqp->qp_ex.wr_start        = dev_qp_wr_start;
+    fqp->qp_ex.wr_rdma_read    = dev_qp_wr_rdma_read;
+    fqp->qp_ex.wr_set_sge_list = dev_qp_wr_set_sge_list;
+    fqp->qp_ex.wr_set_ud_addr  = dev_qp_wr_set_ud_addr;
+    fqp->qp_ex.wr_complete     = dev_qp_wr_complete;
+
+    list_init(&fqp->recv_reqs);
+
+    qp             = &fqp->qp_ex.qp_base;
+    qp->qp_context = pd->context;
+    qp->context    = pd->context;
+    qp->qp_type    = driver_qp_type;
+    qp->send_cq    = attr->send_cq;
+    qp->recv_cq    = attr->recv_cq;
+    qp->pd         = pd;
+    qp->state      = IBV_QPS_RESET;
+
+    lock();
+    qp->qp_num = ++fake_qpn;
+    array_append(&fake_qps, &fqp, sizeof(fqp));
+    unlock();
+    return qp;
+}
+
+struct ibv_qp *efadv_create_driver_qp(struct ibv_pd           *pd,
+                                      struct ibv_qp_init_attr *attr,
+                                      uint32_t                driver_qp_type)
+{
+    if ((attr->qp_type != IBV_QPT_DRIVER) ||
+        (driver_qp_type != EFADV_QP_DRIVER_TYPE_SRD)) {
+        return NULL;
+    }
+
+    return efadv_create_driver_qp_impl(pd, attr, driver_qp_type);
+}
+
+struct ibv_qp *efadv_create_qp_ex(struct ibv_context         *context,
+                                  struct ibv_qp_init_attr_ex *attr_ex,
+                                  struct efadv_qp_init_attr  *efa_attr,
+                                  uint32_t                   inlen)
+{
+    struct ibv_qp_init_attr attr = {
+        .qp_type = efa_attr->driver_qp_type,
+        .send_cq = attr_ex->send_cq,
+        .recv_cq = attr_ex->recv_cq
+    };
+    (void)context;
+
+    if (inlen != sizeof(*efa_attr)) {
+        return NULL;
+    }
+
+    return efadv_create_driver_qp_impl(attr_ex->pd, &attr,
+                                       efa_attr->driver_qp_type);
+}