From abe8cc361cef52ab8f1ced5c6d55acfd6ffb1cf3 Mon Sep 17 00:00:00 2001 From: Mike Uttormark Date: Mon, 16 Dec 2024 15:03:31 -0600 Subject: [PATCH] prov/cxi: Test monitor unsubscribe An MR cache utilizing kdreg2 will have incorrect MR cache count stats if unsubscribe is not called. Signed-off-by: Mike Uttormark Signed-off-by: Ian Ziemba --- prov/cxi/Makefile.include | 3 +- prov/cxi/test/mr_cache.c | 121 ++++++++++++++++++++++++++++++++++++++ prov/cxi/test/test.sh | 3 + 3 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 prov/cxi/test/mr_cache.c diff --git a/prov/cxi/Makefile.include b/prov/cxi/Makefile.include index b529f942ce7..9faa7874d8f 100644 --- a/prov/cxi/Makefile.include +++ b/prov/cxi/Makefile.include @@ -129,7 +129,8 @@ nodist_prov_cxi_test_cxitest_SOURCES = \ prov/cxi/test/auth_key.c \ prov/cxi/test/fork.c \ prov/cxi/test/mem_reg.c \ - prov/cxi/test/nic.c + prov/cxi/test/nic.c \ + prov/cxi/test/mr_cache.c prov_cxi_test_cxitest_CPPFLAGS = $(AM_CPPFLAGS) $(cxi_CPPFLAGS) \ $(cxitest_CPPFLAGS) $(PTHREAD_CFLAGS) diff --git a/prov/cxi/test/mr_cache.c b/prov/cxi/test/mr_cache.c new file mode 100644 index 00000000000..b2035cb8063 --- /dev/null +++ b/prov/cxi/test/mr_cache.c @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2024 Hewlett Packard Enterprise Development LP + * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "libcxi/libcxi.h" +#include "cxip.h" +#include "cxip_test_common.h" + +#define SETENV_OVERWRITE 1 + +TestSuite(mr_cache, .timeout = CXIT_DEFAULT_TIMEOUT); + +Test(mr_cache, cache_full) +{ + static struct { + const char *name; + const char *value; + } envs[] = { + { .name = "FI_MR_CACHE_MONITOR", .value = "kdreg2", }, + { .name = "FI_MR_CACHE_MAX_COUNT", .value = "4", }, + }; + struct { + void *addr; + struct fid_mr *mr; + } *region_data; + size_t i; + int ret; + long page_size; + unsigned long num_regions, total_regions; + struct ofi_mr_cache *cache; + struct cxip_domain *cxip_dom; + + /* setup the environment */ + for (i = 0; i < ARRAY_SIZE(envs); i++) { + ret = setenv(envs[i].name, envs[i].value, SETENV_OVERWRITE); + cr_assert_eq(ret, 0, "Failed to set %s to %s: %d", + envs[i].name, envs[i].value, errno); + } + + /* allocate the memory regions */ + page_size = sysconf(_SC_PAGESIZE); + cr_assert(page_size > 0, + "sysconf(_SC_PAGESIZE) return %ld: errno = %d", page_size, errno); + + ret = sscanf(getenv("FI_MR_CACHE_MAX_COUNT"), "%lu", &num_regions); + cr_assert_eq(ret, 1, "Failed to get number of regions: %d %d:%s", + ret, errno, strerror(errno)); + + /* one extra to push one out of the cache */ + total_regions = num_regions + 1; + region_data = calloc(total_regions, sizeof(*region_data)); + cr_assert_not_null(region_data); + + for (i = 0; i < total_regions; i++) { + region_data[i].addr = mmap(NULL, page_size, + PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, + -1, 0); + cr_assert_not_null(region_data[i].addr); + } + + /* create the domain */ + cxit_setup_domain(); + cxit_create_domain(); + + /* Register the max number of regions */ + for (i = 0; i < num_regions; i++) { + ret = fi_mr_reg(cxit_domain, region_data[i].addr, + page_size, FI_READ | FI_WRITE, + 0, 0, 0, ®ion_data[i].mr, NULL); + cr_assert_eq(ret, FI_SUCCESS, + "fi_mr_reg failed for region %lu: %d", i, ret); + } + + /* See that the cache is full */ + cxip_dom = container_of(cxit_domain, struct cxip_domain, + util_domain.domain_fid); + cache = &cxip_dom->iomm; + cr_assert(cache->cached_cnt == cache->cached_max_cnt, + "Cache is not full: %zu != %zu", + cache->cached_cnt, cache->cached_max_cnt); + cr_assert(cache->uncached_cnt == 0, + "Cache has uncached entries: %zu", + cache->uncached_cnt); + + /* release the registrations, this should put them on the LRU list */ + for(i = 0; i < num_regions; i++) { + ret = fi_close(®ion_data[i].mr->fid); + cr_assert_eq(ret, FI_SUCCESS, + "Failed to close mr %zu: %d", + i, ret); + } + + /* Register one more, this should push one off LRU list */ + ret = fi_mr_reg(cxit_domain, region_data[num_regions].addr, + page_size, FI_READ | FI_WRITE, + 0, 0, 0, ®ion_data[num_regions].mr, NULL); + cr_assert_eq(ret, FI_SUCCESS, + "fi_mr_reg failed for region %lu: %d", num_regions, ret); + + /* Cache should remain full */ + cr_assert(cache->cached_cnt == cache->cached_max_cnt, + "Cache is not full: %zu != %zu", + cache->cached_cnt, cache->cached_max_cnt); + cr_assert(cache->uncached_cnt == 0, + "Cache has uncached entries: %zu", + cache->uncached_cnt); + + cxit_teardown_domain(); +} diff --git a/prov/cxi/test/test.sh b/prov/cxi/test/test.sh index ea6a913703f..21914365f87 100755 --- a/prov/cxi/test/test.sh +++ b/prov/cxi/test/test.sh @@ -149,6 +149,8 @@ fork_safe_kdreg2_test=( unlimited_triggered_ops_test=( "FI_CXI_ENABLE_TRIG_OP_LIMIT=0 ./cxitest -j 1 --verbose --filter=\"deferred_work_trig_op_limit/*\" --tap=cxitest-disable-trig-op-limit.tap") +mr_cache_test=("./cxitest --verbose --tap=cxitest-mr_cache_test.tap --filter=\"mr_cache/*\" -j 1") + long_test_suite=( "basic_test" "swget_test" @@ -174,6 +176,7 @@ long_test_suite=( "fork_safe_memhooks_test" "fork_safe_kdreg2_test" "unlimited_triggered_ops_test" + "mr_cache_test" ) # ################################################################