diff --git a/CMakeLists.txt b/CMakeLists.txt index 780c4f70d..c5e9633b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -105,17 +105,19 @@ endif(USE_HIP) ##################### ## GOOGLETEST ## ##################### -include(FetchContent) - -FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - # Specify the commit you depend on and update it regularly. - GIT_TAG 58d77fa8070e8cec2dc1ed015d66b454c8d78850 -) -# For Windows: Prevent overriding the parent project's compiler/linker settings -set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) -FetchContent_MakeAvailable(googletest) +if (ENABLE_UNIT_TESTS) + include(FetchContent) + + FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + # Specify the commit you depend on and update it regularly. + GIT_TAG 58d77fa8070e8cec2dc1ed015d66b454c8d78850 + ) + # For Windows: Prevent overriding the parent project's compiler/linker settings + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(googletest) +endif() ##################### include_directories(.) @@ -137,5 +139,7 @@ endif(USE_CUDA) add_subdirectory(benchmarks) -enable_testing() +if (ENABLE_UNIT_TESTS) + enable_testing() +endif() diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 838f1efb3..d5d8e37be 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -22,3 +22,6 @@ target_link_libraries(alltoallv_crs mpi_advance ${MPI_LIBRARIES} ) add_executable(microbenchmarks microbenchmarks.cpp) target_link_libraries(microbenchmarks mpi_advance ${MPI_LIBRARIES} ) + +add_executable(alltoall_topology alltoall_topology.cpp) +target_link_libraries(alltoall_topology mpi_advance ${MPI_LIBRARIES} ) diff --git a/benchmarks/alltoall_topology.cpp b/benchmarks/alltoall_topology.cpp new file mode 100644 index 000000000..97519d25e --- /dev/null +++ b/benchmarks/alltoall_topology.cpp @@ -0,0 +1,864 @@ +#include +#include + +#include +#include +#include "mpi_advance.h" + +using A = int (*)(const void *, int, MPI_Datatype, void *, int, MPI_Datatype, MPI_Comm); +template +double time_alltoall(F alltoall_func, const void *sendbuf, const int sendcount, + MPI_Datatype sendtype, void *recvbuf, const int recvcount, MPI_Datatype recvtype, + C comm, int n_iters) +{ + MPI_Barrier(MPI_COMM_WORLD); + double t0 = MPI_Wtime(); + for (int i = 0; i < n_iters; i++) + { + alltoall_func(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); + } + double tfinal = (MPI_Wtime() - t0) / n_iters; + MPI_Allreduce(&tfinal, &t0, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + return t0; +} + +// template +double time_alltoall_subset(A alltoall_func, const void *sendbuf, const int sendcount, + MPI_Datatype sendtype, void *recvbuf, const int recvcount, + MPI_Datatype recvtype, MPI_Comm comm, int n_iters) +{ + MPI_Barrier(comm); + double t0 = MPI_Wtime(); + for (int i = 0; i < n_iters; i++) + { + alltoall_func(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); + } + double tfinal = (MPI_Wtime() - t0) / n_iters; + MPI_Allreduce(&tfinal, &t0, 1, MPI_DOUBLE, MPI_MAX, comm); + return t0; +} + +template +int estimate_alltoall_iters(F alltoall_func, const void *sendbuf, int sendcount, + MPI_Datatype sendtype, void *recvbuf, int recvcount, + MPI_Datatype recvtype, C comm) +{ + double time = time_alltoall(alltoall_func, sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, 1); + MPI_Allreduce(MPI_IN_PLACE, &time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + int n_iters = 1; + if (time > 1) + n_iters = 1; + else + { + if (time > 1e-01) + time = time_alltoall(alltoall_func, sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, 2); + else if (time > 1e-02) + time = time_alltoall(alltoall_func, sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, 10); + else + time = time_alltoall(alltoall_func, sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, 100); + + MPI_Allreduce(MPI_IN_PLACE, &time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + n_iters = (1.0 / time) + 1; + if (n_iters < 1) + n_iters = 1; + } + + return n_iters; +} + +double time_gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, + int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm, int n_iters) +{ + MPI_Barrier(MPI_COMM_WORLD); + double t0 = MPI_Wtime(); + for (int i = 0; i < n_iters; i++) + { + MPI_Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm); + } + double tfinal = (MPI_Wtime() - t0) / n_iters; + MPI_Allreduce(&tfinal, &t0, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + return t0; +} + +int estimate_gather_iters(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, + int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm) +{ + double time = time_gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, 1); + MPI_Allreduce(MPI_IN_PLACE, &time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + int n_iters = 1; + if (time > 1) + n_iters = 1; + else + { + if (time > 1e-01) + time = time_gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, 2); + else if (time > 1e-02) + time = time_gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, 10); + else + time = time_gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, 100); + + MPI_Allreduce(MPI_IN_PLACE, &time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + n_iters = (1.0 / time) + 1; + if (n_iters < 1) + n_iters = 1; + } + + return n_iters; +} + +double time_scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, + int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm, int n_iters) +{ + MPI_Barrier(MPI_COMM_WORLD); + double t0 = MPI_Wtime(); + for (int i = 0; i < n_iters; i++) + { + MPI_Scatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm); + } + double tfinal = (MPI_Wtime() - t0) / n_iters; + MPI_Allreduce(&tfinal, &t0, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + return t0; +} + +int estimate_scatter_iters(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, + MPI_Comm comm) +{ + double time = time_scatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, 1); + int n_iters = 1; + if (time > 1) + n_iters = 1; + else + { + if (time > 1e-01) + time = time_scatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, 2); + else if (time > 1e-02) + time = time_scatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, 10); + else + time = time_scatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, 100); + + MPI_Allreduce(MPI_IN_PLACE, &time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + n_iters = (1.0 / time) + 1; + if (n_iters < 1) + n_iters = 1; + } + + return n_iters; +} + +template +double test_alltoall(F alltoall_func, const void *sendbuf, const int sendcount, + MPI_Datatype sendtype, void *recvbuf, const int recvcount, MPI_Datatype recvtype, + C comm) +{ + double time; + int n_iters; + + // Warm-Up + time_alltoall(alltoall_func, sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, comm, 1); + + // Estimate Iterations + n_iters = estimate_alltoall_iters(alltoall_func, sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, comm); + // time = time_alltoall(alltoall_func, sendbuf, sendcount, sendtype, + // recvbuf, recvcount, recvtype, comm, 2); + // MPI_Allreduce(MPI_IN_PLACE, &time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + // n_iters = (1.0 / time) + 1; + + // Time Alltoall + time = time_alltoall(alltoall_func, sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, comm, n_iters); + MPI_Allreduce(MPI_IN_PLACE, &time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + return time; +} + +// template +void internal_hierarchical_timing(A alltoall_func, const void *sendbuf, const int sendcount, + MPI_Datatype sendtype, void *recvbuf, const int recvcount, + MPI_Datatype recvtype, MPIX_Comm *comm, const char *name, + int procsPerLeader) +{ + printf("internal multileader timing\n"); + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char *recv_buffer = (char *)recvbuf; + char *send_buffer = (char *)sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int local_rank, ppn; + MPI_Comm_rank(comm->local_comm, &local_rank); + MPI_Comm_size(comm->local_comm, &ppn); + + int n_nodes = num_procs / ppn; + + char *local_send_buffer = NULL; + char *local_recv_buffer = NULL; + + if (local_rank == 0) + { + local_send_buffer = (char *)malloc(ppn * num_procs * sendcount * send_size); + local_recv_buffer = (char *)malloc(ppn * num_procs * recvcount * recv_size); + } + else + { + local_send_buffer = (char *)malloc(sizeof(char)); + local_recv_buffer = (char *)malloc(sizeof(char)); + } + + int nInternalIters = estimate_gather_iters(send_buffer, sendcount * num_procs, sendtype, local_recv_buffer, + sendcount * num_procs, sendtype, 0, comm->local_comm); + printf("Number of iterations for gather: %d\n", nInternalIters); + + MPI_Barrier(MPI_COMM_WORLD); // should this be local_comm? + double t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + MPI_Gather(send_buffer, sendcount * num_procs, sendtype, local_recv_buffer, sendcount * num_procs, sendtype, + 0, comm->local_comm); + } + + double time = (MPI_Wtime() - t0) / nInternalIters; + double tFinalAllgather; + double tFinalAlltoall; + MPI_Allreduce(&time, &tFinalAllgather, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + if (local_rank == 0) + { + + // estimate iterations for alltoall on a subset of ranks + time = time_alltoall_subset(alltoall_func, local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->group_comm, 1); + MPI_Allreduce(MPI_IN_PLACE, &time, 1, MPI_DOUBLE, MPI_MAX, comm->group_comm); + if (time > 1) + nInternalIters = 1; + else + { + if (time > 1e-01) + time = time_alltoall_subset(alltoall_func, local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->group_comm, 2); + else if (time > 1e-02) + time = time_alltoall_subset(alltoall_func, local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->group_comm, 10); + else + time = time_alltoall_subset(alltoall_func, local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->group_comm, 100); + + nInternalIters = (1.0 / time) + 1; + if (nInternalIters < 1) + nInternalIters = 1; + } + + printf("nInteralIters for alltoall: %d\n", nInternalIters); + MPI_Barrier(comm->group_comm); + t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + alltoall_func(local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->group_comm); + } + + time = (MPI_Wtime() - t0) / nInternalIters; + MPI_Allreduce(&time, &tFinalAlltoall, 1, MPI_DOUBLE, MPI_MAX, comm->group_comm); + } + + nInternalIters = estimate_scatter_iters(local_send_buffer, recvcount * num_procs, recvtype, recv_buffer, + recvcount * num_procs, recvtype, 0, comm->local_comm); + printf("nInternalIters for scatter: %d\n", nInternalIters); + MPI_Barrier(MPI_COMM_WORLD); + t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + MPI_Scatter(local_send_buffer, recvcount * num_procs, recvtype, recv_buffer, recvcount * num_procs, recvtype, + 0, comm->local_comm); + } + + time = (MPI_Wtime() - t0) / nInternalIters; + double tFinalScatter; + MPI_Allreduce(&time, &tFinalScatter, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + free(local_send_buffer); + free(local_recv_buffer); + + MPI_Comm_rank(comm->global_comm, &rank); + if (rank == 0) + { + printf("%s, %d procs per leader, allgather: %e, alltoall: %e, scatter: %e\n", name, procsPerLeader, + tFinalAllgather, tFinalAlltoall, tFinalScatter); + } +} + +void internal_multileader_timing(A alltoallFunc, const void *sendbuf, const int sendcount, + MPI_Datatype sendtype, void *recvbuf, const in recvcount, + MPI_Datatype recvtype, MPIX_Comm *comm, const char *name, + int procsPerLeader) +{ + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int num_leaders_per_node = 4; + int procs_per_node; + MPI_Comm_size(comm->local_comm, &procs_per_node); + int procs_per_leader = procs_per_node / num_leaders_per_node; + if (procs_per_node = < num_leaders_per_node) + { + num_leaders_per_node = procs_per_node; + procs_per_leader = 1; + } + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char* recv_buffer = (char*) recvbuf; + char* send_buffer = (char*) sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + if (comm->leader_comm == MPI_COMM_NULL) + MPIX_Comm_leader_init(comm, procs_per_leader); + + int local_rank, ppn; + MPI_Comm_rank(comm->leader_comm, &local_rank); + MPI_Comm_size(comm->leader_comm, &ppn); + + int n_nodes = num_procs / ppn; + + char* local_send_buffer = NULL; + char* local_recv_buffer = NULL; + + if (local_rank == 0) + { + local_send_buffer = (char*) malloc(ppn*num_procs*sendcount*send_size); + local_recv_buffer = (char*) malloc(ppn*num_procs*recvcount*recv_size); + } + else + { + local_send_buffer = (char*) malloc(sizeof(char)); + local_recv_buffer = (char*) malloc(sizeof(char)); + } + + int nInternalIters = estimate_gather_iters(send_buffer, sendcount*num_procs, sendtype, local_recv_buffer, + sendcount*num_procs, sendtype, 0, comm->leader_comm); + MPI_Barrier(MPI_COMM_WORLD); + double t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + MPI_Gather(send_buffer, sendcount*num_procs, sendtype, local_recv_buffer, sendcount*num_procs, sendtype, + 0, comm->leader_comm); + } + double tfinal = (MPI_Wtime() - t0) / nInternalIters; + double tFinalGather; + MPI_Allreduce(tfinal, &tFinalGather, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + double tFinalAlltoall; + if (local_rank == 0) + { + time = time_alltoall_subset(alltoall_func, local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->leader_group_comm, 1); + if (time > 1) + nInternalIters = 1; + else + { + if (time > 1e-01) + time = time_alltoall_subset(alltoall_func, local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->leader_group_comm, 2); + else if (time > 1e-02) + time = time_alltoall_subset(alltoall_func, local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->leader_group_comm, 10); + else + time = time_alltoall_sbuset(alltoall_func, local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->leader_group_comm, 100); + + nInternalIters = (1.0 / time) + 1; + if (n_iters < 1) n_iters = 1; + } + + MPI_Barrier(comm->leader_group_comm); + t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + alltoall_func(local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->leader_group_comm); + } + + tfinal = (MPI_Wtime() - t0) / nInternalIters; + double tFinalAlltoall; + MPI_Allreduce(&tfinal, &tFinalAlltoall, 0, MPI_DOUBLE, MPI_MAX, comm->leader_group_comm); + } + + nInternalIters = estimate_scatter_iters(local_send_buffer, recvcount * num_procs, recvtype, recv_buffer, + recvcount * num_procs, recvtype, 0, comm->leader_comm); + MPI_Barrier(MPI_COMM_WORLD); + t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + MPI_Scatter(local_send_buffer, recvcount * num_procs, recvtype, recv_buffer, recvcount * num_procs, recvtype, 0, comm->leader_comm); + } + tfinal = (MPI_Wtime() - t0) / nInternalIters; + double tFinalScatter; + MPI_Allreduce(&tfinal, &tFinalScatter, 0, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + if (rank == 0) + printf("%s, %d procs per leader, gather: %e, leader comm alltoall: %e, scatter: %e\n", name, procsPerLeader, + tFinalGather, tFinalAlltoall, tFinalScatter); +} + +void internal_node_aware_timing(A alltoallFunc, const void *sendbuf, const int sendcount, + MPI_Datatype sendtype, void *recvbuf, const int recvcount, + MPI_Datatype recvtype, MPIX_Comm *comm, const char* name, + int procsPerLeader) +{ + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &size); + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int local_rank, ppn; + MPI_Comm_rank(comm->local_comm, &local_rank); + MPI_Comm_size(comm->local-comm, &ppn); + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char* recv_buffer = (char*) recvbuf; + char* send_buffer = (char*) sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + int n_nodes = num_procs / ppn; + char* tmpbuf = (char*) mallc(num_procs * sendcount * send_size); + int nInternalIters = estimate_alltoall_iters(alltoallFunc, sendbuf, ppn*sendcount, sendtype, tempbuf, ppn*recvcount, recvtype, comm->group_comm); + MPI_Barrier(MPI_COMM_WORLD); + double t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + alltoallFunc(sendbuf, ppn*sendcount, sendtype, tempbuf, ppn*recvcount, recvtype, comm->group_comm); + } + + double tfinal = (MPI_Wtime() - t0) / nInternalIters; + double tfinalGroupAlltoall; + MPI_Allreduce(&tfinal, &tfinalGroupAlltoall, 0, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + nInternalIters = estimate_alltoall_iters(alltoallFunc, recvbuf, n_nodes*recvcount, recvtype, tmpbuf, n_nodes*recvcount, recvtype, comm->local_comm); + MPI_Barrier(MPI_COMM_WORLD); + t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + alltoallFunc(recvbuf, n_nodes*recvcount, recvtype, tmpbuf, n_nodes*recvcount, recvtype, comm->local_comm); + } + + tfinal = (MPI_Wtime() - t0) / nInternalIters; + double tfinalLocalAlltoall; + MPI_Allreduce(&tfinal, &tfinalLocalAlltoall, 0, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + free(tmpbuf); + if (rank == 0) + printf("%s, %d procs per leader, leader group comm alltoall: %e, leader comm alltoall: %e\n", name, procsPerLeader, + tfinalGroupAlltoall, tfinalLocalAlltoall); +} + +// template +void internal_locality_aware_timing(A alltoallFunc, const void *sendbuf, const int sendcount, + MPI_Datatype sendtype, void *recvbuf, const int recvcount, + MPI_Datatype recvtype, MPIX_Comm *comm, const char *name, + int procsPerLeader) +{ + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int num_leaders_per_node = 4; + int procs_per_node; + MPI_Comm_size(comm->local_comm, &procs_per_node); + int procs_per_leader = procs_per_node / num_leaders_per_node; + if (procs_per_node < num_leaders_per_node) + { + num_leaders_per_node = procs_per_node; + procs_per_leader = 1; + } + + if (comm->leader_comm == MPI_COMM_NULL) + MPIX_Comm_leader_init(comm, procs_per_leader); + + int ppn; + MPI_Comm_size(comm->leader_comm, &ppn); + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char *recv_buffer = (char *)recvbuf; + char *send_buffer = (char *)sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + int n_nodes = num_procs / ppn; + + char *tmpbuf = (char *)malloc(num_procs * sendcount * send_size); + + int nInternalIters = estimate_alltoall_iters(alltoallFunc, sendbuf, ppn * sendcount, sendtype, tmpbuf, + ppn * recvcount, recvtype, comm->leader_group_comm); + MPI_Barrier(MPI_COMM_WORLD); + double t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + alltoallFunc(sendbuf, ppn * sendcount, sendtype, tmpbuf, ppn * recvcount, recvtype, comm->leader_group_comm); + } + + double time = (MPI_Wtime() - t0) / nInternalIters; + double tFinalLeaderGroupAlltoall; + MPI_Allreduce(&time, &tFinalLeaderGroupAlltoall, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + nInternalIters = estimate_alltoall_iters(alltoallFunc, recvbuf, n_nodes * recvcount, recvtype, tmpbuf, n_nodes * recvcount, recvtype, comm->leader_comm); + MPI_Barrier(MPI_COMM_WORLD); + t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + alltoallFunc(recvbuf, n_nodes * recvcount, recvtype, tmpbuf, n_nodes * recvcount, recvtype, comm->leader_comm); + } + + time = (MPI_Wtime() - t0) / nInternalIters; + double tFinalLeaderAlltoall; + MPI_Allreduce(&time, &tFinalLeaderAlltoall, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + if (rank == 0) + printf("%s, %d procs per leader, leader group comm alltoall: %e, leader comm alltoall: %e\n", name, procsPerLeader, + tFinalLeaderGroupAlltoall, tFinalLeaderAlltoall); +} + +// template +void internal_multileader_locality_aware_timing(A alltoallFunc, const void *sendbuf, const int sendcount, + MPI_Datatype sendtype, void *recvbuf, const int recvcount, + MPI_Datatype recvtype, MPIX_Comm *comm, const char *name, + int procsPerLeader) +{ + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int local_rank, ppn; + MPI_Comm_rank(comm->local_comm, &local_rank); + MPI_Comm_size(comm->local_comm, &ppn); + + if (comm->leader_comm == MPI_COMM_NULL) + { + int num_leaders_per_node = 4; + if (ppn < num_leaders_per_node) + num_leaders_per_node = ppn; + MPIX_Comm_leader_init(comm, ppn / num_leaders_per_node); + } + + int procs_per_leader, leader_rank; + MPI_Comm_rank(comm->leader_comm, &leader_rank); + MPI_Comm_size(comm->leader_comm, &procs_per_leader); + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char *recv_buffer = (char *)recvbuf; + char *send_buffer = (char *)sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + int n_nodes = num_procs / ppn; + int n_leaders = num_procs / procs_per_leader; + + int leaders_per_node; + MPI_Comm_size(comm->leader_local_comm, &leaders_per_node); + + char *local_send_buffer = NULL; + char *local_recv_buffer = NULL; + if (leader_rank == 0) + { + local_send_buffer = (char *)malloc(procs_per_leader * num_procs * sendcount * send_size); + local_recv_buffer = (char *)malloc(procs_per_leader * num_procs * recvcount * recv_size); + } + else + { + local_send_buffer = (char *)malloc(sizeof(char)); + local_recv_buffer = (char *)malloc(sizeof(char)); + } + + // 1. Local gather + int nInternalIters = estimate_gather_iters(send_buffer, sendcount * num_procs, sendtype, + local_recv_buffer, sendcount * num_procs, sendtype, + 0, comm->leader_comm); + MPI_Barrier(MPI_COMM_WORLD); + double t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + MPI_Gather(send_buffer, sendcount * num_procs, sendtype, local_recv_buffer, sendcount * num_procs, sendtype, + 0, comm->leader_comm); + } + double time = (MPI_Wtime() - t0) / nInternalIters; + double tFinalGather; + MPI_Allreduce(&time, &tFinalGather, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + double tFinalGroupAlltoall; + double tFinalLeaderLocalAlltoall; + if (leader_rank == 0) + { + time = time_alltoall_subset(alltoallFunc, local_send_buffer, ppn * procs_per_leader * sendcount, sendtype, + local_recv_buffer, ppn * procs_per_leader * recvcount, recvtype, comm->group_comm, 1); + MPI_Allreduce(MPI_IN_PLACE, &time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + if (time > 1) + nInternalIters = 1; + else + { + if (time > 1e-01) + time = time_alltoall_subset(alltoallFunc, local_send_buffer, ppn * procs_per_leader * sendcount, sendtype, + local_recv_buffer, ppn * procs_per_leader * recvcount, recvtype, + comm->group_comm, 2); + else if (time > 1e-02) + time = time_alltoall_subset(alltoallFunc, local_send_buffer, ppn * procs_per_leader * sendcount, sendtype, + local_recv_buffer, ppn * procs_per_leader * recvcount, recvtype, + comm->group_comm, 10); + else + time = time_alltoall_subset(alltoallFunc, local_send_buffer, ppn * procs_per_leader * sendcount, sendtype, + local_recv_buffer, ppn * procs_per_leader * recvcount, recvtype, + comm->group_comm, 100); + MPI_Allreduce(MPI_IN_PLACE, &time, 1, MPI_DOUBLE, MPI_MAX, comm->group_comm); + + nInternalIters = (1.0 / time) + 1; + if (nInternalIters < 1) + nInternalIters = 1; + } + + MPI_Barrier(comm->group_comm); + t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + alltoallFunc(local_send_buffer, ppn * procs_per_leader * sendcount, sendtype, + local_recv_buffer, ppn * procs_per_leader * recvcount, recvtype, comm->group_comm); + } + + time = (MPI_Wtime() - t0) / nInternalIters; + + MPI_Allreduce(&time, &tFinalGroupAlltoall, 1, MPI_DOUBLE, MPI_MAX, comm->group_comm); + + time = time_alltoall_subset(alltoallFunc, local_send_buffer, n_nodes * procs_per_leader * procs_per_leader * sendcount, + sendtype, local_recv_buffer, n_nodes * procs_per_leader * procs_per_leader * recvcount, + recvtype, comm->leader_local_comm, 1); + MPI_Allreduce(MPI_IN_PLACE, &time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + if (time > 1) + nInternalIters = 1; + else + { + if (time > 1e-01) + time = time_alltoall_subset(alltoallFunc, local_send_buffer, n_nodes * procs_per_leader * procs_per_leader * sendcount, + sendtype, local_recv_buffer, n_nodes * procs_per_leader * procs_per_leader * recvcount, + recvtype, comm->leader_local_comm, 2); + else if (time > 1e-02) + time = time_alltoall_subset(alltoallFunc, local_send_buffer, n_nodes * procs_per_leader * procs_per_leader * sendcount, + sendtype, local_recv_buffer, n_nodes * procs_per_leader * procs_per_leader * recvcount, + recvtype, comm->leader_local_comm, 10); + else + time = time_alltoall_subset(alltoallFunc, local_send_buffer, n_nodes * procs_per_leader * procs_per_leader * recvcount, + sendtype, local_recv_buffer, n_nodes * procs_per_leader * procs_per_leader * recvcount, + recvtype, comm->leader_local_comm, 100); + nInternalIters = (1.0 / time) + 1; + if (nInternalIters < 1) + nInternalIters = 1; + } + + MPI_Barrier(comm->leader_local_comm); + t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + alltoallFunc(local_send_buffer, n_nodes * procs_per_leader * procs_per_leader * sendcount, sendtype, + local_recv_buffer, n_nodes * procs_per_leader * procs_per_leader * recvcount, recvtype, + comm->leader_local_comm); + } + + time = (MPI_Wtime() - t0) / nInternalIters; + MPI_Allreduce(&time, &tFinalLeaderLocalAlltoall, 1, MPI_DOUBLE, MPI_MAX, comm->leader_local_comm); + } + + nInternalIters = estimate_scatter_iters(local_send_buffer, recvcount * num_procs, recvtype, recv_buffer, + recvcount * num_procs, recvtype, 0, comm->leader_comm); + MPI_Barrier(MPI_COMM_WORLD); + t0 = MPI_Wtime(); + for (int i = 0; i < nInternalIters; i++) + { + MPI_Scatter(local_send_buffer, recvcount * num_procs, recvtype, recv_buffer, recvcount * num_procs, recvtype, + 0, comm->leader_comm); + } + time = (MPI_Wtime() - t0) / nInternalIters; + double tFinalScatter; + MPI_Allreduce(&time, &tFinalScatter, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + if (rank == 0) + { + printf("%s, %d procs per leader, all gather: %e, group alltoall: %e, leader local alltoall: %e, scatter: %e\n", + name, procsPerLeader, tFinalGather, tFinalGroupAlltoall, tFinalLeaderLocalAlltoall, tFinalScatter); + } +} + +// template +void no_op_internal_timing(A alltoallFunc, const void *sendbuff, const int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int recvcount, MPI_Datatype recvtype, MPIX_Comm *comm, + const char *name, int procsPerLeader) +{ + // no op +} + +template +void print_alltoalls(int max_p, const T *sendbuf, + MPI_Datatype sendtype, T *recvbuf, MPI_Datatype recvtype, + MPIX_Comm *comm, T *recvbuf_std) +{ + int rank; + MPI_Comm_rank(comm->global_comm, &rank); + + int local_rank, ppn; + MPI_Comm_rank(comm->local_comm, &local_rank); + MPI_Comm_size(comm->local_comm, &ppn); + + double time; + + using F = int (*)(const void *, int, MPI_Datatype, void *, int, MPI_Datatype, _MPIX_Comm *); + using I = void (*)(A, const void *, const int, MPI_Datatype, void *, const int, MPI_Datatype, _MPIX_Comm *, const char *, int); + std::vector alltoall_funcs = {alltoall_pairwise, alltoall_nonblocking, alltoall_hierarchical, alltoall_node_aware, alltoall_hierarchical_nb, alltoall_node_aware_nb}; + std::vector names = {"Pairwise", "NonBlocking", "Pairwise Hierarchical", "Pairwise Node Aware", "Nonblocking Hierarchical", "Nonblocking Node Aware"}; + std::vector timingFuncs = {no_op_internal_timing, no_op_internal_timing, internal_hierarchical_timing, internal_node_aware_timing, internal_hierarchical_timing, internal_node_aware_timing}; + std::vector internalAlltoallFuncs = {pairwise_helper, nonblocking_helper, pairwise_helper, pairwise_helper, nonblocking_helper, nonblocking_helper}; + + std::vector multileader_funcs = {alltoall_multileader, alltoall_locality_aware, alltoall_multileader_locality, alltoall_multileader_nb, alltoall_locality_aware_nb, alltoall_multileader_locality_nb}; + std::vector multileader_names = {"Pairwise Multileader", "Pairwise Locality Aware", "Pairwise Multileader Locality", "Nonblocking Multileader", "Nonblocking Locality Aware", "Nonblocking Multileader Locality"}; + std::vector multileaderTimingFuncs = {internal_multileader_timing, internal_locality_aware_timing, internal_multileader_locality_aware_timing, internal_multileader_timing, internal_locality_aware_timing, internal_multileader_locality_aware_timing}; + std::vector multileaderInternalAlltoallFuncs = {pairwise_helper, pairwise_helper, pairwise_helper, nonblocking_helper, nonblocking_helper, nonblocking_helper}; + for (int i = 0; i < max_p; i++) + { + int s = pow(2, i); + + if (rank == 0) + printf("Size %d\n", s); + + // Standard PMPI Alltoall (system MPI) + PMPI_Alltoall(sendbuf, s, sendtype, recvbuf, s, recvtype, comm->global_comm); + std::memcpy(recvbuf_std, recvbuf, s * sizeof(T)); + time = test_alltoall(PMPI_Alltoall, sendbuf, s, sendtype, + recvbuf, s, recvtype, comm->global_comm); + if (rank == 0) + printf("PMPI: %e\n", time); + + // MPI Advance Alltoall Functions (not multileader) + for (int idx = 0; idx < alltoall_funcs.size(); idx++) + { + alltoall_funcs[idx](sendbuf, s, sendtype, recvbuf, s, recvtype, comm); + for (int j = 0; j < s; j++) + if (fabs(recvbuf_std[j] - recvbuf[j]) > 1e-06) + { + printf("DIFF RESULTS %d vs %d\n", recvbuf_std[j], recvbuf[j]); + MPI_Abort(comm->global_comm, -1); + } + time = test_alltoall(alltoall_funcs[idx], sendbuf, s, sendtype, + recvbuf, s, recvtype, comm); + if (rank == 0) + printf("%s: %e\n", names[idx], time); + timingFuncs[idx](internalAlltoallFuncs[idx], sendbuf, s, sendtype, recvbuf, s, recvtype, comm, names[idx], 1); + } + + // MPI Advance Multileader Alltoall Functions + // std::vector procs_per_leader_list = {4, 8, 16}; + // for (int ctr = 0; ctr < procs_per_leader_list.size(); ctr++) + //{ + // int n_procs = procs_per_leader_list[ctr]; + // if (ppn < n_procs) + // break; + // MPIX_Comm_leader_init(comm, n_procs); + // + // for (int idx = 0; idx < multileader_funcs.size(); idx++) + //{ + // multileader_funcs[idx](sendbuf, s, sendtype, recvbuf, s, recvtype, comm); + // for (int j = 0; j < s; j++) + // if (fabs(recvbuf_std[j] - recvbuf[j]) > 1e-06) + // { + // printf("DIFF RESULTS %d vs %d\n", recvbuf_std[j], recvbuf[j]); + // MPI_Abort(comm->global_comm, -1); + // } + // time = test_alltoall(multileader_funcs[idx], sendbuf, s, sendtype, + // recvbuf, s, recvtype, comm); + // if (rank == 0) printf("%s, %d procs per leader: %e\n", multileader_names[idx], n_procs, time); + // multileaderTimingFuncs[idx](multileaderInternalAlltoallFuncs[idx], sendbuf, s, sendtype, recvbuf, s, recvtype, comm, multileader_names[idx], n_procs); + // } + + // MPIX_Comm_leader_free(comm); + // } + } +} + +int main(int argc, char *argv[]) +{ + MPI_Init(&argc, &argv); + + int max_p = 15; + int max_size = pow(2, max_p); + + MPIX_Comm *xcomm; + MPIX_Comm_init(&xcomm, MPI_COMM_WORLD); + + int rank, num_procs; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &num_procs); + + MPIX_Comm_topo_init(xcomm); + + int local_rank, ppn; + MPI_Comm_rank(xcomm->local_comm, &local_rank); + MPI_Comm_size(xcomm->local_comm, &ppn); + + // To test a different number of leaders, change here: + // TODO : currently need num_leaders_per_node to evenly divide ppn + std::vector sendbuf(max_size * num_procs); + std::vector recvbuf(max_size * num_procs); + std::vector recvbuf_std(max_size * num_procs); + + for (int j = 0; j < num_procs; j++) + { + for (int k = 0; k < max_size; k++) + { + sendbuf[j * max_size + k] = rank * 10000 + j * 100 + k; + } + } + + print_alltoalls(max_p, sendbuf.data(), MPI_FLOAT, recvbuf.data(), MPI_FLOAT, + xcomm, recvbuf_std.data()); +} diff --git a/plots/dane/locality_n16.pdf b/plots/dane/locality_n16.pdf new file mode 100644 index 000000000..f4ad1a78c Binary files /dev/null and b/plots/dane/locality_n16.pdf differ diff --git a/plots/dane/locality_n2.pdf b/plots/dane/locality_n2.pdf new file mode 100644 index 000000000..384c60372 Binary files /dev/null and b/plots/dane/locality_n2.pdf differ diff --git a/plots/dane/locality_n32.pdf b/plots/dane/locality_n32.pdf new file mode 100644 index 000000000..73cc7e06a Binary files /dev/null and b/plots/dane/locality_n32.pdf differ diff --git a/plots/dane/locality_n4.pdf b/plots/dane/locality_n4.pdf new file mode 100644 index 000000000..0ed797cf7 Binary files /dev/null and b/plots/dane/locality_n4.pdf differ diff --git a/plots/dane/locality_n8.pdf b/plots/dane/locality_n8.pdf new file mode 100644 index 000000000..0068540e4 Binary files /dev/null and b/plots/dane/locality_n8.pdf differ diff --git a/plots/dane/multileader_loc_n16.pdf b/plots/dane/multileader_loc_n16.pdf new file mode 100644 index 000000000..43f3726c9 Binary files /dev/null and b/plots/dane/multileader_loc_n16.pdf differ diff --git a/plots/dane/multileader_loc_n2.pdf b/plots/dane/multileader_loc_n2.pdf new file mode 100644 index 000000000..ce3bc1010 Binary files /dev/null and b/plots/dane/multileader_loc_n2.pdf differ diff --git a/plots/dane/multileader_loc_n32.pdf b/plots/dane/multileader_loc_n32.pdf new file mode 100644 index 000000000..48b9cf3b7 Binary files /dev/null and b/plots/dane/multileader_loc_n32.pdf differ diff --git a/plots/dane/multileader_loc_n4.pdf b/plots/dane/multileader_loc_n4.pdf new file mode 100644 index 000000000..e7b56198d Binary files /dev/null and b/plots/dane/multileader_loc_n4.pdf differ diff --git a/plots/dane/multileader_loc_n8.pdf b/plots/dane/multileader_loc_n8.pdf new file mode 100644 index 000000000..ca49c6520 Binary files /dev/null and b/plots/dane/multileader_loc_n8.pdf differ diff --git a/plots/dane/multileader_n16.pdf b/plots/dane/multileader_n16.pdf new file mode 100644 index 000000000..f22647815 Binary files /dev/null and b/plots/dane/multileader_n16.pdf differ diff --git a/plots/dane/multileader_n2.pdf b/plots/dane/multileader_n2.pdf new file mode 100644 index 000000000..dc859e19e Binary files /dev/null and b/plots/dane/multileader_n2.pdf differ diff --git a/plots/dane/multileader_n32.pdf b/plots/dane/multileader_n32.pdf new file mode 100644 index 000000000..9f2d6b1c4 Binary files /dev/null and b/plots/dane/multileader_n32.pdf differ diff --git a/plots/dane/multileader_n4.pdf b/plots/dane/multileader_n4.pdf new file mode 100644 index 000000000..50ea56c64 Binary files /dev/null and b/plots/dane/multileader_n4.pdf differ diff --git a/plots/dane/multileader_n8.pdf b/plots/dane/multileader_n8.pdf new file mode 100644 index 000000000..eb8ba5e62 Binary files /dev/null and b/plots/dane/multileader_n8.pdf differ diff --git a/plots/dane/scaling_size1.pdf b/plots/dane/scaling_size1.pdf new file mode 100644 index 000000000..21cd4b504 Binary files /dev/null and b/plots/dane/scaling_size1.pdf differ diff --git a/plots/dane/scaling_size1024.pdf b/plots/dane/scaling_size1024.pdf new file mode 100644 index 000000000..ee479f824 Binary files /dev/null and b/plots/dane/scaling_size1024.pdf differ diff --git a/plots/dane/sizes_n16.pdf b/plots/dane/sizes_n16.pdf new file mode 100644 index 000000000..98794267c Binary files /dev/null and b/plots/dane/sizes_n16.pdf differ diff --git a/plots/dane/sizes_n2.pdf b/plots/dane/sizes_n2.pdf new file mode 100644 index 000000000..ca8c0895e Binary files /dev/null and b/plots/dane/sizes_n2.pdf differ diff --git a/plots/dane/sizes_n32.pdf b/plots/dane/sizes_n32.pdf new file mode 100644 index 000000000..998155a7c Binary files /dev/null and b/plots/dane/sizes_n32.pdf differ diff --git a/plots/dane/sizes_n4.pdf b/plots/dane/sizes_n4.pdf new file mode 100644 index 000000000..08a43d896 Binary files /dev/null and b/plots/dane/sizes_n4.pdf differ diff --git a/plots/dane/sizes_n8.pdf b/plots/dane/sizes_n8.pdf new file mode 100644 index 000000000..282fd534b Binary files /dev/null and b/plots/dane/sizes_n8.pdf differ diff --git a/plots/plot.py b/plots/plot.py new file mode 100644 index 000000000..2bed639e4 --- /dev/null +++ b/plots/plot.py @@ -0,0 +1,289 @@ +import numpy as np +import glob + +computer = "dane" +leaders = [4, 10, 20] + +import pyfancyplot as plt +import matplotlib +matplotlib.use("qtagg") + +class multileader: + ml = "" + loc = "" + ml_loc = "" + n_leaders = 0 + + def __init__(self, n_leaders): + self.n_leaders = n_leaders + self.ml = list() + self.loc = list() + self.ml_loc = list() + + def add_size(self): + self.ml.append(np.inf) + self.loc.append(np.inf) + self.ml_loc.append(np.inf) + + def add_time(self, name, idx, time): + if "Multileader Locality" in name: + if (time < self.ml_loc[idx]): + self.ml_loc[idx] = time + elif "Multileader" in name: + if (time < self.ml[idx]): + self.ml[idx] = time + elif "Locality" in name: + if (time < self.loc[idx]): + self.loc[idx] = time + + +class implementation: + std = "" + hierarchical = "" + node_aware = "" + leader = "" + + def __init__(self): + self.std = list() + self.hierarchical = list() + self.node_aware = list() + self.leader = list() + for l in leaders: + self.leader.append(multileader(l)) + + def add_size(self): + self.std.append(np.inf) + self.hierarchical.append(np.inf) + self.node_aware.append(np.inf) + for l in self.leader: + l.add_size() + + def add_time(self, name, idx, time): + if "Hierarchical" in name: + if (time < self.hierarchical[idx]): + self.hierarchical[idx] = time + elif "Node Aware" in name: + if (time < self.node_aware[idx]): + self.node_aware[idx] = time + elif "leaders" in name: + n_leaders = (int)(name.rsplit(' ')[-2]) + leader_idx = leaders.index(n_leaders) + self.leader[leader_idx].add_time(name, idx, time) + else: + if (time < self.std[idx]): + self.std[idx] = time + + + +class alltoalls: + pmpi = "" + pairwise = "" + nonblocking = "" + sizes = "" + + def __init__(self): + self.sizes = list() + self.pmpi = list() + self.pairwise = implementation() + self.nonblocking = implementation() + + def add_size(self, size): + if (self.sizes.count(size)): + return + self.sizes.append(size) + self.pmpi.append(np.inf) + self.pairwise.add_size() + self.nonblocking.add_size() + + def add_time(self, name, size, time): + idx = self.sizes.index(size) + if "PMPI" in name: + if (time < self.pmpi[idx]): + self.pmpi[idx] = time + elif "Pairwise" in name: + self.pairwise.add_time(name, idx, time) + elif "Nonblocking" in name or "NonBlocking" in line: + self.nonblocking.add_time(name, idx, time) + + + + +nodes = [2, 4, 8, 16, 32] +timings = [] +size = 0 +for n in nodes: + timings.append(alltoalls()) + for f in glob.glob("../runscripts/%s_results/alltoall_N%d*.out"%(computer, n)): + file = open(f) + + for line in file: + if "Size" in line: + size = (int)((line.rsplit('\n')[0]).rsplit(' ')[1]) + timings[-1].add_size(size) + elif "PMPI" in line or "Pairwise" in line or "Nonblocking" in line or "NonBlocking" in line: + name = line.rsplit(':')[0] + time = (float)((line.rsplit('\n')[0]).rsplit(' ')[-1]) + timings[-1].add_time(name, size, time) + file.close() + + +print(timings[-1].nonblocking.std) +print(timings[-1].pairwise.leader[-1].ml_loc) + + +for i in range(len(nodes)): + plt.add_luke_options() + plt.line_plot(timings[i].pmpi, timings[i].sizes, color='black', label='PMPI') + + plt.line_plot(timings[i].pairwise.std, timings[i].sizes, color='red', label='Pairwise') + plt.line_plot(timings[i].nonblocking.std, timings[i].sizes, color='red', tickmark='--', label="Nonblocking") + + plt.line_plot(timings[i].pairwise.hierarchical, timings[i].sizes, color='blue', label='Hierarchical') + plt.line_plot(timings[i].nonblocking.hierarchical, timings[i].sizes, color='blue', tickmark='--') + + plt.line_plot(timings[i].pairwise.node_aware, timings[i].sizes, color='green', label='Node-Aware') + plt.line_plot(timings[i].nonblocking.node_aware, timings[i].sizes, color='green', tickmark='--') + + plt.line_plot(timings[i].pairwise.leader[0].ml, timings[i].sizes, color='orange', label='Multileader') + plt.line_plot(timings[i].nonblocking.leader[0].ml, timings[i].sizes, color='orange', tickmark='--') + + plt.line_plot(timings[i].pairwise.leader[0].loc, timings[i].sizes, color='brown', label='Locality-Aware') + plt.line_plot(timings[i].nonblocking.leader[0].loc, timings[i].sizes, color='brown', tickmark='--') + + plt.line_plot(timings[i].pairwise.leader[0].ml_loc, timings[i].sizes, color='purple', label='Multileader + Locality') + plt.line_plot(timings[i].nonblocking.leader[0].ml_loc, timings[i].sizes, color='purple', tickmark='--') + + plt.add_anchored_legend() + plt.set_scale('log', 'log') + plt.add_labels("Msg Size", "Timing (seconds)") + plt.save_plot("%s/sizes_n%d.pdf"%(computer,nodes[i])) + plt.plt.clf() + + + plt.add_luke_options() + plt.line_plot(timings[i].pmpi, timings[i].sizes, color='black', label='PMPI') + + plt.line_plot(timings[i].pairwise.leader[0].ml, timings[i].sizes, color='purple', label="4 Leaders Per Node") + plt.line_plot(timings[i].nonblocking.leader[0].ml, timings[i].sizes, color='purple', tickmark='--') + + plt.line_plot(timings[i].pairwise.leader[1].ml, timings[i].sizes, color='purple', label="10 Leaders Per Node") + plt.line_plot(timings[i].nonblocking.leader[1].ml, timings[i].sizes, color='purple', tickmark='--') + + plt.line_plot(timings[i].pairwise.leader[2].ml, timings[i].sizes, color='purple', label="20 Leaders Per Node") + plt.line_plot(timings[i].nonblocking.leader[2].ml, timings[i].sizes, color='purple', tickmark='--') + + plt.add_anchored_legend() + plt.set_scale('log', 'log') + plt.add_labels("Msg Size", "Timing (seconds)") + plt.save_plot("%s/multileader_n%d.pdf"%(computer,nodes[i])) + plt.plt.clf() + + + plt.add_luke_options() + plt.line_plot(timings[i].pmpi, timings[i].sizes, color='black', label='PMPI') + + plt.line_plot(timings[i].pairwise.leader[0].loc, timings[i].sizes, color='purple', label="4 Leaders Per Node") + plt.line_plot(timings[i].nonblocking.leader[0].loc, timings[i].sizes, color='purple', tickmark='--') + + plt.line_plot(timings[i].pairwise.leader[1].loc, timings[i].sizes, color='purple', label="10 Leaders Per Node") + plt.line_plot(timings[i].nonblocking.leader[1].loc, timings[i].sizes, color='purple', tickmark='--') + + plt.line_plot(timings[i].pairwise.leader[2].loc, timings[i].sizes, color='purple', label="20 Leaders Per Node") + plt.line_plot(timings[i].nonblocking.leader[2].loc, timings[i].sizes, color='purple', tickmark='--') + + plt.add_anchored_legend() + plt.set_scale('log', 'log') + plt.add_labels("Msg Size", "Timing (seconds)") + plt.save_plot("%s/locality_n%d.pdf"%(computer,nodes[i])) + plt.plt.clf() + + + plt.add_luke_options() + plt.line_plot(timings[i].pmpi, timings[i].sizes, color='black', label='PMPI') + + plt.line_plot(timings[i].pairwise.leader[0].ml_loc, timings[i].sizes, color='red', label="4 Leaders Per Node") + plt.line_plot(timings[i].nonblocking.leader[0].ml_loc, timings[i].sizes, color='red', tickmark='--') + + plt.line_plot(timings[i].pairwise.leader[1].ml_loc, timings[i].sizes, color='purple', label="10 Leaders Per Node") + plt.line_plot(timings[i].nonblocking.leader[1].ml_loc, timings[i].sizes, color='purple', tickmark='--') + + plt.line_plot(timings[i].pairwise.leader[2].ml_loc, timings[i].sizes, color='green', label="20 Leaders Per Node") + plt.line_plot(timings[i].nonblocking.leader[2].ml_loc, timings[i].sizes, color='green', tickmark='--') + + plt.line_plot(timings[i].pairwise.node_aware, timings[i].sizes, color='blue', label="Node-Aware") + plt.line_plot(timings[i].nonblocking.node_aware, timings[i].sizes, color='blue', tickmark='--') + + plt.add_anchored_legend() + plt.set_scale('log', 'log') + plt.add_labels("Msg Size", "Timing (seconds)") + plt.save_plot("%s/multileader_loc_n%d.pdf"%(computer,nodes[i])) + plt.plt.clf() + + + + +## Plot Size 1 Across Nodes +plt.add_luke_options() +plt.line_plot([timings[i].pmpi[0] for i in range(len(timings))], nodes, color='black', label='PMPI') + +plt.line_plot([timings[i].pairwise.std[0] for i in range(len(timings))], nodes, color='red', label='Pairwise') +plt.line_plot([timings[i].nonblocking.std[0] for i in range(len(timings))], nodes, color='red', tickmark='--', label='Nonblocking') + +plt.line_plot([timings[i].pairwise.hierarchical[0] for i in range(len(timings))], nodes, color='blue', label='Hierarchical') +plt.line_plot([timings[i].nonblocking.hierarchical[0] for i in range(len(timings))], nodes, color='blue', tickmark='--') + +plt.line_plot([timings[i].pairwise.node_aware[0] for i in range(len(timings))], nodes, color='green', label='Node-Aware') +plt.line_plot([timings[i].nonblocking.node_aware[0] for i in range(len(timings))], nodes, color='green', tickmark='--') + +plt.line_plot([timings[i].pairwise.leader[-1].ml[0] for i in range(len(timings))], nodes, color='orange', label='Multileader') +plt.line_plot([timings[i].nonblocking.leader[-1].ml[0] for i in range(len(timings))], nodes, color='orange', tickmark='--') + +plt.line_plot([timings[i].pairwise.leader[-1].loc[0] for i in range(len(timings))], nodes, color='brown', label='Locality-Aware') +plt.line_plot([timings[i].nonblocking.leader[-1].loc[0] for i in range(len(timings))], nodes, color='brown', tickmark='--') + +plt.line_plot([timings[i].pairwise.leader[-1].ml_loc[0] for i in range(len(timings))], nodes, color='purple', label='Multileader + Locality') +plt.line_plot([timings[i].nonblocking.leader[-1].ml_loc[0] for i in range(len(timings))], nodes, color='purple', tickmark='--') + +plt.add_anchored_legend() +ax = plt.get_ax() +ax.set_xscale('log', base=2) +ax.set_yscale('log') +plt.add_labels("Nodes", "Timing (seconds)") +plt.save_plot("%s/scaling_size1.pdf"%computer) +plt.plt.clf() + + +## Plot Largest Size Across Nodes +size = timings[-1].sizes[-2] +idx = len(timings[-1].sizes) - 2 +print("Size:", timings[-1].sizes[idx]) + +plt.add_luke_options() +plt.line_plot([timings[i].pmpi[idx] for i in range(len(timings))], nodes, color='black', label='PMPI') + +plt.line_plot([timings[i].pairwise.std[idx] for i in range(len(timings))], nodes, color='red', label='Pairwise') +plt.line_plot([timings[i].nonblocking.std[idx] for i in range(len(timings))], nodes, color='red', tickmark='--', label='Nonblocking') + +plt.line_plot([timings[i].pairwise.hierarchical[idx] for i in range(len(timings))], nodes, color='blue', label='Hierarchical') +plt.line_plot([timings[i].nonblocking.hierarchical[idx] for i in range(len(timings))], nodes, color='blue', tickmark='--') + +plt.line_plot([timings[i].pairwise.node_aware[idx] for i in range(len(timings))], nodes, color='green', label='Node-Aware') +plt.line_plot([timings[i].nonblocking.node_aware[idx] for i in range(len(timings))], nodes, color='green', tickmark='--') + +plt.line_plot([timings[i].pairwise.leader[-1].ml[idx] for i in range(len(timings))], nodes, color='orange', label='Multileader') +plt.line_plot([timings[i].nonblocking.leader[-1].ml[idx] for i in range(len(timings))], nodes, color='orange', tickmark='--') + +plt.line_plot([timings[i].pairwise.leader[-1].loc[idx] for i in range(len(timings))], nodes, color='brown', label='Locality-Aware') +plt.line_plot([timings[i].nonblocking.leader[-1].loc[idx] for i in range(len(timings))], nodes, color='brown', tickmark='--') + +plt.line_plot([timings[i].pairwise.leader[-1].ml_loc[idx] for i in range(len(timings))], nodes, color='purple', label='Multileader + Locality') +plt.line_plot([timings[i].nonblocking.leader[-1].ml_loc[idx] for i in range(len(timings))], nodes, color='purple', tickmark='--') + +plt.add_anchored_legend() +ax = plt.get_ax() +ax.set_xscale('log', base=2) +ax.set_yscale('log') +plt.add_labels("Nodes", "Timing (seconds)") +plt.save_plot("%s/scaling_size%d.pdf"%(computer,size)) +plt.plt.clf() + diff --git a/plots/pyfancyplot.py b/plots/pyfancyplot.py new file mode 100644 index 000000000..bf5562b4c --- /dev/null +++ b/plots/pyfancyplot.py @@ -0,0 +1,562 @@ +import numpy as np +import matplotlib.pylab as plt +import pandas as pd +import seaborn as sns + +sns.set(style="whitegrid", palette="muted") +font_style = 'serif' +linewidth = 3 +xscale = 'linear' +yscale = 'linear' +palette_name = "tab20" +palette_n_colors = 7 +palette_desat = None +color_ctr = 0 +lgd = None +fig = None + +################################ +### Set Default Options +### Serif Font, Size 24 +### Initial standard color palette +################################ +def default_options(): + fontsize = 24 + fig_width_pt = 700.0 + inches_per_pt = 1.0/72.27 # Convert pt to inch + golden_mean = (np.sqrt(5)-1.0)/2.0 # Aesthetic ratio + fig_width = fig_width_pt*inches_per_pt # width in inches + fig_height = fig_width*golden_mean # height in inches + fig_size = [fig_width, fig_height] + params = {'backend': 'ps', + 'font.family': 'serif', + 'font.serif': 'cm', + 'font.sans-serif': 'arial', + 'axes.labelsize': fontsize, + 'font.size': fontsize, + 'axes.titlesize': fontsize, + 'legend.fontsize': fontsize-2, + 'xtick.labelsize': fontsize, + 'ytick.labelsize': fontsize, + 'text.usetex': True, + 'figure.figsize': fig_size, + 'lines.linewidth': 4, + 'hatch.linewidth': 3.0} + plt.rcParams.update(params) + + global font_style + global linewidth + global xscale + global yscale + global palette_name + global palette_n_colors + global palette_desat + global color_ctr + global ldg + global fig + + font_style = 'serif' + linewidth = 3 + xscale = 'linear' + yscale = 'linear' + palette_name = "tab20" + palette_n_colors = 7 + palette_desat = None + color_ctr = 0 + lgd = None + + fig = plt.figure(figsize=(fig_width, fig_height)); + plt.gcf() + +################################ +### The reason you're +### using this script +################################ +def add_luke_options(): + from matplotlib import rc + font = {'family' : 'serif', + 'size' : 22} + rc("font", **font) + rc("lines", linewidth=3) + + ax = plt.gca() + ax.xaxis.grid(False) + #plt.grid(True) + sns.despine(left=True, right=True) + +################################ +### Clear previously plotted data +### Reset all global variables +### Initialize default options +################################ +def clear(): + plt.clf() + plt.close('all') + default_options() + +# Automatically set default options +default_options() +add_luke_options() ## add these by default, too + + +################################ +### Set color palette +### Palette options: +### - Seaborn color palette name +### - list of colors +### - list of RGB values +################################ +def set_palette(palette = "tab20", n_colors = None, desat = None): + global palette_name + global palette_n_colors + global palette_desat + global color_ctr + + palette_name = palette + palette_n_colors = n_colors + palette_desat = desat + color_ctr = 0 + +################################ +### Returns Seaborn color palette +### If num colors is passed, will +### override palette_n_colors +### For other options, first +### call set_palette +################################ +def get_palette(num_colors = None): + global palette_name + global palette_n_colors + global palette_desat + if not num_colors is None: + palette_n_colors = num_colors + return sns.color_palette(palette_name, palette_n_colors, palette_desat) + +################################ +### Returns next color in +### Seaborn palette +################################ +def next_color(): + global palette_name + global palette_n_colors + global palette_desat + global color_ctr + + color_palette = sns.color_palette(palette_name, palette_n_colors, palette_desat) + color = color_palette[color_ctr]; + color_ctr = (color_ctr + 1) % len(color_palette) + return color + +################################ +### Set scale of x and y dimensions +### Either 'linear' or 'log' +################################ +def set_scale(xscale, yscale): + ax = plt.gca() + ax.set_xscale(xscale) + ax.set_yscale(yscale) + ax.set_autoscaley_on(False) + +def get_ax(): + return plt.gca() + +################################ +### Set figure size (by inches) +################################ +def set_figure_size(dim_inches): + global fig + fig.set_size_inches(dim_inches) + +################################ +### Set figure size (by dpi) +################################ +def set_figure_dpi(dpi): + fig.set_dpi(dpi) + +################################ +### Add standard matplotlib legend +################################ +def add_legend(ncol = 1, + loc = 'best', + frameon = False, + fontsize = 20, + **kargs): + + plt.legend(loc = loc, ncol = ncol, frameon = frameon, fontsize = + fontsize, **kargs) + +################################ +### Add multi column legend +### Anchored about plot by default +################################ +def add_anchored_legend(ncol = 2, + loc = "upper center", + anchor = (0., 1.10, 1.,.102), + frameon = False, + fontsize = 22, + **kargs): + + global lgd + lgd = plt.legend(loc = loc, ncol = (int)(ncol), bbox_to_anchor = anchor, + frameon = frameon, fontsize = fontsize, **kargs) + +################################ +### Adds legend for barplots +### By selecting rectangles +### Adds multi column legend +### Anchored about plot by default +################################ +def barplot_legend(labels, positions, ax, n_cols = 0, **kargs): + import matplotlib.patches as patches + objs = ax.findobj(match=patches.Rectangle) + legend_lines = list() + + for i in range(len(labels)): + idx = positions[i] + legend_lines.append(objs[idx]) + if n_cols <= 0: + n_cols = ((len(labels) - 1) / 2) + 1 + if (n_cols < 2): + n_cols = 2 + add_anchored_legend(handles=legend_lines, labels=labels, ncol=n_cols, + **kargs) + +################################ +### Add a title +################################ +def add_title(title): + plt.title(title) + +################################ +### Add labels for x and y dims +################################ +def add_labels(xlabel, + ylabel): + plt.xlabel(xlabel) + plt.ylabel(ylabel) + + + +################################ +### Set limits for x dimension +################################ +def set_xlim(xmin = None, + xmax = None): + ax = plt.gca() + if xmin is None: + xmin, _ = ax.get_xlim() + if xmax is None: + _, x1 = ax.get_xlim() + xmax = x1 + (x1 / 20.0) + ax.set_xlim((xmin, xmax)) + +################################ +### Set limits for y dimension +################################ +def set_ylim(ymin = None, + ymax = None): + ax = plt.gca() + ax.set_autoscaley_on(False) + if ymin is None: + y0, _ = ax.get_ylim() + ymin = y0 - (y0 / 50.0) + if ymax is None: + _, ymax = ax.get_ylim() + ax.set_ylim(ymin, ymax) + ax.axis('tight') + + + +################################ +### Sets x-tick labels +################################ +def set_xticklabels(xticklabels, + rotation = 'vertical', + fontsize = 16, + **kargs): + ax = plt.gca() + ax.set_xticklabels(xticklabels, rotation = rotation, + fontsize = fontsize, **kargs) + +################################ +### Sets x-ticks and corresponding labels +################################ +def set_xticks(xdata, xticklabels, rotation = 'horizontal', **kargs): + ax = plt.gca() + ax.set_xticks(xdata) + ax.set_xticklabels(xticklabels, rotation = rotation, **kargs) + + +################################ +### Sets y-tick labels +################################ +def set_yticklabels(yticklabels, + rotation = 'horizontal', + fontsize = 16, + **kargs): + ax = plt.gca() + ax.set_yticklabels(yticklabels, rotation = rotation, + fontsize = fontsize, **kargs) + +################################ +### Set y-ticks and corresponding labels +################################ +def set_yticks(ydata, + yticklabels, + rotation = 'horizontal', + **kargs): + ax = plt.gca() + ax.set_yticks(ydata) + ax.set_yticklabels(yticklabels, rotation = rotation, **kargs) + + + +################################ +### Standard line plot +################################ +def line_plot(y_data, + x_data = None, + tickmark = '-', + alpha = 1.0, + linewidth=3, + ax = plt, + color = None, + **kargs): + if x_data is None: + x_data = np.arange(0, len(y_data)) + if color is None: + color = next_color() + return ax.plot(x_data, y_data, tickmark, + color = color, clip_on = False, + alpha = alpha, linewidth = linewidth, **kargs) + + +def violin_plot(x_data, y_data, labels = None, add_legend = True, ax = None,**kargs): + if ax is None: + ax = plt.gca() + + if labels is None: + vplot = sns.barplot(x=x_data, y=y_data) + else: + pd_dict = dict() + pd_dict['x'] = x_data + for i in range(len(labels)): + pd_dict[labels[i]] = y_data[i] + df = pd.DataFrame(pd_dict) + df = df.melt(id_vars=['x'], var_name='measure', value_vars=labels, + value_name='time') + vplot = sns.violinplot(data=df, x='x', y='time', hue='measure', ax = + ax, palette = get_palette(), edgecolor='black', **kargs) + + return vplot + +################################ +### Standard scatter plot +################################ +def scatter_plot(x_data, + y_data, + marker = 'o', + color = None, + **kargs): + if color is None: + color = next_color() + return plt.scatter(x_data, y_data, c = color, edgecolors='none', + clip_on = False, marker = marker, **kargs) + +################################ +### Spy of Matrix +################################ +def spy(A, color = 'black', markersize = None): + plt.spy(A, rasterized=True, markersize=markersize) + +################################ +### Creates a standard barplot +################################ +def barplot(x_data, + y_data, + labels = None, + ax = None, + add_legend = True, + color = None, + **kargs): + if ax is None: + ax = plt.gca() + + bplot = "" + if labels is None: + if color is None: + color = next_color() + bplot = sns.barplot(x=x_data, y=y_data, color=color, ax = ax, + edgecolor='black', **kargs) + else: + pd_dict = dict() + pd_dict['x'] = x_data + for i in range(len(labels)): + pd_dict[labels[i]] = y_data[i] + df = pd.DataFrame(pd_dict) + df = df.melt(id_vars=['x'], var_name='measure', value_vars=labels, + value_name='time') + bplot = sns.barplot(data=df, x='x', y='time', hue='measure', ax = + ax, palette = get_palette(), edgecolor='black', **kargs) + if add_legend: + positions = [i * len(x_data) for i in range(len(labels))] + barplot_legend(labels, positions, ax) + + + return bplot + +################################ +### Creates a stacked barplot +################################ +def stacked_barplot(x_data, # simple list + y_data, #list of lists (each of len(x_data)) + labels, #list of labels corresponding to y_data + ax = None, + **kargs): + if ax is None: + ax = plt.gca() + bplots = list() + + new_y_data = list() + for i in range(len(y_data)): + new_y_data.append(list()) + for j in range(len(y_data[i])): + new_y_data[i].append(y_data[i][j]) + + for i in range(len(labels)): + for j in range(len(labels)-1, i, -1): + for k in range(0, len(new_y_data[i])): + new_y_data[i][k] += new_y_data[j][k] + + colors = get_palette(len(labels)) + plots = list() + for i in range(len(labels)): + pd_dict = dict() + pd_dict['x'] = x_data + pd_dict[labels[i]] = new_y_data[i] + df = pd.DataFrame(pd_dict) + df = df.melt(id_vars=['x'], var_name='measure', + value_vars=[labels[i]], + value_name='time') + bplots.append(sns.barplot(ax = ax, data=df, x='x', y='time', + color=colors[i], edgecolor='black', **kargs)) + positions = [i * len(x_data) for i in range(len(labels))] + barplot_legend(labels, positions, ax) + return bplots + +################################ +### Create a partially stacked barplot +### Stacking some bars, but not all +### All bars share the same x_data +################################ +def partially_stacked_barplot(x_data, # simple list + y_data, # list of data and lists e.g. + # [y0, y1, [y2, y3], y4] would leave + # y0, y1, and y4 as simple bars + # but would stack y3 on top of y2 + labels, # labels in same format as y_data + # e.g. [l0, l1, [l2, l3], l4] corresponds + # to example y_data + ax = None, + **kargs): + if ax is None: + ax = plt.gca() + bplots = list() + max_stack_size = 1 + num_bars = len(y_data) + num_colors = 0 + bar_num_stacked = list() + stacked_bars = list() + positions = list() + for i in range(num_bars): + if type(labels[i]) == type("string"): + bar_num_stacked.append(1) + positions.append(i) + else: + stack_size = len(y_data[i]) + bar_num_stacked.append(stack_size) + stacked_bars.append(i) + for j in range(stack_size): + positions.append(i) + if stack_size > max_stack_size: + max_stack_size = stack_size + num_colors += bar_num_stacked[-1] + indices = np.zeros((num_colors, ), dtype = 'int') + + colors = get_palette(num_colors) + iter_colors = list() + iter_ctrs = list() + iter_labels = list() + iter_y_data = list() + + ctr = 0 + for i in range(num_bars): + stack_size = bar_num_stacked[i] + if stack_size == 1: + iter_labels.append(labels[i]) + iter_y_data.append(y_data[i]) + else: + iter_labels.append(labels[i][-1]) + iter_y_data.append(y_data[i][0]) + for j in range(1, stack_size): + for k in range(0, len(iter_y_data[i])): + iter_y_data[i][k] += y_data[i][j][k] + iter_colors.append(colors[ctr + stack_size - 1]) + iter_ctrs.append(ctr + stack_size - 1) + ctr += stack_size + + # Bar plot for each stack + for i in range(max_stack_size): + set_palette(iter_colors, len(iter_colors)) + barplot(x_data, iter_y_data, iter_labels) + for idx in stacked_bars: + stack_size = bar_num_stacked[idx] + if stack_size <= i+1: + continue + iter_ctrs[idx] -= 1 + indices[iter_ctrs[idx]] = i+1 + iter_labels[idx] = labels[idx][stack_size-i-2] + for k in range(len(iter_y_data[idx])): + iter_y_data[idx][k] -= y_data[idx][stack_size-i-1][k] + iter_colors[idx] = colors[iter_ctrs[idx]] + + label_list = list() + for i in range(num_bars): + stack_size = bar_num_stacked[i] + if stack_size == 1: + label_list.append(labels[i]) + else: + for j in range(stack_size): + label_list.append(labels[i][j]) + for i in range(num_colors): + positions[i] = (indices[i]*num_bars*len(x_data)) + (positions[i]*len(x_data)) + barplot_legend(label_list, positions, ax) + + +################################ +### Save the plot to a file +### Clears all data after by default +################################ +def save_plot(filename, + clear_plot = True, + **kargs): + global lgd + + if lgd is None: + plt.savefig(filename, bbox_inches = "tight", + transparent=True, **kargs) + else: + plt.savefig(filename, bbox_extra_artists=(lgd,), bbox_inches = "tight", + transparent=True, **kargs) + + if clear_plot: + clear() + +################################ +### Display your plot +################################ +def display_plot(): + plt.show() + + + diff --git a/runscripts/04_28_25/alltoall_N16.2361050.err b/runscripts/04_28_25/alltoall_N16.2361050.err new file mode 100644 index 000000000..db36c9dc2 --- /dev/null +++ b/runscripts/04_28_25/alltoall_N16.2361050.err @@ -0,0 +1,393 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2361050.0 ON dane132 CANCELLED AT 2025-04-28T16:01:45 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane395: tasks 300-399: Killed +srun: error: dane1518: tasks 1500-1599: Killed +srun: error: dane984: tasks 1000-1099: Killed +srun: error: dane1508: tasks 1400-1499: Killed +srun: error: dane911: tasks 900-999: Killed +srun: error: dane1084: tasks 1100-1199: Killed +srun: error: dane474: tasks 500-599: Killed +srun: error: dane1340: tasks 1300-1399: Killed +srun: error: dane719: tasks 700-799: Killed +srun: error: dane881: tasks 800-899: Killed +srun: error: dane1087: tasks 1200-1299: Killed +srun: error: dane267: tasks 200-299: Killed +srun: error: dane208: tasks 100-199: Killed +srun: error: dane578: tasks 600-699: Killed +srun: error: dane132: tasks 0-99: Killed +srun: error: dane445: tasks 400-499: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fc709010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fc709010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2361050.1 ON dane132 CANCELLED AT 2025-04-28T16:06:04 *** +slurmstepd: error: *** STEP 2361050.1 ON dane132 CANCELLED AT 2025-04-28T16:06:04 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane395: tasks 300-399: Killed +srun: error: dane1518: tasks 1500-1599: Killed +srun: error: dane984: tasks 1000-1099: Killed +srun: error: dane1508: tasks 1400-1499: Killed +srun: error: dane911: tasks 900-999: Killed +srun: error: dane1084: tasks 1100-1199: Killed +srun: error: dane474: tasks 500-599: Killed +srun: error: dane1340: tasks 1300-1399: Killed +srun: error: dane719: tasks 700-799: Killed +srun: error: dane881: tasks 800-899: Killed +srun: error: dane1087: tasks 1200-1299: Killed +srun: error: dane267: tasks 200-299: Killed +srun: error: dane578: tasks 600-699: Killed +srun: error: dane208: tasks 100-199: Killed +srun: error: dane132: tasks 0-99: Killed +srun: error: dane445: tasks 400-499: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2361050.2 ON dane132 CANCELLED AT 2025-04-28T16:11:07 *** +slurmstepd: error: *** STEP 2361050.2 ON dane132 CANCELLED AT 2025-04-28T16:11:07 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane445: task 405: Killed +srun: error: dane395: tasks 300-399: Killed +srun: error: dane1518: tasks 1500-1599: Killed +srun: error: dane984: tasks 1000-1099: Killed +srun: error: dane1508: tasks 1400-1499: Killed +srun: error: dane911: tasks 900-999: Killed +srun: error: dane1084: tasks 1100-1199: Killed +srun: error: dane474: tasks 500-599: Killed +srun: error: dane1340: tasks 1300-1399: Killed +srun: error: dane719: tasks 700-799: Killed +srun: error: dane881: tasks 800-899: Killed +srun: error: dane1087: tasks 1200-1299: Killed +srun: error: dane578: tasks 600-699: Killed +srun: error: dane267: tasks 200-299: Killed +srun: error: dane132: tasks 0-99: Killed +srun: error: dane208: tasks 100-199: Killed +srun: error: dane445: tasks 400-404,406-499: Killed diff --git a/runscripts/04_28_25/alltoall_N16.2361050.out b/runscripts/04_28_25/alltoall_N16.2361050.out new file mode 100644 index 000000000..869f509fc --- /dev/null +++ b/runscripts/04_28_25/alltoall_N16.2361050.out @@ -0,0 +1,519 @@ +Test 1 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 3.904470e-04 +Pairwise: 1.015757e-02 +NonBlocking: 9.989815e-02 +Pairwise Hierarchical: 1.204318e-03 +Pairwise Multileader: 1.080417e-03 +Pairwise Node Aware: 4.439701e-04 +Pairwise Locality Aware: 7.614780e-04 +Pairwise Multileader Locality: 5.735216e-04 +Nonblocking Hierarchical: 9.488916e-04 +Nonblocking Multileader: 3.280074e-04 +Nonblocking Node Aware: 2.782877e-04 +Nonblocking Locality Aware: 3.479868e-04 +Nonblocking Multileader Locality: 4.180302e-04 +Size 2 +PMPI: 5.355768e-04 +Pairwise: 8.180266e-03 +NonBlocking: 9.741700e-02 +Pairwise Hierarchical: 2.816547e-03 +Pairwise Multileader: 1.189534e-03 +Pairwise Node Aware: 5.484794e-04 +Pairwise Locality Aware: 8.380343e-04 +Pairwise Multileader Locality: 9.368978e-04 +Nonblocking Hierarchical: 2.215741e-03 +Nonblocking Multileader: 5.684860e-04 +Nonblocking Node Aware: 3.647819e-04 +Nonblocking Locality Aware: 5.362290e-04 +Nonblocking Multileader Locality: 6.938924e-04 +Size 4 +PMPI: 9.181966e-04 +Pairwise: 1.551127e-02 +NonBlocking: 1.635261e-01 +Pairwise Hierarchical: 4.448523e-03 +Pairwise Multileader: 1.778576e-03 +Pairwise Node Aware: 6.873928e-04 +Pairwise Locality Aware: 1.087874e-03 +Pairwise Multileader Locality: 1.429416e-03 +Nonblocking Hierarchical: 3.472883e-03 +Nonblocking Multileader: 9.404494e-04 +Nonblocking Node Aware: 5.917454e-04 +Nonblocking Locality Aware: 4.750888e-04 +Nonblocking Multileader Locality: 1.083314e-03 +Size 8 +PMPI: 1.580968e-03 +Pairwise: 2.551227e-02 +NonBlocking: 2.275557e-01 +Pairwise Hierarchical: 9.718899e-03 +Pairwise Multileader: 3.134214e-03 +Pairwise Node Aware: 9.379527e-04 +Pairwise Locality Aware: 1.623475e-03 +Pairwise Multileader Locality: 3.517527e-03 +Nonblocking Hierarchical: 8.256516e-03 +Nonblocking Multileader: 2.075083e-03 +Nonblocking Node Aware: 1.001777e-03 +Nonblocking Locality Aware: 7.014891e-04 +Nonblocking Multileader Locality: 2.655792e-03 +Size 16 +PMPI: 3.061137e-03 +Pairwise: 2.197802e-02 +NonBlocking: 1.401895e-01 +Pairwise Hierarchical: 2.019523e-02 +Pairwise Multileader: 5.174253e-03 +Pairwise Node Aware: 1.734849e-03 +Pairwise Locality Aware: 1.694275e-03 +Pairwise Multileader Locality: 6.266972e-03 +Nonblocking Hierarchical: 1.636769e-02 +Nonblocking Multileader: 3.983880e-03 +Nonblocking Node Aware: 1.234513e-03 +Nonblocking Locality Aware: 1.332869e-03 +Nonblocking Multileader Locality: 4.785077e-03 +Size 32 +PMPI: 4.747770e-03 +Pairwise: 2.909341e-02 +NonBlocking: 1.400999e-01 +Pairwise Hierarchical: 4.143412e-02 +Pairwise Multileader: 1.475912e-02 +Pairwise Node Aware: 2.754932e-03 +Pairwise Locality Aware: 2.509821e-03 +Pairwise Multileader Locality: 1.119474e-02 +Nonblocking Hierarchical: 3.306597e-02 +Nonblocking Multileader: 8.682827e-03 +Nonblocking Node Aware: 2.377207e-03 +Nonblocking Locality Aware: 2.547594e-03 +Nonblocking Multileader Locality: 8.838162e-03 +Size 64 +PMPI: 8.942962e-03 +Pairwise: 2.391537e-02 +NonBlocking: 1.407550e-01 +Pairwise Hierarchical: 8.252718e-02 +Pairwise Multileader: 2.610517e-02 +Pairwise Node Aware: 4.897786e-03 +Pairwise Locality Aware: 4.525842e-03 +Pairwise Multileader Locality: 2.232009e-02 +Nonblocking Hierarchical: 6.373445e-02 +Nonblocking Multileader: 1.647486e-02 +Nonblocking Node Aware: 4.123372e-03 +Nonblocking Locality Aware: 5.979264e-03 +Nonblocking Multileader Locality: 1.716870e-02 +Size 128 +PMPI: 1.620160e-02 +Pairwise: 2.727110e-02 +NonBlocking: 1.398663e-01 +Pairwise Hierarchical: 1.562663e-01 +Pairwise Multileader: 5.158359e-02 +Pairwise Node Aware: 1.054778e-02 +Pairwise Locality Aware: 9.998199e-03 +Pairwise Multileader Locality: 1.974956e-01 +Nonblocking Hierarchical: 1.302888e-01 +Nonblocking Multileader: 3.664195e-02 +Nonblocking Node Aware: 8.607103e-03 +Nonblocking Locality Aware: 1.061724e-02 +Nonblocking Multileader Locality: 4.012623e-02 +Size 256 +PMPI: 3.020167e-02 +Pairwise: 3.601652e-02 +NonBlocking: 1.451508e-01 +Pairwise Hierarchical: 2.966210e-01 +Pairwise Multileader: 9.496537e-02 +Pairwise Node Aware: 3.121089e-02 +Pairwise Locality Aware: 1.614516e-02 +Pairwise Multileader Locality: 1.031166e-01 +Nonblocking Hierarchical: 2.455383e-01 +Nonblocking Multileader: 6.815936e-02 +Nonblocking Node Aware: 2.704126e-02 +Nonblocking Locality Aware: 1.389283e-02 +Nonblocking Multileader Locality: 8.125949e-02 +Size 512 +PMPI: 8.919328e-02 +Pairwise: 4.792274e-02 +NonBlocking: 2.103336e-01 +Pairwise Hierarchical: 5.807283e-01 +Pairwise Multileader: 1.809058e-01 +Pairwise Node Aware: 6.043961e-02 +Pairwise Locality Aware: 3.517041e-02 +Pairwise Multileader Locality: 1.980674e-01 +Nonblocking Hierarchical: 5.374810e-01 +Nonblocking Multileader: 1.298174e-01 +Nonblocking Node Aware: 2.787925e-02 +Nonblocking Locality Aware: 3.239136e-02 +Nonblocking Multileader Locality: 1.533290e-01 +Size 1024 +PMPI: 1.234171e-01 +Pairwise: 1.001119e-01 +NonBlocking: 1.664863e-01 +Pairwise Hierarchical: 1.192228e+00 +Pairwise Multileader: 3.231694e-01 +Pairwise Node Aware: 5.647645e-02 +Pairwise Locality Aware: 6.286526e-02 +Pairwise Multileader Locality: 3.709937e-01 +Nonblocking Hierarchical: 1.088141e+00 +Nonblocking Multileader: 2.447068e-01 +Nonblocking Node Aware: 5.089854e-02 +Nonblocking Locality Aware: 4.637694e-02 +Nonblocking Multileader Locality: 2.951201e-01 +Size 2048 +PMPI: 1.610121e-01 +Pairwise: 1.570130e-01 +NonBlocking: 2.344915e-01 +Pairwise Hierarchical: 2.421365e+00 +Pairwise Multileader: 6.599351e-01 +Pairwise Node Aware: 1.148635e-01 +Pairwise Locality Aware: 1.225454e-01 +Pairwise Multileader Locality: 7.404191e-01 +Nonblocking Hierarchical: 2.125905e+00 +Nonblocking Multileader: 5.358387e-01 +Nonblocking Node Aware: 9.748197e-02 +Nonblocking Locality Aware: 9.024167e-02 +Nonblocking Multileader Locality: 6.424130e-01 +Size 4096 +PMPI: 3.363515e-01 +Pairwise: 3.291363e-01 +NonBlocking: 5.886965e-01 +Test 2 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 4.474513e-04 +Pairwise: 9.873309e-03 +NonBlocking: 9.696428e-02 +Pairwise Hierarchical: 1.378321e-03 +Pairwise Multileader: 1.651359e-03 +Pairwise Node Aware: 5.558236e-04 +Pairwise Locality Aware: 9.348718e-04 +Pairwise Multileader Locality: 5.899990e-04 +Nonblocking Hierarchical: 9.282500e-04 +Nonblocking Multileader: 3.146761e-04 +Nonblocking Node Aware: 2.849175e-04 +Nonblocking Locality Aware: 3.638990e-04 +Nonblocking Multileader Locality: 3.873515e-04 +Size 2 +PMPI: 5.288371e-04 +Pairwise: 9.990707e-03 +NonBlocking: 1.238522e-01 +Pairwise Hierarchical: 2.824927e-03 +Pairwise Multileader: 1.650065e-03 +Pairwise Node Aware: 6.505040e-04 +Pairwise Locality Aware: 1.050187e-03 +Pairwise Multileader Locality: 9.550015e-04 +Nonblocking Hierarchical: 2.143294e-03 +Nonblocking Multileader: 5.671355e-04 +Nonblocking Node Aware: 3.821007e-04 +Nonblocking Locality Aware: 4.341386e-04 +Nonblocking Multileader Locality: 6.647732e-04 +Size 4 +PMPI: 8.748781e-04 +Pairwise: 3.906911e-02 +NonBlocking: 1.242233e-01 +Pairwise Hierarchical: 4.277773e-03 +Pairwise Multileader: 2.143263e-03 +Pairwise Node Aware: 7.483007e-04 +Pairwise Locality Aware: 1.385879e-03 +Pairwise Multileader Locality: 1.437874e-03 +Nonblocking Hierarchical: 3.515345e-03 +Nonblocking Multileader: 1.089848e-03 +Nonblocking Node Aware: 6.998692e-04 +Nonblocking Locality Aware: 5.227505e-04 +Nonblocking Multileader Locality: 1.115044e-03 +Size 8 +PMPI: 1.451169e-03 +Pairwise: 1.928191e-02 +NonBlocking: 1.901250e-01 +Pairwise Hierarchical: 1.031663e-02 +Pairwise Multileader: 4.298527e-03 +Pairwise Node Aware: 1.121907e-03 +Pairwise Locality Aware: 1.471797e-03 +Pairwise Multileader Locality: 3.487219e-03 +Nonblocking Hierarchical: 8.121106e-03 +Nonblocking Multileader: 2.084389e-03 +Nonblocking Node Aware: 7.963580e-04 +Nonblocking Locality Aware: 8.521543e-04 +Nonblocking Multileader Locality: 2.592262e-03 +Size 16 +PMPI: 3.024831e-03 +Pairwise: 2.759490e-02 +NonBlocking: 2.163359e-01 +Pairwise Hierarchical: 2.114886e-02 +Pairwise Multileader: 6.050336e-03 +Pairwise Node Aware: 1.506147e-03 +Pairwise Locality Aware: 1.572597e-03 +Pairwise Multileader Locality: 6.186781e-03 +Nonblocking Hierarchical: 1.657076e-02 +Nonblocking Multileader: 3.710732e-03 +Nonblocking Node Aware: 1.233514e-03 +Nonblocking Locality Aware: 1.350326e-03 +Nonblocking Multileader Locality: 4.720827e-03 +Size 32 +PMPI: 4.785070e-03 +Pairwise: 1.804744e-02 +NonBlocking: 1.405765e-01 +Pairwise Hierarchical: 5.029883e-02 +Pairwise Multileader: 2.530176e-02 +Pairwise Node Aware: 7.848940e-03 +Pairwise Locality Aware: 6.599260e-03 +Pairwise Multileader Locality: 1.659107e-02 +Nonblocking Hierarchical: 3.493807e-02 +Nonblocking Multileader: 1.621595e-02 +Nonblocking Node Aware: 1.052916e-02 +Nonblocking Locality Aware: 3.508531e-03 +Nonblocking Multileader Locality: 9.615082e-03 +Size 64 +PMPI: 9.063035e-03 +Pairwise: 2.628343e-02 +NonBlocking: 1.415124e-01 +Pairwise Hierarchical: 8.304449e-02 +Pairwise Multileader: 2.642586e-02 +Pairwise Node Aware: 5.770148e-03 +Pairwise Locality Aware: 9.239199e-03 +Pairwise Multileader Locality: 2.509935e-02 +Nonblocking Hierarchical: 6.363078e-02 +Nonblocking Multileader: 1.659730e-02 +Nonblocking Node Aware: 3.986120e-03 +Nonblocking Locality Aware: 5.544172e-03 +Nonblocking Multileader Locality: 1.734231e-02 +Size 128 +PMPI: 1.641043e-02 +Pairwise: 2.718779e-02 +NonBlocking: 1.399058e-01 +Pairwise Hierarchical: 1.580483e-01 +Pairwise Multileader: 5.474688e-02 +Pairwise Node Aware: 1.220296e-02 +Pairwise Locality Aware: 9.920610e-03 +Pairwise Multileader Locality: 5.543730e-02 +Nonblocking Hierarchical: 1.298192e-01 +Nonblocking Multileader: 3.707331e-02 +Nonblocking Node Aware: 8.163369e-03 +Nonblocking Locality Aware: 1.406145e-02 +Nonblocking Multileader Locality: 4.075956e-02 +Size 256 +PMPI: 3.102591e-02 +Pairwise: 3.357834e-02 +NonBlocking: 1.443820e-01 +Pairwise Hierarchical: 2.963840e-01 +Pairwise Multileader: 9.502753e-02 +Pairwise Node Aware: 1.419007e-02 +Pairwise Locality Aware: 1.727370e-02 +Pairwise Multileader Locality: 1.047367e-01 +Nonblocking Hierarchical: 2.475986e-01 +Nonblocking Multileader: 6.803528e-02 +Nonblocking Node Aware: 1.434952e-02 +Nonblocking Locality Aware: 1.445371e-02 +Nonblocking Multileader Locality: 7.842438e-02 +Size 512 +PMPI: 8.411810e-02 +Pairwise: 5.373306e-02 +NonBlocking: 1.557251e-01 +Pairwise Hierarchical: 5.939739e-01 +Pairwise Multileader: 1.804935e-01 +Pairwise Node Aware: 2.769468e-02 +Pairwise Locality Aware: 3.580747e-02 +Pairwise Multileader Locality: 1.999405e-01 +Nonblocking Hierarchical: 5.179011e-01 +Nonblocking Multileader: 1.311917e-01 +Nonblocking Node Aware: 2.753933e-02 +Nonblocking Locality Aware: 3.147426e-02 +Nonblocking Multileader Locality: 1.530371e-01 +Size 1024 +PMPI: 8.335926e-02 +Pairwise: 8.851125e-02 +NonBlocking: 1.682647e-01 +Pairwise Hierarchical: 1.207520e+00 +Pairwise Multileader: 3.371188e-01 +Pairwise Node Aware: 5.641617e-02 +Pairwise Locality Aware: 6.477774e-02 +Pairwise Multileader Locality: 3.769550e-01 +Nonblocking Hierarchical: 1.105918e+00 +Nonblocking Multileader: 2.462719e-01 +Nonblocking Node Aware: 4.989127e-02 +Nonblocking Locality Aware: 4.531039e-02 +Nonblocking Multileader Locality: 2.972360e-01 +Size 2048 +PMPI: 1.592613e-01 +Pairwise: 1.579304e-01 +NonBlocking: 2.664679e-01 +Pairwise Hierarchical: 2.409357e+00 +Pairwise Multileader: 6.663831e-01 +Pairwise Node Aware: 1.110786e-01 +Pairwise Locality Aware: 1.215727e-01 +Pairwise Multileader Locality: 7.412632e-01 +Nonblocking Hierarchical: 2.117233e+00 +Nonblocking Multileader: 5.318824e-01 +Nonblocking Node Aware: 9.762955e-02 +Nonblocking Locality Aware: 9.044396e-02 +Nonblocking Multileader Locality: 6.312989e-01 +Size 4096 +PMPI: 3.288203e-01 +Pairwise: 3.290161e-01 +NonBlocking: 5.355945e-01 +Test 3 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 3.918832e-04 +Pairwise: 8.061284e-03 +NonBlocking: 8.086323e-02 +Pairwise Hierarchical: 1.218690e-03 +Pairwise Multileader: 1.049310e-03 +Pairwise Node Aware: 4.430708e-04 +Pairwise Locality Aware: 8.841077e-04 +Pairwise Multileader Locality: 5.685668e-04 +Nonblocking Hierarchical: 1.725492e-03 +Nonblocking Multileader: 3.694942e-04 +Nonblocking Node Aware: 2.906499e-04 +Nonblocking Locality Aware: 4.475603e-04 +Nonblocking Multileader Locality: 4.010789e-04 +Size 2 +PMPI: 5.644817e-04 +Pairwise: 1.388749e-02 +NonBlocking: 9.549697e-02 +Pairwise Hierarchical: 2.944074e-03 +Pairwise Multileader: 1.509350e-03 +Pairwise Node Aware: 5.135218e-04 +Pairwise Locality Aware: 1.058768e-03 +Pairwise Multileader Locality: 1.052597e-03 +Nonblocking Hierarchical: 2.268230e-03 +Nonblocking Multileader: 5.748317e-04 +Nonblocking Node Aware: 4.069474e-04 +Nonblocking Locality Aware: 4.167586e-04 +Nonblocking Multileader Locality: 7.328814e-04 +Size 4 +PMPI: 1.012272e-03 +Pairwise: 2.555053e-02 +NonBlocking: 1.643512e-01 +Pairwise Hierarchical: 4.331877e-03 +Pairwise Multileader: 2.222204e-03 +Pairwise Node Aware: 9.797071e-04 +Pairwise Locality Aware: 1.306422e-03 +Pairwise Multileader Locality: 1.562330e-03 +Nonblocking Hierarchical: 3.469400e-03 +Nonblocking Multileader: 9.649211e-04 +Nonblocking Node Aware: 5.834012e-04 +Nonblocking Locality Aware: 8.658117e-04 +Nonblocking Multileader Locality: 1.118411e-03 +Size 8 +PMPI: 1.503797e-03 +Pairwise: 1.882147e-02 +NonBlocking: 1.864566e-01 +Pairwise Hierarchical: 1.008415e-02 +Pairwise Multileader: 3.596610e-03 +Pairwise Node Aware: 1.259257e-03 +Pairwise Locality Aware: 1.408532e-03 +Pairwise Multileader Locality: 3.724486e-03 +Nonblocking Hierarchical: 8.423218e-03 +Nonblocking Multileader: 2.359355e-03 +Nonblocking Node Aware: 9.588440e-04 +Nonblocking Locality Aware: 1.176968e-03 +Nonblocking Multileader Locality: 2.807076e-03 +Size 16 +PMPI: 3.079999e-03 +Pairwise: 2.310991e-02 +NonBlocking: 1.858813e-01 +Pairwise Hierarchical: 2.140077e-02 +Pairwise Multileader: 5.779051e-03 +Pairwise Node Aware: 1.491097e-03 +Pairwise Locality Aware: 1.774887e-03 +Pairwise Multileader Locality: 6.725857e-03 +Nonblocking Hierarchical: 1.679583e-02 +Nonblocking Multileader: 4.366458e-03 +Nonblocking Node Aware: 1.725710e-03 +Nonblocking Locality Aware: 1.682314e-03 +Nonblocking Multileader Locality: 4.720573e-03 +Size 32 +PMPI: 4.601547e-03 +Pairwise: 1.917841e-02 +NonBlocking: 1.407452e-01 +Pairwise Hierarchical: 4.221455e-02 +Pairwise Multileader: 1.419570e-02 +Pairwise Node Aware: 2.670870e-03 +Pairwise Locality Aware: 3.395138e-03 +Pairwise Multileader Locality: 1.129007e-02 +Nonblocking Hierarchical: 3.355986e-02 +Nonblocking Multileader: 8.609600e-03 +Nonblocking Node Aware: 2.391144e-03 +Nonblocking Locality Aware: 2.791184e-03 +Nonblocking Multileader Locality: 8.681803e-03 +Size 64 +PMPI: 9.376339e-03 +Pairwise: 2.727059e-02 +NonBlocking: 1.413501e-01 +Pairwise Hierarchical: 8.250081e-02 +Pairwise Multileader: 2.731430e-02 +Pairwise Node Aware: 5.071020e-03 +Pairwise Locality Aware: 4.084812e-03 +Pairwise Multileader Locality: 2.318903e-02 +Nonblocking Hierarchical: 6.412840e-02 +Nonblocking Multileader: 1.653894e-02 +Nonblocking Node Aware: 4.045530e-03 +Nonblocking Locality Aware: 4.434825e-03 +Nonblocking Multileader Locality: 1.746746e-02 +Size 128 +PMPI: 1.740819e-02 +Pairwise: 2.658580e-02 +NonBlocking: 1.398717e-01 +Pairwise Hierarchical: 1.567479e-01 +Pairwise Multileader: 5.239586e-02 +Pairwise Node Aware: 1.067602e-02 +Pairwise Locality Aware: 9.979320e-03 +Pairwise Multileader Locality: 5.538442e-02 +Nonblocking Hierarchical: 1.314141e-01 +Nonblocking Multileader: 3.718273e-02 +Nonblocking Node Aware: 8.080450e-03 +Nonblocking Locality Aware: 1.008805e-02 +Nonblocking Multileader Locality: 4.086261e-02 +Size 256 +PMPI: 3.472897e-02 +Pairwise: 3.358536e-02 +NonBlocking: 1.459414e-01 +Pairwise Hierarchical: 2.967721e-01 +Pairwise Multileader: 9.623978e-02 +Pairwise Node Aware: 4.874312e-02 +Pairwise Locality Aware: 1.674983e-02 +Pairwise Multileader Locality: 1.049443e-01 +Nonblocking Hierarchical: 2.476474e-01 +Nonblocking Multileader: 6.796220e-02 +Nonblocking Node Aware: 1.432048e-02 +Nonblocking Locality Aware: 1.425371e-02 +Nonblocking Multileader Locality: 7.811796e-02 +Size 512 +PMPI: 1.052678e-01 +Pairwise: 4.682587e-02 +NonBlocking: 1.812305e-01 +Pairwise Hierarchical: 5.961809e-01 +Pairwise Multileader: 1.800852e-01 +Pairwise Node Aware: 2.874808e-02 +Pairwise Locality Aware: 3.490520e-02 +Pairwise Multileader Locality: 2.013135e-01 +Nonblocking Hierarchical: 5.261334e-01 +Nonblocking Multileader: 1.310490e-01 +Nonblocking Node Aware: 2.745115e-02 +Nonblocking Locality Aware: 3.168883e-02 +Nonblocking Multileader Locality: 1.529888e-01 +Size 1024 +PMPI: 9.880165e-02 +Pairwise: 7.951258e-02 +NonBlocking: 1.694415e-01 +Pairwise Hierarchical: 1.190531e+00 +Pairwise Multileader: 3.270255e-01 +Pairwise Node Aware: 5.846385e-02 +Pairwise Locality Aware: 6.144319e-02 +Pairwise Multileader Locality: 3.712357e-01 +Nonblocking Hierarchical: 1.118953e+00 +Nonblocking Multileader: 2.509718e-01 +Nonblocking Node Aware: 5.081590e-02 +Nonblocking Locality Aware: 4.614368e-02 +Nonblocking Multileader Locality: 2.944906e-01 +Size 2048 +PMPI: 1.584758e-01 +Pairwise: 1.632405e-01 +NonBlocking: 2.519457e-01 +Pairwise Hierarchical: 2.467747e+00 +Pairwise Multileader: 6.673177e-01 +Pairwise Node Aware: 1.102928e-01 +Pairwise Locality Aware: 1.234748e-01 +Pairwise Multileader Locality: 7.424467e-01 +Nonblocking Hierarchical: 2.245363e+00 +Nonblocking Multileader: 5.315671e-01 +Nonblocking Node Aware: 9.724532e-02 +Nonblocking Locality Aware: 8.957767e-02 +Nonblocking Multileader Locality: 6.382273e-01 +Size 4096 +PMPI: 3.243954e-01 +Pairwise: 3.350969e-01 +NonBlocking: 5.050783e-01 diff --git a/runscripts/04_28_25/alltoall_N2.2361038.err b/runscripts/04_28_25/alltoall_N2.2361038.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/04_28_25/alltoall_N2.2361038.out b/runscripts/04_28_25/alltoall_N2.2361038.out new file mode 100644 index 000000000..365ba721a --- /dev/null +++ b/runscripts/04_28_25/alltoall_N2.2361038.out @@ -0,0 +1,633 @@ +Test 1 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 5.322727e-05 +Pairwise: 4.662741e-04 +NonBlocking: 1.937750e-04 +Pairwise Hierarchical: 1.854800e-04 +Pairwise Multileader: 7.455504e-05 +Pairwise Node Aware: 1.816279e-04 +Pairwise Locality Aware: 9.253347e-05 +Pairwise Multileader Locality: 6.127904e-05 +Nonblocking Hierarchical: 1.814854e-04 +Nonblocking Multileader: 4.969996e-05 +Nonblocking Node Aware: 1.772781e-04 +Nonblocking Locality Aware: 5.337703e-05 +Nonblocking Multileader Locality: 6.125966e-05 +Size 2 +PMPI: 5.644378e-05 +Pairwise: 4.687677e-04 +NonBlocking: 2.154717e-04 +Pairwise Hierarchical: 2.268821e-04 +Pairwise Multileader: 8.800770e-05 +Pairwise Node Aware: 1.892135e-04 +Pairwise Locality Aware: 8.710803e-05 +Pairwise Multileader Locality: 8.677735e-05 +Nonblocking Hierarchical: 2.218708e-04 +Nonblocking Multileader: 6.769134e-05 +Nonblocking Node Aware: 1.765739e-04 +Nonblocking Locality Aware: 5.914665e-05 +Nonblocking Multileader Locality: 8.161447e-05 +Size 4 +PMPI: 7.389654e-05 +Pairwise: 9.409267e-04 +NonBlocking: 2.759614e-02 +Pairwise Hierarchical: 4.482746e-04 +Pairwise Multileader: 1.412747e-04 +Pairwise Node Aware: 2.799005e-04 +Pairwise Locality Aware: 1.062137e-04 +Pairwise Multileader Locality: 1.229920e-04 +Nonblocking Hierarchical: 4.416712e-04 +Nonblocking Multileader: 1.081138e-04 +Nonblocking Node Aware: 1.832149e-04 +Nonblocking Locality Aware: 6.963982e-05 +Nonblocking Multileader Locality: 1.183275e-04 +Size 8 +PMPI: 1.183200e-04 +Pairwise: 9.231766e-04 +NonBlocking: 2.488259e-02 +Pairwise Hierarchical: 7.772702e-04 +Pairwise Multileader: 2.363640e-04 +Pairwise Node Aware: 2.671438e-04 +Pairwise Locality Aware: 1.286336e-04 +Pairwise Multileader Locality: 3.141192e-04 +Nonblocking Hierarchical: 7.921373e-04 +Nonblocking Multileader: 1.860546e-04 +Nonblocking Node Aware: 1.856463e-04 +Nonblocking Locality Aware: 9.272892e-05 +Nonblocking Multileader Locality: 2.759434e-04 +Size 16 +PMPI: 1.856379e-04 +Pairwise: 9.996620e-04 +NonBlocking: 6.183001e-02 +Pairwise Hierarchical: 1.485553e-03 +Pairwise Multileader: 4.675547e-04 +Pairwise Node Aware: 3.543179e-04 +Pairwise Locality Aware: 1.682386e-04 +Pairwise Multileader Locality: 5.630934e-04 +Nonblocking Hierarchical: 1.474454e-03 +Nonblocking Multileader: 3.618078e-04 +Nonblocking Node Aware: 1.954265e-04 +Nonblocking Locality Aware: 1.214632e-04 +Nonblocking Multileader Locality: 4.972647e-04 +Size 32 +PMPI: 3.270388e-04 +Pairwise: 1.100327e-03 +NonBlocking: 4.955409e-02 +Pairwise Hierarchical: 2.997215e-03 +Pairwise Multileader: 1.077690e-03 +Pairwise Node Aware: 4.629891e-04 +Pairwise Locality Aware: 3.145953e-04 +Pairwise Multileader Locality: 1.118689e-03 +Nonblocking Hierarchical: 2.985277e-03 +Nonblocking Multileader: 9.090690e-04 +Nonblocking Node Aware: 2.773364e-04 +Nonblocking Locality Aware: 2.354882e-04 +Nonblocking Multileader Locality: 9.612052e-04 +Size 64 +PMPI: 5.456203e-04 +Pairwise: 1.105914e-03 +NonBlocking: 4.973180e-02 +Pairwise Hierarchical: 6.944062e-03 +Pairwise Multileader: 2.346313e-03 +Pairwise Node Aware: 6.041262e-04 +Pairwise Locality Aware: 4.383460e-04 +Pairwise Multileader Locality: 2.289475e-03 +Nonblocking Hierarchical: 6.810824e-03 +Nonblocking Multileader: 2.020981e-03 +Nonblocking Node Aware: 3.927816e-04 +Nonblocking Locality Aware: 3.484678e-04 +Nonblocking Multileader Locality: 2.122328e-03 +Size 128 +PMPI: 9.810487e-04 +Pairwise: 1.325036e-03 +NonBlocking: 7.436998e-02 +Pairwise Hierarchical: 1.420411e-02 +Pairwise Multileader: 4.568863e-03 +Pairwise Node Aware: 1.039082e-03 +Pairwise Locality Aware: 7.604484e-04 +Pairwise Multileader Locality: 4.734763e-03 +Nonblocking Hierarchical: 1.362026e-02 +Nonblocking Multileader: 3.856464e-03 +Nonblocking Node Aware: 8.538773e-04 +Nonblocking Locality Aware: 8.726953e-04 +Nonblocking Multileader Locality: 4.384547e-03 +Size 256 +PMPI: 2.171018e-03 +Pairwise: 2.132844e-03 +NonBlocking: 7.463013e-02 +Pairwise Hierarchical: 2.837308e-02 +Pairwise Multileader: 7.926889e-03 +Pairwise Node Aware: 1.419240e-03 +Pairwise Locality Aware: 1.443234e-03 +Pairwise Multileader Locality: 8.433561e-03 +Nonblocking Hierarchical: 2.829445e-02 +Nonblocking Multileader: 6.680704e-03 +Nonblocking Node Aware: 1.157881e-03 +Nonblocking Locality Aware: 1.329136e-03 +Nonblocking Multileader Locality: 7.806889e-03 +Size 512 +PMPI: 2.664911e-03 +Pairwise: 3.690350e-03 +NonBlocking: 1.198749e-02 +Pairwise Hierarchical: 5.763806e-02 +Pairwise Multileader: 1.744698e-02 +Pairwise Node Aware: 2.488338e-03 +Pairwise Locality Aware: 2.666160e-03 +Pairwise Multileader Locality: 1.695918e-02 +Nonblocking Hierarchical: 5.775506e-02 +Nonblocking Multileader: 1.388147e-02 +Nonblocking Node Aware: 2.402871e-03 +Nonblocking Locality Aware: 2.290979e-03 +Nonblocking Multileader Locality: 1.542131e-02 +Size 1024 +PMPI: 6.174838e-03 +Pairwise: 4.998066e-03 +NonBlocking: 5.929183e-03 +Pairwise Hierarchical: 1.051341e-01 +Pairwise Multileader: 3.876491e-02 +Pairwise Node Aware: 6.170976e-03 +Pairwise Locality Aware: 5.323262e-03 +Pairwise Multileader Locality: 4.052341e-02 +Nonblocking Hierarchical: 1.129250e-01 +Nonblocking Multileader: 3.334351e-02 +Nonblocking Node Aware: 5.056591e-03 +Nonblocking Locality Aware: 5.350588e-03 +Nonblocking Multileader Locality: 3.764224e-02 +Size 2048 +PMPI: 1.113522e-02 +Pairwise: 1.026604e-02 +NonBlocking: 1.316342e-02 +Pairwise Hierarchical: 2.106222e-01 +Pairwise Multileader: 7.344055e-02 +Pairwise Node Aware: 1.122405e-02 +Pairwise Locality Aware: 9.631745e-03 +Pairwise Multileader Locality: 7.826523e-02 +Nonblocking Hierarchical: 2.128901e-01 +Nonblocking Multileader: 6.270283e-02 +Nonblocking Node Aware: 9.580881e-03 +Nonblocking Locality Aware: 8.895488e-03 +Nonblocking Multileader Locality: 7.341903e-02 +Size 4096 +PMPI: 2.081220e-02 +Pairwise: 1.871391e-02 +NonBlocking: 2.358586e-02 +Pairwise Hierarchical: 4.410384e-01 +Pairwise Multileader: 1.355652e-01 +Pairwise Node Aware: 2.341815e-02 +Pairwise Locality Aware: 2.328952e-02 +Pairwise Multileader Locality: 1.472866e-01 +Nonblocking Hierarchical: 4.445635e-01 +Nonblocking Multileader: 1.213292e-01 +Nonblocking Node Aware: 2.356430e-02 +Nonblocking Locality Aware: 1.929068e-02 +Nonblocking Multileader Locality: 1.403941e-01 +Size 8192 +PMPI: 3.663580e-02 +Pairwise: 3.747401e-02 +NonBlocking: 4.296840e-02 +Pairwise Hierarchical: 9.342160e-01 +Pairwise Multileader: 2.615054e-01 +Pairwise Node Aware: 4.616930e-02 +Pairwise Locality Aware: 5.137616e-02 +Pairwise Multileader Locality: 2.937743e-01 +Nonblocking Hierarchical: 9.336232e-01 +Nonblocking Multileader: 2.345209e-01 +Nonblocking Node Aware: 4.209648e-02 +Nonblocking Locality Aware: 4.042492e-02 +Nonblocking Multileader Locality: 2.709144e-01 +Size 16384 +PMPI: 6.646016e-02 +Pairwise: 8.052603e-02 +NonBlocking: 6.806394e-02 +Pairwise Hierarchical: 1.898351e+00 +Pairwise Multileader: 5.316889e-01 +Pairwise Node Aware: 1.050416e-01 +Pairwise Locality Aware: 9.849505e-02 +Pairwise Multileader Locality: 6.193053e-01 +Nonblocking Hierarchical: 1.906504e+00 +Nonblocking Multileader: 5.038207e-01 +Nonblocking Node Aware: 8.912927e-02 +Nonblocking Locality Aware: 9.639005e-02 +Nonblocking Multileader Locality: 5.759712e-01 +Test 2 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 5.829179e-05 +Pairwise: 4.698466e-04 +NonBlocking: 1.914274e-04 +Pairwise Hierarchical: 1.865858e-04 +Pairwise Multileader: 7.376726e-05 +Pairwise Node Aware: 1.891161e-04 +Pairwise Locality Aware: 8.680437e-05 +Pairwise Multileader Locality: 6.477483e-05 +Nonblocking Hierarchical: 1.831618e-04 +Nonblocking Multileader: 5.104079e-05 +Nonblocking Node Aware: 1.812109e-04 +Nonblocking Locality Aware: 5.183687e-05 +Nonblocking Multileader Locality: 6.117799e-05 +Size 2 +PMPI: 5.849895e-05 +Pairwise: 5.323366e-04 +NonBlocking: 2.060694e-04 +Pairwise Hierarchical: 2.373582e-04 +Pairwise Multileader: 1.015681e-04 +Pairwise Node Aware: 1.920936e-04 +Pairwise Locality Aware: 8.556404e-05 +Pairwise Multileader Locality: 8.630696e-05 +Nonblocking Hierarchical: 2.239544e-04 +Nonblocking Multileader: 6.726761e-05 +Nonblocking Node Aware: 1.787852e-04 +Nonblocking Locality Aware: 5.603849e-05 +Nonblocking Multileader Locality: 8.402646e-05 +Size 4 +PMPI: 8.465745e-05 +Pairwise: 9.707587e-04 +NonBlocking: 3.718035e-02 +Pairwise Hierarchical: 4.305986e-04 +Pairwise Multileader: 1.401051e-04 +Pairwise Node Aware: 2.540387e-04 +Pairwise Locality Aware: 1.119351e-04 +Pairwise Multileader Locality: 1.313480e-04 +Nonblocking Hierarchical: 4.271762e-04 +Nonblocking Multileader: 1.110204e-04 +Nonblocking Node Aware: 1.859320e-04 +Nonblocking Locality Aware: 6.353097e-05 +Nonblocking Multileader Locality: 1.186696e-04 +Size 8 +PMPI: 9.990419e-05 +Pairwise: 1.032202e-03 +NonBlocking: 2.926517e-02 +Pairwise Hierarchical: 7.917143e-04 +Pairwise Multileader: 2.406895e-04 +Pairwise Node Aware: 2.701399e-04 +Pairwise Locality Aware: 1.048913e-04 +Pairwise Multileader Locality: 3.015283e-04 +Nonblocking Hierarchical: 7.497214e-04 +Nonblocking Multileader: 1.797509e-04 +Nonblocking Node Aware: 1.876791e-04 +Nonblocking Locality Aware: 1.205776e-04 +Nonblocking Multileader Locality: 2.619434e-04 +Size 16 +PMPI: 1.693155e-04 +Pairwise: 9.117677e-04 +NonBlocking: 4.953269e-02 +Pairwise Hierarchical: 1.492962e-03 +Pairwise Multileader: 4.440108e-04 +Pairwise Node Aware: 3.459908e-04 +Pairwise Locality Aware: 1.754998e-04 +Pairwise Multileader Locality: 5.445543e-04 +Nonblocking Hierarchical: 1.493253e-03 +Nonblocking Multileader: 3.763367e-04 +Nonblocking Node Aware: 2.123733e-04 +Nonblocking Locality Aware: 1.267476e-04 +Nonblocking Multileader Locality: 4.808483e-04 +Size 32 +PMPI: 2.952147e-04 +Pairwise: 1.205246e-03 +NonBlocking: 5.218866e-02 +Pairwise Hierarchical: 2.953143e-03 +Pairwise Multileader: 1.044434e-03 +Pairwise Node Aware: 4.669516e-04 +Pairwise Locality Aware: 2.271647e-04 +Pairwise Multileader Locality: 1.060218e-03 +Nonblocking Hierarchical: 2.900064e-03 +Nonblocking Multileader: 8.583500e-04 +Nonblocking Node Aware: 2.566494e-04 +Nonblocking Locality Aware: 2.204919e-04 +Nonblocking Multileader Locality: 9.562972e-04 +Size 64 +PMPI: 6.125565e-04 +Pairwise: 1.173209e-03 +NonBlocking: 6.190870e-02 +Pairwise Hierarchical: 6.851511e-03 +Pairwise Multileader: 2.371536e-03 +Pairwise Node Aware: 6.022782e-04 +Pairwise Locality Aware: 4.221777e-04 +Pairwise Multileader Locality: 2.337209e-03 +Nonblocking Hierarchical: 6.637486e-03 +Nonblocking Multileader: 1.910265e-03 +Nonblocking Node Aware: 3.508641e-04 +Nonblocking Locality Aware: 3.720748e-04 +Nonblocking Multileader Locality: 2.133283e-03 +Size 128 +PMPI: 9.872275e-04 +Pairwise: 1.304581e-03 +NonBlocking: 8.665652e-02 +Pairwise Hierarchical: 1.396024e-02 +Pairwise Multileader: 4.563525e-03 +Pairwise Node Aware: 9.500207e-04 +Pairwise Locality Aware: 7.353632e-04 +Pairwise Multileader Locality: 4.749769e-03 +Nonblocking Hierarchical: 1.340029e-02 +Nonblocking Multileader: 3.814288e-03 +Nonblocking Node Aware: 7.706443e-04 +Nonblocking Locality Aware: 7.358212e-04 +Nonblocking Multileader Locality: 4.296056e-03 +Size 256 +PMPI: 1.902754e-03 +Pairwise: 1.964136e-03 +NonBlocking: 7.490329e-02 +Pairwise Hierarchical: 2.846930e-02 +Pairwise Multileader: 7.981792e-03 +Pairwise Node Aware: 1.518117e-03 +Pairwise Locality Aware: 1.260177e-03 +Pairwise Multileader Locality: 8.487545e-03 +Nonblocking Hierarchical: 2.823622e-02 +Nonblocking Multileader: 6.648099e-03 +Nonblocking Node Aware: 1.234278e-03 +Nonblocking Locality Aware: 1.503708e-03 +Nonblocking Multileader Locality: 7.694345e-03 +Size 512 +PMPI: 2.474874e-03 +Pairwise: 3.540130e-03 +NonBlocking: 1.113028e-02 +Pairwise Hierarchical: 5.682810e-02 +Pairwise Multileader: 1.716969e-02 +Pairwise Node Aware: 3.224473e-03 +Pairwise Locality Aware: 2.760437e-03 +Pairwise Multileader Locality: 1.702444e-02 +Nonblocking Hierarchical: 5.702203e-02 +Nonblocking Multileader: 1.395674e-02 +Nonblocking Node Aware: 2.142348e-03 +Nonblocking Locality Aware: 2.562515e-03 +Nonblocking Multileader Locality: 1.554184e-02 +Size 1024 +PMPI: 5.008008e-03 +Pairwise: 5.487011e-03 +NonBlocking: 7.029955e-03 +Pairwise Hierarchical: 1.037185e-01 +Pairwise Multileader: 3.783729e-02 +Pairwise Node Aware: 5.476362e-03 +Pairwise Locality Aware: 4.677732e-03 +Pairwise Multileader Locality: 3.999093e-02 +Nonblocking Hierarchical: 1.114869e-01 +Nonblocking Multileader: 3.226375e-02 +Nonblocking Node Aware: 4.886020e-03 +Nonblocking Locality Aware: 4.768744e-03 +Nonblocking Multileader Locality: 3.782786e-02 +Size 2048 +PMPI: 9.785893e-03 +Pairwise: 1.139522e-02 +NonBlocking: 1.093456e-02 +Pairwise Hierarchical: 2.083806e-01 +Pairwise Multileader: 7.270505e-02 +Pairwise Node Aware: 1.105867e-02 +Pairwise Locality Aware: 9.999110e-03 +Pairwise Multileader Locality: 7.736171e-02 +Nonblocking Hierarchical: 2.125213e-01 +Nonblocking Multileader: 6.324911e-02 +Nonblocking Node Aware: 9.866488e-03 +Nonblocking Locality Aware: 9.309630e-03 +Nonblocking Multileader Locality: 7.248914e-02 +Size 4096 +PMPI: 2.097173e-02 +Pairwise: 2.345650e-02 +NonBlocking: 2.047454e-02 +Pairwise Hierarchical: 4.340778e-01 +Pairwise Multileader: 1.337329e-01 +Pairwise Node Aware: 2.668685e-02 +Pairwise Locality Aware: 2.105806e-02 +Pairwise Multileader Locality: 1.479105e-01 +Nonblocking Hierarchical: 4.390655e-01 +Nonblocking Multileader: 1.210512e-01 +Nonblocking Node Aware: 2.112265e-02 +Nonblocking Locality Aware: 1.904996e-02 +Nonblocking Multileader Locality: 1.402961e-01 +Size 8192 +PMPI: 3.993515e-02 +Pairwise: 4.391041e-02 +NonBlocking: 3.532392e-02 +Pairwise Hierarchical: 9.213113e-01 +Pairwise Multileader: 2.572029e-01 +Pairwise Node Aware: 4.700272e-02 +Pairwise Locality Aware: 4.629742e-02 +Pairwise Multileader Locality: 2.918940e-01 +Nonblocking Hierarchical: 9.098918e-01 +Nonblocking Multileader: 2.302829e-01 +Nonblocking Node Aware: 4.070375e-02 +Nonblocking Locality Aware: 4.151146e-02 +Nonblocking Multileader Locality: 2.703587e-01 +Size 16384 +PMPI: 7.340886e-02 +Pairwise: 7.052150e-02 +NonBlocking: 6.294783e-02 +Pairwise Hierarchical: 1.914134e+00 +Pairwise Multileader: 5.305902e-01 +Pairwise Node Aware: 9.132127e-02 +Pairwise Locality Aware: 9.498272e-02 +Pairwise Multileader Locality: 6.208632e-01 +Nonblocking Hierarchical: 1.883127e+00 +Nonblocking Multileader: 4.939101e-01 +Nonblocking Node Aware: 8.926914e-02 +Nonblocking Locality Aware: 9.499180e-02 +Nonblocking Multileader Locality: 5.678431e-01 +Test 3 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 5.036154e-05 +Pairwise: 4.585713e-04 +NonBlocking: 2.241115e-04 +Pairwise Hierarchical: 1.842693e-04 +Pairwise Multileader: 8.034901e-05 +Pairwise Node Aware: 2.109972e-04 +Pairwise Locality Aware: 8.921220e-05 +Pairwise Multileader Locality: 6.280071e-05 +Nonblocking Hierarchical: 1.800936e-04 +Nonblocking Multileader: 4.893002e-05 +Nonblocking Node Aware: 1.779718e-04 +Nonblocking Locality Aware: 5.725002e-05 +Nonblocking Multileader Locality: 6.118332e-05 +Size 2 +PMPI: 4.894529e-05 +Pairwise: 4.289414e-04 +NonBlocking: 2.076285e-04 +Pairwise Hierarchical: 2.285807e-04 +Pairwise Multileader: 8.902732e-05 +Pairwise Node Aware: 1.870691e-04 +Pairwise Locality Aware: 1.055314e-04 +Pairwise Multileader Locality: 9.124698e-05 +Nonblocking Hierarchical: 2.391968e-04 +Nonblocking Multileader: 6.817955e-05 +Nonblocking Node Aware: 1.804310e-04 +Nonblocking Locality Aware: 5.761724e-05 +Nonblocking Multileader Locality: 8.145885e-05 +Size 4 +PMPI: 7.228376e-05 +Pairwise: 1.114859e-03 +NonBlocking: 2.264346e-02 +Pairwise Hierarchical: 4.234916e-04 +Pairwise Multileader: 1.335028e-04 +Pairwise Node Aware: 2.508934e-04 +Pairwise Locality Aware: 1.030947e-04 +Pairwise Multileader Locality: 1.273911e-04 +Nonblocking Hierarchical: 4.320187e-04 +Nonblocking Multileader: 1.087826e-04 +Nonblocking Node Aware: 1.824617e-04 +Nonblocking Locality Aware: 6.615998e-05 +Nonblocking Multileader Locality: 1.189537e-04 +Size 8 +PMPI: 1.104490e-04 +Pairwise: 8.841861e-04 +NonBlocking: 2.450929e-02 +Pairwise Hierarchical: 7.808417e-04 +Pairwise Multileader: 2.401024e-04 +Pairwise Node Aware: 2.821354e-04 +Pairwise Locality Aware: 1.086301e-04 +Pairwise Multileader Locality: 3.003771e-04 +Nonblocking Hierarchical: 7.453990e-04 +Nonblocking Multileader: 1.822670e-04 +Nonblocking Node Aware: 1.804916e-04 +Nonblocking Locality Aware: 9.661393e-05 +Nonblocking Multileader Locality: 2.586205e-04 +Size 16 +PMPI: 1.431359e-04 +Pairwise: 8.794811e-04 +NonBlocking: 1.254663e-02 +Pairwise Hierarchical: 1.453356e-03 +Pairwise Multileader: 4.423338e-04 +Pairwise Node Aware: 3.582659e-04 +Pairwise Locality Aware: 1.530966e-04 +Pairwise Multileader Locality: 5.346696e-04 +Nonblocking Hierarchical: 1.448956e-03 +Nonblocking Multileader: 3.685029e-04 +Nonblocking Node Aware: 1.938452e-04 +Nonblocking Locality Aware: 1.580842e-04 +Nonblocking Multileader Locality: 4.770547e-04 +Size 32 +PMPI: 3.156958e-04 +Pairwise: 8.921341e-04 +NonBlocking: 3.726514e-02 +Pairwise Hierarchical: 2.938720e-03 +Pairwise Multileader: 1.058546e-03 +Pairwise Node Aware: 4.635598e-04 +Pairwise Locality Aware: 2.451479e-04 +Pairwise Multileader Locality: 1.028303e-03 +Nonblocking Hierarchical: 2.859818e-03 +Nonblocking Multileader: 8.708417e-04 +Nonblocking Node Aware: 2.698795e-04 +Nonblocking Locality Aware: 2.211526e-04 +Nonblocking Multileader Locality: 9.238502e-04 +Size 64 +PMPI: 5.277759e-04 +Pairwise: 1.122752e-03 +NonBlocking: 4.960165e-02 +Pairwise Hierarchical: 6.892511e-03 +Pairwise Multileader: 2.385032e-03 +Pairwise Node Aware: 5.766599e-04 +Pairwise Locality Aware: 3.820253e-04 +Pairwise Multileader Locality: 2.315884e-03 +Nonblocking Hierarchical: 6.695885e-03 +Nonblocking Multileader: 1.882657e-03 +Nonblocking Node Aware: 3.356705e-04 +Nonblocking Locality Aware: 3.296783e-04 +Nonblocking Multileader Locality: 2.116763e-03 +Size 128 +PMPI: 9.582998e-04 +Pairwise: 1.265187e-03 +NonBlocking: 6.207556e-02 +Pairwise Hierarchical: 1.394165e-02 +Pairwise Multileader: 4.559980e-03 +Pairwise Node Aware: 9.519117e-04 +Pairwise Locality Aware: 6.421810e-04 +Pairwise Multileader Locality: 4.691611e-03 +Nonblocking Hierarchical: 1.324173e-02 +Nonblocking Multileader: 3.781665e-03 +Nonblocking Node Aware: 6.087286e-04 +Nonblocking Locality Aware: 7.040703e-04 +Nonblocking Multileader Locality: 4.336091e-03 +Size 256 +PMPI: 1.733534e-03 +Pairwise: 1.643596e-03 +NonBlocking: 7.458801e-02 +Pairwise Hierarchical: 2.896809e-02 +Pairwise Multileader: 7.899758e-03 +Pairwise Node Aware: 1.563386e-03 +Pairwise Locality Aware: 1.280541e-03 +Pairwise Multileader Locality: 8.447764e-03 +Nonblocking Hierarchical: 2.855162e-02 +Nonblocking Multileader: 6.700241e-03 +Nonblocking Node Aware: 1.055960e-03 +Nonblocking Locality Aware: 1.361459e-03 +Nonblocking Multileader Locality: 7.657303e-03 +Size 512 +PMPI: 2.497526e-03 +Pairwise: 3.135673e-03 +NonBlocking: 1.125800e-02 +Pairwise Hierarchical: 5.626505e-02 +Pairwise Multileader: 1.741476e-02 +Pairwise Node Aware: 2.438145e-03 +Pairwise Locality Aware: 2.481670e-03 +Pairwise Multileader Locality: 1.719080e-02 +Nonblocking Hierarchical: 5.709614e-02 +Nonblocking Multileader: 1.394172e-02 +Nonblocking Node Aware: 2.184699e-03 +Nonblocking Locality Aware: 2.637323e-03 +Nonblocking Multileader Locality: 1.531143e-02 +Size 1024 +PMPI: 5.038062e-03 +Pairwise: 6.176164e-03 +NonBlocking: 4.748717e-03 +Pairwise Hierarchical: 1.056149e-01 +Pairwise Multileader: 3.804427e-02 +Pairwise Node Aware: 4.931012e-03 +Pairwise Locality Aware: 5.012348e-03 +Pairwise Multileader Locality: 4.037326e-02 +Nonblocking Hierarchical: 1.142967e-01 +Nonblocking Multileader: 3.277072e-02 +Nonblocking Node Aware: 4.273287e-03 +Nonblocking Locality Aware: 4.006568e-03 +Nonblocking Multileader Locality: 3.724824e-02 +Size 2048 +PMPI: 1.128348e-02 +Pairwise: 1.010017e-02 +NonBlocking: 9.968100e-03 +Pairwise Hierarchical: 2.083892e-01 +Pairwise Multileader: 7.291828e-02 +Pairwise Node Aware: 1.252632e-02 +Pairwise Locality Aware: 8.996553e-03 +Pairwise Multileader Locality: 7.762113e-02 +Nonblocking Hierarchical: 2.103401e-01 +Nonblocking Multileader: 6.357150e-02 +Nonblocking Node Aware: 1.061714e-02 +Nonblocking Locality Aware: 9.300278e-03 +Nonblocking Multileader Locality: 7.294009e-02 +Size 4096 +PMPI: 2.038147e-02 +Pairwise: 1.932110e-02 +NonBlocking: 2.171449e-02 +Pairwise Hierarchical: 4.342042e-01 +Pairwise Multileader: 1.355433e-01 +Pairwise Node Aware: 2.403933e-02 +Pairwise Locality Aware: 2.073823e-02 +Pairwise Multileader Locality: 1.501961e-01 +Nonblocking Hierarchical: 4.422022e-01 +Nonblocking Multileader: 1.208973e-01 +Nonblocking Node Aware: 2.073605e-02 +Nonblocking Locality Aware: 2.128966e-02 +Nonblocking Multileader Locality: 1.387401e-01 +Size 8192 +PMPI: 3.737043e-02 +Pairwise: 4.004769e-02 +NonBlocking: 3.339730e-02 +Pairwise Hierarchical: 9.202448e-01 +Pairwise Multileader: 2.552240e-01 +Pairwise Node Aware: 5.069760e-02 +Pairwise Locality Aware: 4.171861e-02 +Pairwise Multileader Locality: 2.923417e-01 +Nonblocking Hierarchical: 9.345821e-01 +Nonblocking Multileader: 2.329009e-01 +Nonblocking Node Aware: 4.112090e-02 +Nonblocking Locality Aware: 4.632164e-02 +Nonblocking Multileader Locality: 2.709965e-01 +Size 16384 +PMPI: 7.476979e-02 +Pairwise: 6.384177e-02 +NonBlocking: 6.729634e-02 +Pairwise Hierarchical: 1.923605e+00 +Pairwise Multileader: 5.298662e-01 +Pairwise Node Aware: 1.008503e-01 +Pairwise Locality Aware: 8.525612e-02 +Pairwise Multileader Locality: 6.106966e-01 +Nonblocking Hierarchical: 1.908492e+00 +Nonblocking Multileader: 4.997505e-01 +Nonblocking Node Aware: 8.359124e-02 +Nonblocking Locality Aware: 9.621506e-02 +Nonblocking Multileader Locality: 5.768631e-01 diff --git a/runscripts/04_28_25/alltoall_N32.2361058.err b/runscripts/04_28_25/alltoall_N32.2361058.err new file mode 100644 index 000000000..f895ec877 --- /dev/null +++ b/runscripts/04_28_25/alltoall_N32.2361058.err @@ -0,0 +1,792 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2361058.0 ON dane99 CANCELLED AT 2025-04-28T16:03:34 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1483: task 2605: Killed +srun: error: dane860: tasks 1605,1609: Killed +srun: error: dane366: task 605: Killed +srun: error: dane705: tasks 1403,1405: Killed +srun: error: dane679: tasks 1305,1309,1315: Killed +srun: error: dane756: tasks 1505,1509,1513: Killed +srun: error: dane860: tasks 1600-1604,1606-1608,1610-1699: Killed +srun: error: dane1539: tasks 3100-3199: Killed +srun: error: dane1347: tasks 2300-2399: Killed +srun: error: dane1483: tasks 2600-2604,2606-2699: Killed +srun: error: dane1127: tasks 2000-2099: Killed +srun: error: dane1276: tasks 2100-2199: Killed +srun: error: dane1021: tasks 1800-1899: Killed +srun: error: dane705: tasks 1400-1402,1404,1406-1499: Killed +srun: error: dane679: tasks 1300-1304,1306-1308,1310-1314,1316-1399: Killed +srun: error: dane1446: tasks 2400-2499: Killed +srun: error: dane481: tasks 1000-1099: Killed +srun: error: dane1517: tasks 2900-2999: Killed +srun: error: dane1520: tasks 3000-3099: Killed +srun: error: dane1044: tasks 1900-1999: Killed +srun: error: dane1451: tasks 2500-2599: Killed +srun: error: dane366: tasks 600-604,606-699: Killed +srun: error: dane377: tasks 700-799: Killed +srun: error: dane756: tasks 1500-1504,1506-1508,1510-1512,1514-1599: Killed +srun: error: dane1014: tasks 1700-1799: Killed +srun: error: dane409: tasks 900-999: Killed +srun: error: dane220: tasks 400-499: Killed +srun: error: dane574: tasks 1200-1299: Killed +srun: error: dane528: tasks 1100-1199: Killed +srun: error: dane1493: tasks 2800-2899: Killed +srun: error: dane392: tasks 800-899: Killed +srun: error: dane250: tasks 500-599: Killed +srun: error: dane99: tasks 0-99: Killed +srun: error: dane137: tasks 100-199: Killed +srun: error: dane181: tasks 200-299: Killed +srun: error: dane185: tasks 300-399: Killed +srun: error: dane1318: tasks 2200-2299: Killed +srun: error: dane1492: tasks 2700-2799: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2361058.1 ON dane99 CANCELLED AT 2025-04-28T16:08:07 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1492: task 2703: Killed +srun: error: dane1517: task 2915: Killed +srun: error: dane1347: task 2305: Killed +srun: error: dane1483: tasks 2607,2615: Killed +srun: error: dane679: tasks 1305,1309,1313: Killed +srun: error: dane860: tasks 1600-1699: Killed +srun: error: dane1539: tasks 3100-3199: Killed +srun: error: dane1347: tasks 2300-2304,2306-2399: Killed +srun: error: dane1483: tasks 2600-2606,2608-2614,2616-2699: Killed +srun: error: dane1127: tasks 2000-2099: Killed +srun: error: dane1276: tasks 2100-2199: Killed +srun: error: dane1021: tasks 1800-1899: Killed +srun: error: dane705: tasks 1400-1499: Killed +srun: error: dane1446: tasks 2400-2499: Killed +srun: error: dane1517: tasks 2900-2914,2916-2999: Killed +srun: error: dane679: tasks 1300-1304,1306-1308,1310-1312,1314-1399: Killed +srun: error: dane481: tasks 1000-1099: Killed +srun: error: dane1520: tasks 3000-3099: Killed +srun: error: dane1044: tasks 1900-1999: Killed +srun: error: dane1451: tasks 2500-2599: Killed +srun: error: dane366: tasks 600-699: Killed +srun: error: dane1014: tasks 1700-1799: Killed +srun: error: dane756: tasks 1500-1599: Killed +srun: error: dane377: tasks 700-799: Killed +srun: error: dane409: tasks 900-999: Killed +srun: error: dane220: tasks 400-499: Killed +srun: error: dane574: tasks 1200-1299: Killed +srun: error: dane528: tasks 1100-1199: Killed +srun: error: dane1493: tasks 2800-2899: Killed +srun: error: dane99: tasks 0-99: Killed +srun: error: dane392: tasks 800-899: Killed +srun: error: dane250: tasks 500-599: Killed +srun: error: dane1318: tasks 2200-2299: Killed +srun: error: dane1492: tasks 2700-2702,2704-2799: Killed +srun: error: dane185: tasks 300-399: Killed +srun: error: dane181: tasks 200-299: Killed +srun: error: dane137: tasks 100-199: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2361058.2 ON dane99 CANCELLED AT 2025-04-28T16:12:42 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1044: task 1905: Killed +srun: error: dane679: task 1305: Killed +srun: error: dane756: task 1505: Killed +srun: error: dane392: task 805: Killed +srun: error: dane1483: tasks 2603,2613,2617,2621,2623: Killed +srun: error: dane1493: tasks 2811,2819: Killed +srun: error: dane99: tasks 3,19: Killed +srun: error: dane860: tasks 1600-1699: Killed +srun: error: dane1539: tasks 3100-3199: Killed +srun: error: dane1347: tasks 2300-2399: Killed +srun: error: dane1483: tasks 2600-2602,2604-2612,2614-2616,2618-2620,2622,2624-2699: Killed +srun: error: dane1127: tasks 2000-2099: Killed +srun: error: dane1276: tasks 2100-2199: Killed +srun: error: dane1021: tasks 1800-1899: Killed +srun: error: dane705: tasks 1400-1499: Killed +srun: error: dane1446: tasks 2400-2499: Killed +srun: error: dane1517: tasks 2900-2999: Killed +srun: error: dane679: tasks 1300-1304,1306-1399: Killed +srun: error: dane481: tasks 1000-1099: Killed +srun: error: dane1520: tasks 3000-3099: Killed +srun: error: dane1044: tasks 1900-1904,1906-1999: Killed +srun: error: dane1451: tasks 2500-2599: Killed +srun: error: dane366: tasks 600-699: Killed +srun: error: dane1014: tasks 1700-1799: Killed +srun: error: dane377: tasks 700-799: Killed +srun: error: dane756: tasks 1500-1504,1506-1599: Killed +srun: error: dane409: tasks 900-999: Killed +srun: error: dane220: tasks 400-499: Killed +srun: error: dane574: tasks 1200-1299: Killed +srun: error: dane528: tasks 1100-1199: Killed +srun: error: dane1493: tasks 2800-2810,2812-2818,2820-2899: Killed +srun: error: dane1318: tasks 2200-2299: Killed +srun: error: dane1492: tasks 2700-2799: Killed +srun: error: dane392: tasks 800-804,806-899: Killed +srun: error: dane99: tasks 0-2,4-18,20-99: Killed +srun: error: dane250: tasks 500-599: Killed +srun: error: dane137: tasks 100-199: Killed +srun: error: dane185: tasks 300-399: Killed +srun: error: dane181: tasks 200-299: Killed diff --git a/runscripts/04_28_25/alltoall_N32.2361058.out b/runscripts/04_28_25/alltoall_N32.2361058.out new file mode 100644 index 000000000..416a2ffd1 --- /dev/null +++ b/runscripts/04_28_25/alltoall_N32.2361058.out @@ -0,0 +1,477 @@ +Test 1 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.014898e-03 +Pairwise: 3.340758e-02 +NonBlocking: 3.299628e-01 +Pairwise Hierarchical: 2.770020e-03 +Pairwise Multileader: 2.900331e-03 +Pairwise Node Aware: 8.350737e-04 +Pairwise Locality Aware: 2.280125e-03 +Pairwise Multileader Locality: 1.192307e-03 +Nonblocking Hierarchical: 1.981842e-03 +Nonblocking Multileader: 1.471624e-03 +Nonblocking Node Aware: 4.591923e-04 +Nonblocking Locality Aware: 7.461130e-02 +Nonblocking Multileader Locality: 7.626361e-04 +Size 2 +PMPI: 1.351944e-03 +Pairwise: 2.618433e-02 +NonBlocking: 8.869265e-01 +Pairwise Hierarchical: 6.125918e-03 +Pairwise Multileader: 2.911378e-03 +Pairwise Node Aware: 1.032212e-03 +Pairwise Locality Aware: 2.630895e-03 +Pairwise Multileader Locality: 1.984334e-03 +Nonblocking Hierarchical: 4.345143e-03 +Nonblocking Multileader: 1.702081e-03 +Nonblocking Node Aware: 6.542305e-04 +Nonblocking Locality Aware: 9.199483e-02 +Nonblocking Multileader Locality: 1.320340e-03 +Size 4 +PMPI: 1.972671e-03 +Pairwise: 7.225908e-02 +NonBlocking: 3.816728e-01 +Pairwise Hierarchical: 1.079977e-02 +Pairwise Multileader: 4.520655e-03 +Pairwise Node Aware: 1.072834e-03 +Pairwise Locality Aware: 3.913021e-03 +Pairwise Multileader Locality: 3.397813e-03 +Nonblocking Hierarchical: 8.472152e-03 +Nonblocking Multileader: 3.042084e-03 +Nonblocking Node Aware: 9.186373e-04 +Nonblocking Locality Aware: 8.684877e-02 +Nonblocking Multileader Locality: 2.489963e-03 +Size 8 +PMPI: 4.255068e-03 +Pairwise: 8.775783e-02 +NonBlocking: 3.800824e-01 +Pairwise Hierarchical: 2.282690e-02 +Pairwise Multileader: 7.103742e-03 +Pairwise Node Aware: 1.629330e-03 +Pairwise Locality Aware: 2.911419e-03 +Pairwise Multileader Locality: 7.246280e-03 +Nonblocking Hierarchical: 1.704662e-02 +Nonblocking Multileader: 4.166096e-03 +Nonblocking Node Aware: 1.762901e-03 +Nonblocking Locality Aware: 9.287116e-02 +Nonblocking Multileader Locality: 5.062707e-03 +Size 16 +PMPI: 6.188659e-03 +Pairwise: 7.763675e-02 +NonBlocking: 4.869879e-01 +Pairwise Hierarchical: 4.468176e-02 +Pairwise Multileader: 1.042341e-02 +Pairwise Node Aware: 3.651320e-03 +Pairwise Locality Aware: 3.825096e-03 +Pairwise Multileader Locality: 1.215554e-02 +Nonblocking Hierarchical: 3.422755e-02 +Nonblocking Multileader: 6.924135e-03 +Nonblocking Node Aware: 2.722990e-03 +Nonblocking Locality Aware: 9.384741e-02 +Nonblocking Multileader Locality: 8.811011e-03 +Size 32 +PMPI: 1.239060e-02 +Pairwise: 5.758671e-02 +NonBlocking: 3.291629e-01 +Pairwise Hierarchical: 8.653925e-02 +Pairwise Multileader: 3.172599e-02 +Pairwise Node Aware: 5.393320e-03 +Pairwise Locality Aware: 5.114733e-03 +Pairwise Multileader Locality: 2.451248e-02 +Nonblocking Hierarchical: 6.584741e-02 +Nonblocking Multileader: 1.801410e-02 +Nonblocking Node Aware: 5.628440e-03 +Nonblocking Locality Aware: 6.171557e-02 +Nonblocking Multileader Locality: 1.786712e-02 +Size 64 +PMPI: 2.707086e-02 +Pairwise: 4.268186e-02 +NonBlocking: 3.029344e-01 +Pairwise Hierarchical: 1.651751e-01 +Pairwise Multileader: 5.574032e-02 +Pairwise Node Aware: 9.859881e-03 +Pairwise Locality Aware: 8.914552e-03 +Pairwise Multileader Locality: 5.933412e-02 +Nonblocking Hierarchical: 1.322555e-01 +Nonblocking Multileader: 3.772285e-02 +Nonblocking Node Aware: 8.259329e-03 +Nonblocking Locality Aware: 1.595069e-02 +Nonblocking Multileader Locality: 4.178532e-02 +Size 128 +PMPI: 4.616448e-02 +Pairwise: 6.134620e-02 +NonBlocking: 1.809642e-01 +Pairwise Hierarchical: 3.225215e-01 +Pairwise Multileader: 9.862890e-02 +Pairwise Node Aware: 2.090857e-02 +Pairwise Locality Aware: 2.028265e-02 +Pairwise Multileader Locality: 1.070962e-01 +Nonblocking Hierarchical: 2.476191e-01 +Nonblocking Multileader: 6.821710e-02 +Nonblocking Node Aware: 1.829781e-02 +Nonblocking Locality Aware: 2.551901e-02 +Nonblocking Multileader Locality: 7.842761e-02 +Size 256 +PMPI: 8.931041e-02 +Pairwise: 1.132730e-01 +NonBlocking: 3.524548e-01 +Pairwise Hierarchical: 6.305858e-01 +Pairwise Multileader: 1.866556e-01 +Pairwise Node Aware: 3.606053e-02 +Pairwise Locality Aware: 5.489557e-02 +Pairwise Multileader Locality: 2.099447e-01 +Nonblocking Hierarchical: 5.280741e-01 +Nonblocking Multileader: 1.313165e-01 +Nonblocking Node Aware: 3.030379e-02 +Nonblocking Locality Aware: 3.234349e-02 +Nonblocking Multileader Locality: 1.519146e-01 +Size 512 +PMPI: 5.753912e-01 +Pairwise: 1.122833e-01 +NonBlocking: 2.701518e-01 +Pairwise Hierarchical: 1.261569e+00 +Pairwise Multileader: 3.475921e-01 +Pairwise Node Aware: 6.496565e-02 +Pairwise Locality Aware: 7.817704e-02 +Pairwise Multileader Locality: 3.905106e-01 +Nonblocking Hierarchical: 1.112162e+00 +Nonblocking Multileader: 2.539786e-01 +Nonblocking Node Aware: 5.236427e-02 +Nonblocking Locality Aware: 7.066968e-02 +Nonblocking Multileader Locality: 3.073299e-01 +Size 1024 +PMPI: 2.460819e-01 +Pairwise: 2.255405e-01 +NonBlocking: 3.536874e-01 +Pairwise Hierarchical: 2.511642e+00 +Pairwise Multileader: 7.002035e-01 +Pairwise Node Aware: 1.351465e-01 +Pairwise Locality Aware: 1.595642e-01 +Pairwise Multileader Locality: 7.816454e-01 +Nonblocking Hierarchical: 2.218307e+00 +Nonblocking Multileader: 5.773213e-01 +Nonblocking Node Aware: 1.288831e-01 +Nonblocking Locality Aware: 3.191902e-01 +Nonblocking Multileader Locality: 6.569860e-01 +Size 2048 +PMPI: 3.856768e-01 +Pairwise: 4.133719e-01 +NonBlocking: 5.949811e-01 +Test 2 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.035023e-03 +Pairwise: 3.255329e-02 +NonBlocking: 5.676726e-01 +Pairwise Hierarchical: 3.059244e-03 +Pairwise Multileader: 3.477507e-03 +Pairwise Node Aware: 1.000777e-03 +Pairwise Locality Aware: 3.269248e-03 +Pairwise Multileader Locality: 1.302183e-03 +Nonblocking Hierarchical: 2.220685e-03 +Nonblocking Multileader: 1.653506e-03 +Nonblocking Node Aware: 4.769877e-04 +Nonblocking Locality Aware: 6.905859e-02 +Nonblocking Multileader Locality: 7.364629e-04 +Size 2 +PMPI: 1.323488e-03 +Pairwise: 4.217145e-02 +NonBlocking: 3.276366e-01 +Pairwise Hierarchical: 6.451162e-03 +Pairwise Multileader: 4.030157e-03 +Pairwise Node Aware: 9.676042e-04 +Pairwise Locality Aware: 2.521016e-03 +Pairwise Multileader Locality: 2.521164e-03 +Nonblocking Hierarchical: 4.497177e-03 +Nonblocking Multileader: 1.132076e-03 +Nonblocking Node Aware: 6.707941e-04 +Nonblocking Locality Aware: 8.692026e-02 +Nonblocking Multileader Locality: 1.341005e-03 +Size 4 +PMPI: 1.877885e-03 +Pairwise: 7.545708e-02 +NonBlocking: 4.070895e-01 +Pairwise Hierarchical: 1.117572e-02 +Pairwise Multileader: 4.334639e-03 +Pairwise Node Aware: 1.165938e-03 +Pairwise Locality Aware: 2.987292e-03 +Pairwise Multileader Locality: 3.348545e-03 +Nonblocking Hierarchical: 8.578873e-03 +Nonblocking Multileader: 5.117665e-03 +Nonblocking Node Aware: 9.002204e-04 +Nonblocking Locality Aware: 1.116024e-01 +Nonblocking Multileader Locality: 2.607112e-03 +Size 8 +PMPI: 4.311823e-03 +Pairwise: 5.974037e-02 +NonBlocking: 3.305677e-01 +Pairwise Hierarchical: 2.283720e-02 +Pairwise Multileader: 6.828145e-03 +Pairwise Node Aware: 1.671704e-03 +Pairwise Locality Aware: 3.503331e-03 +Pairwise Multileader Locality: 7.489662e-03 +Nonblocking Hierarchical: 1.737117e-02 +Nonblocking Multileader: 4.145885e-03 +Nonblocking Node Aware: 1.526902e-03 +Nonblocking Locality Aware: 9.989045e-02 +Nonblocking Multileader Locality: 5.075000e-03 +Size 16 +PMPI: 6.156821e-03 +Pairwise: 6.931892e-02 +NonBlocking: 7.558758e-01 +Pairwise Hierarchical: 4.655800e-02 +Pairwise Multileader: 1.082309e-02 +Pairwise Node Aware: 2.937795e-03 +Pairwise Locality Aware: 4.115196e-03 +Pairwise Multileader Locality: 1.267103e-02 +Nonblocking Hierarchical: 3.494393e-02 +Nonblocking Multileader: 7.507766e-03 +Nonblocking Node Aware: 3.279038e-03 +Nonblocking Locality Aware: 9.355937e-02 +Nonblocking Multileader Locality: 9.054149e-03 +Size 32 +PMPI: 1.233434e-02 +Pairwise: 5.082144e-02 +NonBlocking: 3.680333e-01 +Pairwise Hierarchical: 8.586566e-02 +Pairwise Multileader: 3.037076e-02 +Pairwise Node Aware: 5.128899e-03 +Pairwise Locality Aware: 5.934489e-03 +Pairwise Multileader Locality: 2.385536e-02 +Nonblocking Hierarchical: 6.566493e-02 +Nonblocking Multileader: 1.812493e-02 +Nonblocking Node Aware: 5.609655e-03 +Nonblocking Locality Aware: 5.767067e-02 +Nonblocking Multileader Locality: 1.773368e-02 +Size 64 +PMPI: 2.499932e-02 +Pairwise: 6.647933e-02 +NonBlocking: 3.022979e-01 +Pairwise Hierarchical: 1.648930e-01 +Pairwise Multileader: 5.566855e-02 +Pairwise Node Aware: 1.070705e-02 +Pairwise Locality Aware: 9.393946e-03 +Pairwise Multileader Locality: 5.788196e-02 +Nonblocking Hierarchical: 1.329705e-01 +Nonblocking Multileader: 3.809795e-02 +Nonblocking Node Aware: 8.281275e-03 +Nonblocking Locality Aware: 1.132146e-02 +Nonblocking Multileader Locality: 4.135900e-02 +Size 128 +PMPI: 4.736169e-02 +Pairwise: 1.040839e-01 +NonBlocking: 2.783586e-01 +Pairwise Hierarchical: 3.179962e-01 +Pairwise Multileader: 9.819016e-02 +Pairwise Node Aware: 2.184452e-02 +Pairwise Locality Aware: 1.753943e-02 +Pairwise Multileader Locality: 1.065552e-01 +Nonblocking Hierarchical: 2.502576e-01 +Nonblocking Multileader: 6.943223e-02 +Nonblocking Node Aware: 2.070366e-02 +Nonblocking Locality Aware: 3.531672e-02 +Nonblocking Multileader Locality: 7.759573e-02 +Size 256 +PMPI: 1.030862e-01 +Pairwise: 9.174182e-02 +NonBlocking: 2.191997e-01 +Pairwise Hierarchical: 6.347703e-01 +Pairwise Multileader: 1.861660e-01 +Pairwise Node Aware: 3.281840e-02 +Pairwise Locality Aware: 4.489094e-02 +Pairwise Multileader Locality: 2.076850e-01 +Nonblocking Hierarchical: 5.248573e-01 +Nonblocking Multileader: 1.318034e-01 +Nonblocking Node Aware: 3.024649e-02 +Nonblocking Locality Aware: 3.302538e-02 +Nonblocking Multileader Locality: 1.525190e-01 +Size 512 +PMPI: 4.167106e-01 +Pairwise: 1.164126e-01 +NonBlocking: 2.992314e-01 +Pairwise Hierarchical: 1.247197e+00 +Pairwise Multileader: 3.496741e-01 +Pairwise Node Aware: 6.796889e-02 +Pairwise Locality Aware: 8.128931e-02 +Pairwise Multileader Locality: 3.917810e-01 +Nonblocking Hierarchical: 1.100791e+00 +Nonblocking Multileader: 2.568681e-01 +Nonblocking Node Aware: 5.176769e-02 +Nonblocking Locality Aware: 1.421001e-01 +Nonblocking Multileader Locality: 3.063814e-01 +Size 1024 +PMPI: 1.953199e-01 +Pairwise: 2.571676e-01 +NonBlocking: 3.609706e-01 +Pairwise Hierarchical: 2.489897e+00 +Pairwise Multileader: 6.920121e-01 +Pairwise Node Aware: 1.346028e-01 +Pairwise Locality Aware: 1.584037e-01 +Pairwise Multileader Locality: 7.799824e-01 +Nonblocking Hierarchical: 2.220986e+00 +Nonblocking Multileader: 5.528150e-01 +Nonblocking Node Aware: 1.072914e-01 +Nonblocking Locality Aware: 9.932518e-02 +Nonblocking Multileader Locality: 6.512338e-01 +Size 2048 +PMPI: 4.313742e-01 +Pairwise: 4.411821e-01 +NonBlocking: 5.935214e-01 +Test 3 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.281412e-03 +Pairwise: 4.266976e-02 +NonBlocking: 5.710624e-01 +Pairwise Hierarchical: 2.646303e-03 +Pairwise Multileader: 3.038507e-03 +Pairwise Node Aware: 1.021727e-03 +Pairwise Locality Aware: 2.292878e-03 +Pairwise Multileader Locality: 1.457219e-03 +Nonblocking Hierarchical: 2.057232e-03 +Nonblocking Multileader: 3.661524e-03 +Nonblocking Node Aware: 4.940809e-04 +Nonblocking Locality Aware: 9.217950e-02 +Nonblocking Multileader Locality: 7.373768e-04 +Size 2 +PMPI: 1.321544e-03 +Pairwise: 4.234253e-02 +NonBlocking: 5.643810e-01 +Pairwise Hierarchical: 5.816752e-03 +Pairwise Multileader: 3.728168e-03 +Pairwise Node Aware: 1.383628e-03 +Pairwise Locality Aware: 3.954841e-03 +Pairwise Multileader Locality: 1.902228e-03 +Nonblocking Hierarchical: 4.352479e-03 +Nonblocking Multileader: 1.652212e-03 +Nonblocking Node Aware: 7.044811e-04 +Nonblocking Locality Aware: 7.440976e-02 +Nonblocking Multileader Locality: 1.276226e-03 +Size 4 +PMPI: 1.847043e-03 +Pairwise: 6.446829e-02 +NonBlocking: 4.071907e-01 +Pairwise Hierarchical: 1.103184e-02 +Pairwise Multileader: 4.561465e-03 +Pairwise Node Aware: 1.103227e-03 +Pairwise Locality Aware: 2.906566e-03 +Pairwise Multileader Locality: 3.138301e-03 +Nonblocking Hierarchical: 8.506514e-03 +Nonblocking Multileader: 6.154125e-03 +Nonblocking Node Aware: 1.073298e-03 +Nonblocking Locality Aware: 8.689730e-02 +Nonblocking Multileader Locality: 2.514476e-03 +Size 8 +PMPI: 3.970088e-03 +Pairwise: 7.088517e-02 +NonBlocking: 3.677404e-01 +Pairwise Hierarchical: 2.281281e-02 +Pairwise Multileader: 7.346312e-03 +Pairwise Node Aware: 1.606836e-03 +Pairwise Locality Aware: 3.449907e-03 +Pairwise Multileader Locality: 7.507901e-03 +Nonblocking Hierarchical: 1.739799e-02 +Nonblocking Multileader: 6.164667e-03 +Nonblocking Node Aware: 1.507892e-03 +Nonblocking Locality Aware: 8.770908e-02 +Nonblocking Multileader Locality: 5.091920e-03 +Size 16 +PMPI: 6.463974e-03 +Pairwise: 7.424084e-02 +NonBlocking: 4.856864e-01 +Pairwise Hierarchical: 4.661708e-02 +Pairwise Multileader: 1.156584e-02 +Pairwise Node Aware: 2.768755e-03 +Pairwise Locality Aware: 4.485044e-03 +Pairwise Multileader Locality: 1.213215e-02 +Nonblocking Hierarchical: 3.501153e-02 +Nonblocking Multileader: 6.832835e-03 +Nonblocking Node Aware: 2.710757e-03 +Nonblocking Locality Aware: 1.161670e-01 +Nonblocking Multileader Locality: 8.747153e-03 +Size 32 +PMPI: 1.273366e-02 +Pairwise: 6.003952e-02 +NonBlocking: 3.806013e-01 +Pairwise Hierarchical: 8.609326e-02 +Pairwise Multileader: 2.962245e-02 +Pairwise Node Aware: 5.292795e-03 +Pairwise Locality Aware: 5.136965e-03 +Pairwise Multileader Locality: 2.431439e-02 +Nonblocking Hierarchical: 6.626391e-02 +Nonblocking Multileader: 1.828165e-02 +Nonblocking Node Aware: 5.688729e-03 +Nonblocking Locality Aware: 5.768146e-02 +Nonblocking Multileader Locality: 1.815067e-02 +Size 64 +PMPI: 2.288813e-02 +Pairwise: 6.470576e-02 +NonBlocking: 2.913712e-01 +Pairwise Hierarchical: 1.680191e-01 +Pairwise Multileader: 5.545921e-02 +Pairwise Node Aware: 1.029625e-02 +Pairwise Locality Aware: 1.116909e-02 +Pairwise Multileader Locality: 5.844188e-02 +Nonblocking Hierarchical: 1.323954e-01 +Nonblocking Multileader: 3.823379e-02 +Nonblocking Node Aware: 8.320408e-03 +Nonblocking Locality Aware: 1.616150e-02 +Nonblocking Multileader Locality: 4.145783e-02 +Size 128 +PMPI: 4.712772e-02 +Pairwise: 9.052742e-02 +NonBlocking: 1.817579e-01 +Pairwise Hierarchical: 3.214564e-01 +Pairwise Multileader: 1.003371e-01 +Pairwise Node Aware: 2.169729e-02 +Pairwise Locality Aware: 2.014893e-02 +Pairwise Multileader Locality: 1.070954e-01 +Nonblocking Hierarchical: 2.490628e-01 +Nonblocking Multileader: 6.855919e-02 +Nonblocking Node Aware: 1.840594e-02 +Nonblocking Locality Aware: 2.582768e-02 +Nonblocking Multileader Locality: 7.860562e-02 +Size 256 +PMPI: 9.531996e-02 +Pairwise: 8.559559e-02 +NonBlocking: 2.840158e-01 +Pairwise Hierarchical: 6.349636e-01 +Pairwise Multileader: 1.878796e-01 +Pairwise Node Aware: 3.315127e-02 +Pairwise Locality Aware: 3.876604e-02 +Pairwise Multileader Locality: 2.083807e-01 +Nonblocking Hierarchical: 5.358187e-01 +Nonblocking Multileader: 1.321519e-01 +Nonblocking Node Aware: 7.061994e-02 +Nonblocking Locality Aware: 3.565950e-02 +Nonblocking Multileader Locality: 1.539542e-01 +Size 512 +PMPI: 4.144106e-01 +Pairwise: 1.156832e-01 +NonBlocking: 2.617514e-01 +Pairwise Hierarchical: 1.256519e+00 +Pairwise Multileader: 3.528705e-01 +Pairwise Node Aware: 6.507482e-02 +Pairwise Locality Aware: 8.215088e-02 +Pairwise Multileader Locality: 3.939100e-01 +Nonblocking Hierarchical: 1.172324e+00 +Nonblocking Multileader: 2.579284e-01 +Nonblocking Node Aware: 5.163019e-02 +Nonblocking Locality Aware: 7.110167e-02 +Nonblocking Multileader Locality: 3.057447e-01 +Size 1024 +PMPI: 2.332165e-01 +Pairwise: 2.019416e-01 +NonBlocking: 3.528168e-01 +Pairwise Hierarchical: 2.517509e+00 +Pairwise Multileader: 7.000039e-01 +Pairwise Node Aware: 1.295801e-01 +Pairwise Locality Aware: 1.533119e-01 +Pairwise Multileader Locality: 7.745869e-01 +Nonblocking Hierarchical: 2.155590e+00 +Nonblocking Multileader: 5.445448e-01 +Nonblocking Node Aware: 1.106746e-01 +Nonblocking Locality Aware: 9.819913e-02 +Nonblocking Multileader Locality: 6.363497e-01 +Size 2048 +PMPI: 4.125044e-01 +Pairwise: 4.040583e-01 +NonBlocking: 6.363201e-01 diff --git a/runscripts/04_28_25/alltoall_N4.2361042.err b/runscripts/04_28_25/alltoall_N4.2361042.err new file mode 100644 index 000000000..8528a26d1 --- /dev/null +++ b/runscripts/04_28_25/alltoall_N4.2361042.err @@ -0,0 +1,102 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2361042.0 ON dane98 CANCELLED AT 2025-04-28T15:59:57 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1053: tasks 300-399: Killed +srun: error: dane462: tasks 200-299: Killed +srun: error: dane414: tasks 100-199: Killed +srun: error: dane98: tasks 0-99: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2361042.1 ON dane98 CANCELLED AT 2025-04-28T16:04:12 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1053: tasks 300-399: Killed +srun: error: dane462: tasks 200-299: Killed +srun: error: dane414: tasks 100-199: Killed +srun: error: dane98: tasks 0-99: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2361042.2 ON dane98 CANCELLED AT 2025-04-28T16:08:30 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1053: tasks 300-399: Killed +srun: error: dane462: tasks 200-299: Killed +srun: error: dane414: tasks 100-199: Killed +srun: error: dane98: tasks 0-99: Killed diff --git a/runscripts/04_28_25/alltoall_N4.2361042.out b/runscripts/04_28_25/alltoall_N4.2361042.out new file mode 100644 index 000000000..e6b3ea73e --- /dev/null +++ b/runscripts/04_28_25/alltoall_N4.2361042.out @@ -0,0 +1,603 @@ +Test 1 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 8.299948e-05 +Pairwise: 9.419689e-04 +NonBlocking: 3.027696e-04 +Pairwise Hierarchical: 3.173636e-04 +Pairwise Multileader: 1.601756e-04 +Pairwise Node Aware: 1.994785e-04 +Pairwise Locality Aware: 1.252750e-04 +Pairwise Multileader Locality: 1.227282e-04 +Nonblocking Hierarchical: 3.459823e-04 +Nonblocking Multileader: 8.491837e-05 +Nonblocking Node Aware: 1.870314e-04 +Nonblocking Locality Aware: 6.055683e-05 +Nonblocking Multileader Locality: 1.077730e-04 +Size 2 +PMPI: 8.728037e-05 +Pairwise: 8.786952e-04 +NonBlocking: 2.841177e-04 +Pairwise Hierarchical: 6.048040e-04 +Pairwise Multileader: 1.752323e-04 +Pairwise Node Aware: 2.487950e-04 +Pairwise Locality Aware: 1.349097e-04 +Pairwise Multileader Locality: 1.710601e-04 +Nonblocking Hierarchical: 5.449787e-04 +Nonblocking Multileader: 1.238380e-04 +Nonblocking Node Aware: 2.037665e-04 +Nonblocking Locality Aware: 8.748246e-05 +Nonblocking Multileader Locality: 1.444403e-04 +Size 4 +PMPI: 1.576288e-04 +Pairwise: 2.779226e-03 +NonBlocking: 8.661991e-02 +Pairwise Hierarchical: 9.498119e-04 +Pairwise Multileader: 3.434596e-04 +Pairwise Node Aware: 3.206284e-04 +Pairwise Locality Aware: 1.477959e-04 +Pairwise Multileader Locality: 3.199848e-04 +Nonblocking Hierarchical: 8.428197e-04 +Nonblocking Multileader: 2.106086e-04 +Nonblocking Node Aware: 1.972083e-04 +Nonblocking Locality Aware: 1.511179e-04 +Nonblocking Multileader Locality: 2.641694e-04 +Size 8 +PMPI: 2.370187e-04 +Pairwise: 1.804302e-03 +NonBlocking: 8.667467e-02 +Pairwise Hierarchical: 1.765826e-03 +Pairwise Multileader: 5.190166e-04 +Pairwise Node Aware: 4.222567e-04 +Pairwise Locality Aware: 1.857899e-04 +Pairwise Multileader Locality: 7.064234e-04 +Nonblocking Hierarchical: 1.593109e-03 +Nonblocking Multileader: 3.850683e-04 +Nonblocking Node Aware: 2.097670e-04 +Nonblocking Locality Aware: 2.182720e-04 +Nonblocking Multileader Locality: 5.510169e-04 +Size 16 +PMPI: 4.303900e-04 +Pairwise: 1.752934e-03 +NonBlocking: 8.674458e-02 +Pairwise Hierarchical: 3.448008e-03 +Pairwise Multileader: 9.277420e-04 +Pairwise Node Aware: 4.871435e-04 +Pairwise Locality Aware: 2.704165e-04 +Pairwise Multileader Locality: 1.200483e-03 +Nonblocking Hierarchical: 3.116033e-03 +Nonblocking Multileader: 7.415859e-04 +Nonblocking Node Aware: 2.794954e-04 +Nonblocking Locality Aware: 2.646216e-04 +Nonblocking Multileader Locality: 1.003231e-03 +Size 32 +PMPI: 7.109940e-04 +Pairwise: 2.019705e-03 +NonBlocking: 8.675707e-02 +Pairwise Hierarchical: 7.412542e-03 +Pairwise Multileader: 2.852668e-03 +Pairwise Node Aware: 6.756674e-04 +Pairwise Locality Aware: 4.348908e-04 +Pairwise Multileader Locality: 2.591829e-03 +Nonblocking Hierarchical: 6.963853e-03 +Nonblocking Multileader: 2.190644e-03 +Nonblocking Node Aware: 4.410838e-04 +Nonblocking Locality Aware: 4.542744e-04 +Nonblocking Multileader Locality: 2.318296e-03 +Size 64 +PMPI: 1.435408e-03 +Pairwise: 2.851557e-03 +NonBlocking: 8.699725e-02 +Pairwise Hierarchical: 1.625735e-02 +Pairwise Multileader: 5.178375e-03 +Pairwise Node Aware: 1.063586e-03 +Pairwise Locality Aware: 8.802534e-04 +Pairwise Multileader Locality: 5.089983e-03 +Nonblocking Hierarchical: 1.457051e-02 +Nonblocking Multileader: 4.046872e-03 +Nonblocking Node Aware: 6.940626e-04 +Nonblocking Locality Aware: 7.179413e-04 +Nonblocking Multileader Locality: 4.529784e-03 +Size 128 +PMPI: 2.364766e-03 +Pairwise: 2.828427e-03 +NonBlocking: 9.981597e-02 +Pairwise Hierarchical: 3.464704e-02 +Pairwise Multileader: 9.701249e-03 +Pairwise Node Aware: 2.157100e-03 +Pairwise Locality Aware: 1.693579e-03 +Pairwise Multileader Locality: 9.274540e-03 +Nonblocking Hierarchical: 3.196082e-02 +Nonblocking Multileader: 7.172294e-03 +Nonblocking Node Aware: 1.328987e-03 +Nonblocking Locality Aware: 1.342446e-03 +Nonblocking Multileader Locality: 8.076202e-03 +Size 256 +PMPI: 4.748902e-03 +Pairwise: 3.874391e-03 +NonBlocking: 8.839767e-02 +Pairwise Hierarchical: 6.617876e-02 +Pairwise Multileader: 2.085139e-02 +Pairwise Node Aware: 3.023361e-03 +Pairwise Locality Aware: 3.227465e-03 +Pairwise Multileader Locality: 1.890968e-02 +Nonblocking Hierarchical: 6.077584e-02 +Nonblocking Multileader: 1.474232e-02 +Nonblocking Node Aware: 1.893819e-03 +Nonblocking Locality Aware: 2.203229e-03 +Nonblocking Multileader Locality: 1.615670e-02 +Size 512 +PMPI: 7.475307e-03 +Pairwise: 7.301797e-03 +NonBlocking: 9.598347e-02 +Pairwise Hierarchical: 1.246629e-01 +Pairwise Multileader: 4.554981e-02 +Pairwise Node Aware: 5.534436e-03 +Pairwise Locality Aware: 5.770685e-03 +Pairwise Multileader Locality: 4.756285e-02 +Nonblocking Hierarchical: 1.320516e-01 +Nonblocking Multileader: 3.490320e-02 +Nonblocking Node Aware: 4.953848e-03 +Nonblocking Locality Aware: 4.654802e-03 +Nonblocking Multileader Locality: 3.854058e-02 +Size 1024 +PMPI: 1.369892e-02 +Pairwise: 1.245879e-02 +NonBlocking: 7.279030e-02 +Pairwise Hierarchical: 2.326059e-01 +Pairwise Multileader: 8.275745e-02 +Pairwise Node Aware: 1.244809e-02 +Pairwise Locality Aware: 1.122327e-02 +Pairwise Multileader Locality: 8.730626e-02 +Nonblocking Hierarchical: 2.297531e-01 +Nonblocking Multileader: 6.358586e-02 +Nonblocking Node Aware: 8.784979e-03 +Nonblocking Locality Aware: 8.090093e-03 +Nonblocking Multileader Locality: 7.463845e-02 +Size 2048 +PMPI: 2.484389e-02 +Pairwise: 2.957117e-02 +NonBlocking: 3.792544e-02 +Pairwise Hierarchical: 4.739126e-01 +Pairwise Multileader: 1.535725e-01 +Pairwise Node Aware: 2.565817e-02 +Pairwise Locality Aware: 2.418593e-02 +Pairwise Multileader Locality: 1.642141e-01 +Nonblocking Hierarchical: 4.843721e-01 +Nonblocking Multileader: 1.217662e-01 +Nonblocking Node Aware: 2.174510e-02 +Nonblocking Locality Aware: 1.721208e-02 +Nonblocking Multileader Locality: 1.471297e-01 +Size 4096 +PMPI: 5.753910e-02 +Pairwise: 4.974564e-02 +NonBlocking: 8.148725e-02 +Pairwise Hierarchical: 1.038573e+00 +Pairwise Multileader: 2.833494e-01 +Pairwise Node Aware: 4.675990e-02 +Pairwise Locality Aware: 4.984551e-02 +Pairwise Multileader Locality: 3.166813e-01 +Nonblocking Hierarchical: 9.960283e-01 +Nonblocking Multileader: 2.362164e-01 +Nonblocking Node Aware: 3.977094e-02 +Nonblocking Locality Aware: 3.719818e-02 +Nonblocking Multileader Locality: 2.910103e-01 +Size 8192 +PMPI: 9.602378e-02 +Pairwise: 9.917824e-02 +NonBlocking: 8.639222e-02 +Pairwise Hierarchical: 2.162051e+00 +Pairwise Multileader: 5.684229e-01 +Pairwise Node Aware: 9.914189e-02 +Pairwise Locality Aware: 9.931753e-02 +Pairwise Multileader Locality: 6.709789e-01 +Nonblocking Hierarchical: 2.197451e+00 +Nonblocking Multileader: 5.076333e-01 +Nonblocking Node Aware: 7.942497e-02 +Nonblocking Locality Aware: 8.364036e-02 +Nonblocking Multileader Locality: 6.336354e-01 +Size 16384 +PMPI: 1.716184e-01 +Pairwise: 2.067611e-01 +NonBlocking: 1.174262e-01 +Test 2 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 1.058822e-04 +Pairwise: 1.126195e-03 +NonBlocking: 3.097473e-04 +Pairwise Hierarchical: 3.400916e-04 +Pairwise Multileader: 1.768722e-04 +Pairwise Node Aware: 2.034751e-04 +Pairwise Locality Aware: 2.005601e-04 +Pairwise Multileader Locality: 1.340156e-04 +Nonblocking Hierarchical: 3.589191e-04 +Nonblocking Multileader: 9.165982e-05 +Nonblocking Node Aware: 1.974102e-04 +Nonblocking Locality Aware: 1.060195e-04 +Nonblocking Multileader Locality: 1.124563e-04 +Size 2 +PMPI: 1.622011e-04 +Pairwise: 1.217702e-03 +NonBlocking: 3.554170e-04 +Pairwise Hierarchical: 6.163616e-04 +Pairwise Multileader: 1.831696e-04 +Pairwise Node Aware: 2.635660e-04 +Pairwise Locality Aware: 1.667189e-04 +Pairwise Multileader Locality: 1.749932e-04 +Nonblocking Hierarchical: 5.652717e-04 +Nonblocking Multileader: 1.202482e-04 +Nonblocking Node Aware: 1.995835e-04 +Nonblocking Locality Aware: 1.358455e-04 +Nonblocking Multileader Locality: 1.382000e-04 +Size 4 +PMPI: 1.371588e-04 +Pairwise: 2.406750e-03 +NonBlocking: 9.188308e-02 +Pairwise Hierarchical: 9.835459e-04 +Pairwise Multileader: 3.622340e-04 +Pairwise Node Aware: 4.234511e-04 +Pairwise Locality Aware: 3.191302e-04 +Pairwise Multileader Locality: 3.211066e-04 +Nonblocking Hierarchical: 8.746274e-04 +Nonblocking Multileader: 2.239667e-04 +Nonblocking Node Aware: 1.977012e-04 +Nonblocking Locality Aware: 1.515248e-04 +Nonblocking Multileader Locality: 2.591128e-04 +Size 8 +PMPI: 2.655447e-04 +Pairwise: 3.272192e-03 +NonBlocking: 6.214248e-02 +Pairwise Hierarchical: 1.810551e-03 +Pairwise Multileader: 5.599844e-04 +Pairwise Node Aware: 4.124432e-04 +Pairwise Locality Aware: 3.351320e-04 +Pairwise Multileader Locality: 7.012034e-04 +Nonblocking Hierarchical: 1.605155e-03 +Nonblocking Multileader: 3.976604e-04 +Nonblocking Node Aware: 2.247758e-04 +Nonblocking Locality Aware: 2.847745e-04 +Nonblocking Multileader Locality: 5.604180e-04 +Size 16 +PMPI: 4.397530e-04 +Pairwise: 1.996288e-03 +NonBlocking: 8.693004e-02 +Pairwise Hierarchical: 3.408684e-03 +Pairwise Multileader: 1.004174e-03 +Pairwise Node Aware: 5.108406e-04 +Pairwise Locality Aware: 3.229315e-04 +Pairwise Multileader Locality: 1.257503e-03 +Nonblocking Hierarchical: 3.077274e-03 +Nonblocking Multileader: 7.663468e-04 +Nonblocking Node Aware: 3.224193e-04 +Nonblocking Locality Aware: 2.493670e-04 +Nonblocking Multileader Locality: 1.049669e-03 +Size 32 +PMPI: 7.998164e-04 +Pairwise: 2.061595e-03 +NonBlocking: 9.927731e-02 +Pairwise Hierarchical: 7.540973e-03 +Pairwise Multileader: 2.945756e-03 +Pairwise Node Aware: 7.307517e-04 +Pairwise Locality Aware: 4.397853e-04 +Pairwise Multileader Locality: 2.656244e-03 +Nonblocking Hierarchical: 7.000586e-03 +Nonblocking Multileader: 2.243390e-03 +Nonblocking Node Aware: 5.149816e-04 +Nonblocking Locality Aware: 4.920431e-04 +Nonblocking Multileader Locality: 2.359581e-03 +Size 64 +PMPI: 1.879330e-03 +Pairwise: 4.369386e-03 +NonBlocking: 9.943177e-02 +Pairwise Hierarchical: 1.675522e-02 +Pairwise Multileader: 5.307471e-03 +Pairwise Node Aware: 1.111945e-03 +Pairwise Locality Aware: 8.099095e-04 +Pairwise Multileader Locality: 5.581805e-03 +Nonblocking Hierarchical: 1.500709e-02 +Nonblocking Multileader: 4.130317e-03 +Nonblocking Node Aware: 7.626193e-04 +Nonblocking Locality Aware: 8.587310e-04 +Nonblocking Multileader Locality: 4.577008e-03 +Size 128 +PMPI: 2.388559e-03 +Pairwise: 3.397223e-03 +NonBlocking: 9.257630e-02 +Pairwise Hierarchical: 3.567805e-02 +Pairwise Multileader: 9.882524e-03 +Pairwise Node Aware: 2.288413e-03 +Pairwise Locality Aware: 2.406934e-03 +Pairwise Multileader Locality: 9.416231e-03 +Nonblocking Hierarchical: 3.191376e-02 +Nonblocking Multileader: 7.351081e-03 +Nonblocking Node Aware: 1.442249e-03 +Nonblocking Locality Aware: 1.472347e-03 +Nonblocking Multileader Locality: 8.131356e-03 +Size 256 +PMPI: 4.865542e-03 +Pairwise: 3.937357e-03 +NonBlocking: 9.393934e-02 +Pairwise Hierarchical: 6.673818e-02 +Pairwise Multileader: 2.132165e-02 +Pairwise Node Aware: 2.940452e-03 +Pairwise Locality Aware: 3.007677e-03 +Pairwise Multileader Locality: 1.880686e-02 +Nonblocking Hierarchical: 6.265046e-02 +Nonblocking Multileader: 1.487740e-02 +Nonblocking Node Aware: 1.994720e-03 +Nonblocking Locality Aware: 2.663637e-03 +Nonblocking Multileader Locality: 1.646854e-02 +Size 512 +PMPI: 7.641527e-03 +Pairwise: 8.175925e-03 +NonBlocking: 7.267966e-02 +Pairwise Hierarchical: 1.264442e-01 +Pairwise Multileader: 4.605101e-02 +Pairwise Node Aware: 5.769060e-03 +Pairwise Locality Aware: 6.885066e-03 +Pairwise Multileader Locality: 4.769388e-02 +Nonblocking Hierarchical: 1.304506e-01 +Nonblocking Multileader: 3.535839e-02 +Nonblocking Node Aware: 3.759427e-03 +Nonblocking Locality Aware: 5.421871e-03 +Nonblocking Multileader Locality: 3.829598e-02 +Size 1024 +PMPI: 1.437499e-02 +Pairwise: 1.359218e-02 +NonBlocking: 5.162803e-02 +Pairwise Hierarchical: 2.332087e-01 +Pairwise Multileader: 8.218470e-02 +Pairwise Node Aware: 1.281858e-02 +Pairwise Locality Aware: 1.263199e-02 +Pairwise Multileader Locality: 8.714831e-02 +Nonblocking Hierarchical: 2.381791e-01 +Nonblocking Multileader: 6.353237e-02 +Nonblocking Node Aware: 8.513847e-03 +Nonblocking Locality Aware: 8.815406e-03 +Nonblocking Multileader Locality: 7.649433e-02 +Size 2048 +PMPI: 2.802378e-02 +Pairwise: 2.847878e-02 +NonBlocking: 3.481653e-02 +Pairwise Hierarchical: 4.746479e-01 +Pairwise Multileader: 1.558503e-01 +Pairwise Node Aware: 2.580640e-02 +Pairwise Locality Aware: 2.505823e-02 +Pairwise Multileader Locality: 1.644289e-01 +Nonblocking Hierarchical: 5.002103e-01 +Nonblocking Multileader: 1.225119e-01 +Nonblocking Node Aware: 2.065937e-02 +Nonblocking Locality Aware: 1.869419e-02 +Nonblocking Multileader Locality: 1.471812e-01 +Size 4096 +PMPI: 5.923247e-02 +Pairwise: 5.664126e-02 +NonBlocking: 8.677384e-02 +Pairwise Hierarchical: 1.040062e+00 +Pairwise Multileader: 2.911983e-01 +Pairwise Node Aware: 4.849213e-02 +Pairwise Locality Aware: 5.216073e-02 +Pairwise Multileader Locality: 3.166826e-01 +Nonblocking Hierarchical: 1.083990e+00 +Nonblocking Multileader: 2.359153e-01 +Nonblocking Node Aware: 3.972731e-02 +Nonblocking Locality Aware: 3.782211e-02 +Nonblocking Multileader Locality: 2.937914e-01 +Size 8192 +PMPI: 9.731815e-02 +Pairwise: 1.008293e-01 +NonBlocking: 8.491514e-02 +Pairwise Hierarchical: 2.106448e+00 +Pairwise Multileader: 5.661594e-01 +Pairwise Node Aware: 9.534369e-02 +Pairwise Locality Aware: 9.983592e-02 +Pairwise Multileader Locality: 6.832882e-01 +Nonblocking Hierarchical: 2.199918e+00 +Nonblocking Multileader: 5.085322e-01 +Nonblocking Node Aware: 7.777667e-02 +Nonblocking Locality Aware: 7.656427e-02 +Nonblocking Multileader Locality: 6.372752e-01 +Size 16384 +PMPI: 1.807452e-01 +Pairwise: 1.683269e-01 +NonBlocking: 1.153990e-01 +Test 3 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 7.786184e-05 +Pairwise: 1.029906e-03 +NonBlocking: 3.124960e-04 +Pairwise Hierarchical: 3.260391e-04 +Pairwise Multileader: 1.525616e-04 +Pairwise Node Aware: 2.058934e-04 +Pairwise Locality Aware: 1.295281e-04 +Pairwise Multileader Locality: 1.207063e-04 +Nonblocking Hierarchical: 3.528699e-04 +Nonblocking Multileader: 8.828359e-05 +Nonblocking Node Aware: 1.912959e-04 +Nonblocking Locality Aware: 6.626691e-05 +Nonblocking Multileader Locality: 1.087815e-04 +Size 2 +PMPI: 1.014614e-04 +Pairwise: 9.613030e-04 +NonBlocking: 3.662840e-04 +Pairwise Hierarchical: 6.352975e-04 +Pairwise Multileader: 2.051854e-04 +Pairwise Node Aware: 2.605761e-04 +Pairwise Locality Aware: 1.744195e-04 +Pairwise Multileader Locality: 1.728367e-04 +Nonblocking Hierarchical: 5.429065e-04 +Nonblocking Multileader: 1.180895e-04 +Nonblocking Node Aware: 2.130687e-04 +Nonblocking Locality Aware: 1.053451e-04 +Nonblocking Multileader Locality: 1.378161e-04 +Size 4 +PMPI: 1.499128e-04 +Pairwise: 1.824126e-03 +NonBlocking: 7.433827e-02 +Pairwise Hierarchical: 9.362115e-04 +Pairwise Multileader: 2.982819e-04 +Pairwise Node Aware: 3.258425e-04 +Pairwise Locality Aware: 1.700193e-04 +Pairwise Multileader Locality: 6.247591e-04 +Nonblocking Hierarchical: 8.365186e-04 +Nonblocking Multileader: 2.279684e-04 +Nonblocking Node Aware: 2.340084e-04 +Nonblocking Locality Aware: 1.698856e-04 +Nonblocking Multileader Locality: 5.178244e-04 +Size 8 +PMPI: 2.479612e-04 +Pairwise: 1.939268e-03 +NonBlocking: 8.657600e-02 +Pairwise Hierarchical: 1.765066e-03 +Pairwise Multileader: 5.268521e-04 +Pairwise Node Aware: 4.169207e-04 +Pairwise Locality Aware: 2.466300e-04 +Pairwise Multileader Locality: 1.327410e-03 +Nonblocking Hierarchical: 1.610861e-03 +Nonblocking Multileader: 3.888773e-04 +Nonblocking Node Aware: 2.071397e-04 +Nonblocking Locality Aware: 2.541378e-04 +Nonblocking Multileader Locality: 1.141448e-03 +Size 16 +PMPI: 4.734182e-04 +Pairwise: 2.467576e-03 +NonBlocking: 8.661958e-02 +Pairwise Hierarchical: 3.383273e-03 +Pairwise Multileader: 1.693406e-03 +Pairwise Node Aware: 4.978253e-04 +Pairwise Locality Aware: 3.129968e-04 +Pairwise Multileader Locality: 1.766168e-03 +Nonblocking Hierarchical: 3.125000e-03 +Nonblocking Multileader: 7.927201e-04 +Nonblocking Node Aware: 3.058084e-04 +Nonblocking Locality Aware: 2.784326e-04 +Nonblocking Multileader Locality: 1.504607e-03 +Size 32 +PMPI: 7.768946e-04 +Pairwise: 1.933660e-03 +NonBlocking: 8.684402e-02 +Pairwise Hierarchical: 7.672329e-03 +Pairwise Multileader: 3.957678e-03 +Pairwise Node Aware: 7.351861e-04 +Pairwise Locality Aware: 5.239417e-04 +Pairwise Multileader Locality: 3.475324e-03 +Nonblocking Hierarchical: 7.090831e-03 +Nonblocking Multileader: 3.030270e-03 +Nonblocking Node Aware: 4.951657e-04 +Nonblocking Locality Aware: 4.726221e-04 +Nonblocking Multileader Locality: 2.961645e-03 +Size 64 +PMPI: 1.639159e-03 +Pairwise: 2.489258e-03 +NonBlocking: 8.702462e-02 +Pairwise Hierarchical: 1.708877e-02 +Pairwise Multileader: 6.054809e-03 +Pairwise Node Aware: 1.144346e-03 +Pairwise Locality Aware: 7.996716e-04 +Pairwise Multileader Locality: 5.487500e-03 +Nonblocking Hierarchical: 1.556596e-02 +Nonblocking Multileader: 4.728319e-03 +Nonblocking Node Aware: 7.001832e-04 +Nonblocking Locality Aware: 7.061839e-04 +Nonblocking Multileader Locality: 5.389078e-03 +Size 128 +PMPI: 2.476327e-03 +Pairwise: 4.029781e-03 +NonBlocking: 8.735266e-02 +Pairwise Hierarchical: 3.484630e-02 +Pairwise Multileader: 9.600051e-03 +Pairwise Node Aware: 2.122240e-03 +Pairwise Locality Aware: 1.680946e-03 +Pairwise Multileader Locality: 9.655545e-03 +Nonblocking Hierarchical: 3.177613e-02 +Nonblocking Multileader: 7.304897e-03 +Nonblocking Node Aware: 1.506510e-03 +Nonblocking Locality Aware: 1.700303e-03 +Nonblocking Multileader Locality: 8.680520e-03 +Size 256 +PMPI: 5.031912e-03 +Pairwise: 5.603652e-03 +NonBlocking: 9.382585e-02 +Pairwise Hierarchical: 6.710134e-02 +Pairwise Multileader: 2.163301e-02 +Pairwise Node Aware: 2.877868e-03 +Pairwise Locality Aware: 3.361612e-03 +Pairwise Multileader Locality: 1.887944e-02 +Nonblocking Hierarchical: 6.160421e-02 +Nonblocking Multileader: 1.490778e-02 +Nonblocking Node Aware: 1.935098e-03 +Nonblocking Locality Aware: 2.655470e-03 +Nonblocking Multileader Locality: 1.638638e-02 +Size 512 +PMPI: 7.774173e-03 +Pairwise: 7.809164e-03 +NonBlocking: 7.126848e-02 +Pairwise Hierarchical: 1.263003e-01 +Pairwise Multileader: 4.693187e-02 +Pairwise Node Aware: 5.892935e-03 +Pairwise Locality Aware: 6.532255e-03 +Pairwise Multileader Locality: 4.876609e-02 +Nonblocking Hierarchical: 1.313201e-01 +Nonblocking Multileader: 3.503179e-02 +Nonblocking Node Aware: 4.036785e-03 +Nonblocking Locality Aware: 5.578725e-03 +Nonblocking Multileader Locality: 3.890977e-02 +Size 1024 +PMPI: 1.451725e-02 +Pairwise: 1.392834e-02 +NonBlocking: 5.966355e-02 +Pairwise Hierarchical: 2.340271e-01 +Pairwise Multileader: 8.261042e-02 +Pairwise Node Aware: 1.296287e-02 +Pairwise Locality Aware: 1.074224e-02 +Pairwise Multileader Locality: 8.751619e-02 +Nonblocking Hierarchical: 2.380165e-01 +Nonblocking Multileader: 6.472334e-02 +Nonblocking Node Aware: 8.664282e-03 +Nonblocking Locality Aware: 8.280338e-03 +Nonblocking Multileader Locality: 7.732465e-02 +Size 2048 +PMPI: 2.648441e-02 +Pairwise: 2.708955e-02 +NonBlocking: 3.507600e-02 +Pairwise Hierarchical: 4.766802e-01 +Pairwise Multileader: 1.542896e-01 +Pairwise Node Aware: 2.693775e-02 +Pairwise Locality Aware: 2.597193e-02 +Pairwise Multileader Locality: 1.651983e-01 +Nonblocking Hierarchical: 4.927096e-01 +Nonblocking Multileader: 1.239817e-01 +Nonblocking Node Aware: 2.085531e-02 +Nonblocking Locality Aware: 1.731650e-02 +Nonblocking Multileader Locality: 1.494188e-01 +Size 4096 +PMPI: 5.873944e-02 +Pairwise: 5.155365e-02 +NonBlocking: 8.397752e-02 +Pairwise Hierarchical: 1.030939e+00 +Pairwise Multileader: 2.898781e-01 +Pairwise Node Aware: 4.706024e-02 +Pairwise Locality Aware: 5.211074e-02 +Pairwise Multileader Locality: 3.160954e-01 +Nonblocking Hierarchical: 1.078890e+00 +Nonblocking Multileader: 2.385275e-01 +Nonblocking Node Aware: 3.926589e-02 +Nonblocking Locality Aware: 3.746331e-02 +Nonblocking Multileader Locality: 2.965298e-01 +Size 8192 +PMPI: 9.660313e-02 +Pairwise: 1.052549e-01 +NonBlocking: 8.702203e-02 +Pairwise Hierarchical: 2.110725e+00 +Pairwise Multileader: 5.648508e-01 +Pairwise Node Aware: 1.058279e-01 +Pairwise Locality Aware: 1.022917e-01 +Pairwise Multileader Locality: 6.836261e-01 +Nonblocking Hierarchical: 2.137404e+00 +Nonblocking Multileader: 5.270686e-01 +Nonblocking Node Aware: 8.103849e-02 +Nonblocking Locality Aware: 8.420376e-02 +Nonblocking Multileader Locality: 6.313810e-01 +Size 16384 +PMPI: 1.767621e-01 +Pairwise: 1.684311e-01 +NonBlocking: 1.154245e-01 diff --git a/runscripts/04_28_25/alltoall_N8.2361047.err b/runscripts/04_28_25/alltoall_N8.2361047.err new file mode 100644 index 000000000..502d7ddc7 --- /dev/null +++ b/runscripts/04_28_25/alltoall_N8.2361047.err @@ -0,0 +1,200 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2361047.0 ON dane254 CANCELLED AT 2025-04-28T16:00:47 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane268: tasks 100-199: Killed +srun: error: dane1526: tasks 700-799: Killed +srun: error: dane935: tasks 400-499: Killed +srun: error: dane986: tasks 500-599: Killed +srun: error: dane1118: tasks 600-699: Killed +srun: error: dane643: tasks 300-399: Killed +srun: error: dane626: tasks 200-299: Killed +srun: error: dane254: tasks 0-99: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155502b09010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155502b09010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2361047.1 ON dane254 CANCELLED AT 2025-04-28T16:05:05 *** +slurmstepd: error: *** STEP 2361047.1 ON dane254 CANCELLED AT 2025-04-28T16:05:05 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane268: tasks 100-199: Killed +srun: error: dane1526: tasks 700-799: Killed +srun: error: dane986: tasks 500-599: Killed +srun: error: dane935: tasks 400-499: Killed +srun: error: dane1118: tasks 600-699: Killed +srun: error: dane643: tasks 300-399: Killed +srun: error: dane626: tasks 200-299: Killed +srun: error: dane254: tasks 0-99: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2361047.2 ON dane254 CANCELLED AT 2025-04-28T16:09:22 *** +slurmstepd: error: *** STEP 2361047.2 ON dane254 CANCELLED AT 2025-04-28T16:09:22 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane268: tasks 100-199: Killed +srun: error: dane1526: tasks 700-799: Killed +srun: error: dane935: tasks 400-499: Killed +srun: error: dane986: tasks 500-599: Killed +srun: error: dane1118: tasks 600-699: Killed +srun: error: dane643: tasks 300-399: Killed +srun: error: dane626: tasks 200-299: Killed +srun: error: dane254: tasks 0-99: Killed diff --git a/runscripts/04_28_25/alltoall_N8.2361047.out b/runscripts/04_28_25/alltoall_N8.2361047.out new file mode 100644 index 000000000..7527d946e --- /dev/null +++ b/runscripts/04_28_25/alltoall_N8.2361047.out @@ -0,0 +1,561 @@ +Test 1 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.847780e-04 +Pairwise: 2.409100e-03 +NonBlocking: 9.840853e-04 +Pairwise Hierarchical: 6.315059e-04 +Pairwise Multileader: 3.494464e-04 +Pairwise Node Aware: 3.200259e-04 +Pairwise Locality Aware: 3.609715e-04 +Pairwise Multileader Locality: 2.723611e-04 +Nonblocking Hierarchical: 5.113478e-04 +Nonblocking Multileader: 1.568162e-04 +Nonblocking Node Aware: 2.260847e-04 +Nonblocking Locality Aware: 1.961804e-04 +Nonblocking Multileader Locality: 2.015785e-04 +Size 2 +PMPI: 2.408192e-04 +Pairwise: 2.908096e-03 +NonBlocking: 1.001758e-03 +Pairwise Hierarchical: 1.266290e-03 +Pairwise Multileader: 5.273920e-04 +Pairwise Node Aware: 3.542761e-04 +Pairwise Locality Aware: 3.065383e-04 +Pairwise Multileader Locality: 4.305709e-04 +Nonblocking Hierarchical: 1.079617e-03 +Nonblocking Multileader: 2.371151e-04 +Nonblocking Node Aware: 2.274144e-04 +Nonblocking Locality Aware: 2.374643e-04 +Nonblocking Multileader Locality: 2.719703e-04 +Size 4 +PMPI: 3.764731e-04 +Pairwise: 6.395286e-03 +NonBlocking: 1.610062e-01 +Pairwise Hierarchical: 2.009991e-03 +Pairwise Multileader: 7.288504e-04 +Pairwise Node Aware: 5.117556e-04 +Pairwise Locality Aware: 4.443931e-04 +Pairwise Multileader Locality: 6.581992e-04 +Nonblocking Hierarchical: 1.701164e-03 +Nonblocking Multileader: 4.350095e-04 +Nonblocking Node Aware: 3.092698e-04 +Nonblocking Locality Aware: 2.641494e-04 +Nonblocking Multileader Locality: 5.320284e-04 +Size 8 +PMPI: 6.608875e-04 +Pairwise: 5.208172e-03 +NonBlocking: 1.156936e-01 +Pairwise Hierarchical: 3.800966e-03 +Pairwise Multileader: 1.223704e-03 +Pairwise Node Aware: 6.437360e-04 +Pairwise Locality Aware: 4.248324e-04 +Pairwise Multileader Locality: 1.614748e-03 +Nonblocking Hierarchical: 3.321387e-03 +Nonblocking Multileader: 8.238919e-04 +Nonblocking Node Aware: 4.518792e-04 +Nonblocking Locality Aware: 3.758651e-04 +Nonblocking Multileader Locality: 1.245577e-03 +Size 16 +PMPI: 1.092485e-03 +Pairwise: 7.691116e-03 +NonBlocking: 1.153798e-01 +Pairwise Hierarchical: 8.424457e-03 +Pairwise Multileader: 2.446047e-03 +Pairwise Node Aware: 7.926419e-04 +Pairwise Locality Aware: 8.804958e-04 +Pairwise Multileader Locality: 2.922575e-03 +Nonblocking Hierarchical: 7.534396e-03 +Nonblocking Multileader: 1.870973e-03 +Nonblocking Node Aware: 5.766064e-04 +Nonblocking Locality Aware: 6.090077e-04 +Nonblocking Multileader Locality: 2.408274e-03 +Size 32 +PMPI: 2.483033e-03 +Pairwise: 4.772402e-03 +NonBlocking: 1.159256e-01 +Pairwise Hierarchical: 1.828898e-02 +Pairwise Multileader: 6.017809e-03 +Pairwise Node Aware: 1.286702e-03 +Pairwise Locality Aware: 1.003713e-03 +Pairwise Multileader Locality: 5.715319e-03 +Nonblocking Hierarchical: 1.529975e-02 +Nonblocking Multileader: 4.431697e-03 +Nonblocking Node Aware: 1.125188e-03 +Nonblocking Locality Aware: 1.085134e-03 +Nonblocking Multileader Locality: 4.551993e-03 +Size 64 +PMPI: 3.705489e-03 +Pairwise: 8.186247e-03 +NonBlocking: 1.393169e-01 +Pairwise Hierarchical: 3.831604e-02 +Pairwise Multileader: 1.245295e-02 +Pairwise Node Aware: 2.511468e-03 +Pairwise Locality Aware: 2.384890e-03 +Pairwise Multileader Locality: 1.035662e-02 +Nonblocking Hierarchical: 3.203332e-02 +Nonblocking Multileader: 8.099726e-03 +Nonblocking Node Aware: 1.660904e-03 +Nonblocking Locality Aware: 2.133240e-03 +Nonblocking Multileader Locality: 8.461205e-03 +Size 128 +PMPI: 7.302606e-03 +Pairwise: 7.905869e-03 +NonBlocking: 1.393645e-01 +Pairwise Hierarchical: 7.602143e-02 +Pairwise Multileader: 2.397363e-02 +Pairwise Node Aware: 4.603339e-03 +Pairwise Locality Aware: 3.764021e-03 +Pairwise Multileader Locality: 2.080584e-02 +Nonblocking Hierarchical: 6.358040e-02 +Nonblocking Multileader: 1.558501e-02 +Nonblocking Node Aware: 3.461662e-03 +Nonblocking Locality Aware: 3.771680e-03 +Nonblocking Multileader Locality: 1.717781e-02 +Size 256 +PMPI: 1.376845e-02 +Pairwise: 1.081786e-02 +NonBlocking: 1.130128e-01 +Pairwise Hierarchical: 1.394271e-01 +Pairwise Multileader: 4.999369e-02 +Pairwise Node Aware: 6.539942e-03 +Pairwise Locality Aware: 7.987363e-03 +Pairwise Multileader Locality: 5.175414e-02 +Nonblocking Hierarchical: 1.288623e-01 +Nonblocking Multileader: 3.615408e-02 +Nonblocking Node Aware: 5.872614e-03 +Nonblocking Locality Aware: 5.883213e-03 +Nonblocking Multileader Locality: 3.856706e-02 +Size 512 +PMPI: 2.415248e-02 +Pairwise: 1.901554e-02 +NonBlocking: 1.423651e-01 +Pairwise Hierarchical: 2.643883e-01 +Pairwise Multileader: 9.045714e-02 +Pairwise Node Aware: 1.485410e-02 +Pairwise Locality Aware: 1.424986e-02 +Pairwise Multileader Locality: 9.787224e-02 +Nonblocking Hierarchical: 2.462171e-01 +Nonblocking Multileader: 6.622634e-02 +Nonblocking Node Aware: 1.424628e-02 +Nonblocking Locality Aware: 1.183825e-02 +Nonblocking Multileader Locality: 7.751617e-02 +Size 1024 +PMPI: 3.787926e-02 +Pairwise: 3.605185e-02 +NonBlocking: 1.128452e-01 +Pairwise Hierarchical: 5.109167e-01 +Pairwise Multileader: 1.646931e-01 +Pairwise Node Aware: 3.031466e-02 +Pairwise Locality Aware: 2.856197e-02 +Pairwise Multileader Locality: 1.776772e-01 +Nonblocking Hierarchical: 4.948196e-01 +Nonblocking Multileader: 1.234283e-01 +Nonblocking Node Aware: 2.459354e-02 +Nonblocking Locality Aware: 2.049883e-02 +Nonblocking Multileader Locality: 1.458097e-01 +Size 2048 +PMPI: 6.542698e-02 +Pairwise: 7.421615e-02 +NonBlocking: 1.054680e-01 +Pairwise Hierarchical: 1.092223e+00 +Pairwise Multileader: 3.125795e-01 +Pairwise Node Aware: 5.607085e-02 +Pairwise Locality Aware: 5.642066e-02 +Pairwise Multileader Locality: 3.459569e-01 +Nonblocking Hierarchical: 1.034940e+00 +Nonblocking Multileader: 2.449720e-01 +Nonblocking Node Aware: 4.594494e-02 +Nonblocking Locality Aware: 4.087832e-02 +Nonblocking Multileader Locality: 2.956089e-01 +Size 4096 +PMPI: 1.456166e-01 +Pairwise: 1.398138e-01 +NonBlocking: 1.926072e-01 +Pairwise Hierarchical: 2.306751e+00 +Pairwise Multileader: 6.128755e-01 +Pairwise Node Aware: 1.029664e-01 +Pairwise Locality Aware: 1.127262e-01 +Pairwise Multileader Locality: 6.955334e-01 +Nonblocking Hierarchical: 2.236945e+00 +Nonblocking Multileader: 5.311071e-01 +Nonblocking Node Aware: 9.012749e-02 +Nonblocking Locality Aware: 8.728576e-02 +Nonblocking Multileader Locality: 6.314723e-01 +Size 8192 +PMPI: 2.678043e-01 +Pairwise: 2.637480e-01 +NonBlocking: 2.056029e-01 +Test 2 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.670519e-04 +Pairwise: 3.185441e-03 +NonBlocking: 1.027096e-03 +Pairwise Hierarchical: 6.071785e-04 +Pairwise Multileader: 3.450735e-04 +Pairwise Node Aware: 3.044142e-04 +Pairwise Locality Aware: 4.217260e-04 +Pairwise Multileader Locality: 2.534113e-04 +Nonblocking Hierarchical: 5.196602e-04 +Nonblocking Multileader: 1.658895e-04 +Nonblocking Node Aware: 2.276333e-04 +Nonblocking Locality Aware: 1.748510e-04 +Nonblocking Multileader Locality: 2.022715e-04 +Size 2 +PMPI: 3.249729e-04 +Pairwise: 4.047932e-03 +NonBlocking: 1.500663e-03 +Pairwise Hierarchical: 1.389405e-03 +Pairwise Multileader: 5.677010e-04 +Pairwise Node Aware: 3.370053e-04 +Pairwise Locality Aware: 3.470708e-04 +Pairwise Multileader Locality: 4.592930e-04 +Nonblocking Hierarchical: 1.074249e-03 +Nonblocking Multileader: 2.578238e-04 +Nonblocking Node Aware: 2.280507e-04 +Nonblocking Locality Aware: 2.487221e-04 +Nonblocking Multileader Locality: 2.801947e-04 +Size 4 +PMPI: 3.860438e-04 +Pairwise: 7.114641e-03 +NonBlocking: 1.615267e-01 +Pairwise Hierarchical: 2.033172e-03 +Pairwise Multileader: 8.489422e-04 +Pairwise Node Aware: 6.736102e-04 +Pairwise Locality Aware: 6.551086e-04 +Pairwise Multileader Locality: 6.452875e-04 +Nonblocking Hierarchical: 1.789763e-03 +Nonblocking Multileader: 4.344665e-04 +Nonblocking Node Aware: 3.238922e-04 +Nonblocking Locality Aware: 3.378911e-04 +Nonblocking Multileader Locality: 9.267766e-04 +Size 8 +PMPI: 6.657306e-04 +Pairwise: 7.707745e-03 +NonBlocking: 1.387966e-01 +Pairwise Hierarchical: 3.879511e-03 +Pairwise Multileader: 1.183880e-03 +Pairwise Node Aware: 7.227083e-04 +Pairwise Locality Aware: 8.066601e-04 +Pairwise Multileader Locality: 1.776504e-03 +Nonblocking Hierarchical: 3.473170e-03 +Nonblocking Multileader: 8.146914e-04 +Nonblocking Node Aware: 4.712264e-04 +Nonblocking Locality Aware: 4.920324e-04 +Nonblocking Multileader Locality: 1.165225e-03 +Size 16 +PMPI: 1.311132e-03 +Pairwise: 9.326038e-03 +NonBlocking: 1.613490e-01 +Pairwise Hierarchical: 8.536220e-03 +Pairwise Multileader: 2.539642e-03 +Pairwise Node Aware: 8.632919e-04 +Pairwise Locality Aware: 7.747243e-04 +Pairwise Multileader Locality: 3.016721e-03 +Nonblocking Hierarchical: 7.558767e-03 +Nonblocking Multileader: 1.922854e-03 +Nonblocking Node Aware: 7.606462e-04 +Nonblocking Locality Aware: 8.100781e-04 +Nonblocking Multileader Locality: 2.370951e-03 +Size 32 +PMPI: 2.333425e-03 +Pairwise: 8.394076e-03 +NonBlocking: 1.617701e-01 +Pairwise Hierarchical: 1.863662e-02 +Pairwise Multileader: 6.116891e-03 +Pairwise Node Aware: 1.374942e-03 +Pairwise Locality Aware: 1.094109e-03 +Pairwise Multileader Locality: 5.619304e-03 +Nonblocking Hierarchical: 1.575592e-02 +Nonblocking Multileader: 4.351843e-03 +Nonblocking Node Aware: 1.099109e-03 +Nonblocking Locality Aware: 8.784447e-04 +Nonblocking Multileader Locality: 4.547772e-03 +Size 64 +PMPI: 3.964655e-03 +Pairwise: 7.001387e-03 +NonBlocking: 1.159456e-01 +Pairwise Hierarchical: 3.820251e-02 +Pairwise Multileader: 1.243661e-02 +Pairwise Node Aware: 2.594839e-03 +Pairwise Locality Aware: 2.343606e-03 +Pairwise Multileader Locality: 1.074188e-02 +Nonblocking Hierarchical: 3.261958e-02 +Nonblocking Multileader: 7.958841e-03 +Nonblocking Node Aware: 1.683563e-03 +Nonblocking Locality Aware: 2.131670e-03 +Nonblocking Multileader Locality: 8.570535e-03 +Size 128 +PMPI: 7.263954e-03 +Pairwise: 1.032292e-02 +NonBlocking: 1.014082e-01 +Pairwise Hierarchical: 7.696099e-02 +Pairwise Multileader: 2.471063e-02 +Pairwise Node Aware: 5.000889e-03 +Pairwise Locality Aware: 4.544677e-03 +Pairwise Multileader Locality: 2.062280e-02 +Nonblocking Hierarchical: 6.368525e-02 +Nonblocking Multileader: 1.830049e-02 +Nonblocking Node Aware: 3.437446e-03 +Nonblocking Locality Aware: 4.842178e-03 +Nonblocking Multileader Locality: 1.686698e-02 +Size 256 +PMPI: 1.359554e-02 +Pairwise: 1.069641e-02 +NonBlocking: 1.130430e-01 +Pairwise Hierarchical: 1.393251e-01 +Pairwise Multileader: 4.998358e-02 +Pairwise Node Aware: 6.865756e-03 +Pairwise Locality Aware: 8.033484e-03 +Pairwise Multileader Locality: 5.249277e-02 +Nonblocking Hierarchical: 1.294165e-01 +Nonblocking Multileader: 3.647646e-02 +Nonblocking Node Aware: 5.858634e-03 +Nonblocking Locality Aware: 7.340635e-03 +Nonblocking Multileader Locality: 3.924048e-02 +Size 512 +PMPI: 3.176540e-02 +Pairwise: 2.089225e-02 +NonBlocking: 1.431789e-01 +Pairwise Hierarchical: 2.669277e-01 +Pairwise Multileader: 9.119342e-02 +Pairwise Node Aware: 1.495166e-02 +Pairwise Locality Aware: 2.359066e-02 +Pairwise Multileader Locality: 1.031113e-01 +Nonblocking Hierarchical: 2.498533e-01 +Nonblocking Multileader: 7.127302e-02 +Nonblocking Node Aware: 2.143069e-02 +Nonblocking Locality Aware: 4.049576e-02 +Nonblocking Multileader Locality: 7.899191e-02 +Size 1024 +PMPI: 5.937739e-02 +Pairwise: 3.770226e-02 +NonBlocking: 1.110556e-01 +Pairwise Hierarchical: 5.160729e-01 +Pairwise Multileader: 1.672797e-01 +Pairwise Node Aware: 3.102758e-02 +Pairwise Locality Aware: 3.151029e-02 +Pairwise Multileader Locality: 1.837832e-01 +Nonblocking Hierarchical: 5.162305e-01 +Nonblocking Multileader: 1.247444e-01 +Nonblocking Node Aware: 2.492921e-02 +Nonblocking Locality Aware: 2.082117e-02 +Nonblocking Multileader Locality: 1.476957e-01 +Size 2048 +PMPI: 6.973913e-02 +Pairwise: 7.077411e-02 +NonBlocking: 9.682921e-02 +Pairwise Hierarchical: 1.113033e+00 +Pairwise Multileader: 3.214945e-01 +Pairwise Node Aware: 5.729303e-02 +Pairwise Locality Aware: 5.523635e-02 +Pairwise Multileader Locality: 3.485949e-01 +Nonblocking Hierarchical: 1.031637e+00 +Nonblocking Multileader: 2.421658e-01 +Nonblocking Node Aware: 4.503330e-02 +Nonblocking Locality Aware: 4.160043e-02 +Nonblocking Multileader Locality: 2.995346e-01 +Size 4096 +PMPI: 1.677941e-01 +Pairwise: 1.289476e-01 +NonBlocking: 1.905276e-01 +Pairwise Hierarchical: 2.262085e+00 +Pairwise Multileader: 6.180836e-01 +Pairwise Node Aware: 1.049830e-01 +Pairwise Locality Aware: 1.111484e-01 +Pairwise Multileader Locality: 7.021867e-01 +Nonblocking Hierarchical: 2.086369e+00 +Nonblocking Multileader: 5.309747e-01 +Nonblocking Node Aware: 9.070319e-02 +Nonblocking Locality Aware: 8.982666e-02 +Nonblocking Multileader Locality: 6.225382e-01 +Size 8192 +PMPI: 2.794204e-01 +Pairwise: 2.637798e-01 +NonBlocking: 2.075134e-01 +Test 3 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 2.235601e-04 +Pairwise: 3.372144e-03 +NonBlocking: 1.358729e-03 +Pairwise Hierarchical: 6.350218e-04 +Pairwise Multileader: 3.406929e-04 +Pairwise Node Aware: 3.128810e-04 +Pairwise Locality Aware: 3.594784e-04 +Pairwise Multileader Locality: 2.671983e-04 +Nonblocking Hierarchical: 5.184172e-04 +Nonblocking Multileader: 1.616113e-04 +Nonblocking Node Aware: 2.278712e-04 +Nonblocking Locality Aware: 1.409334e-04 +Nonblocking Multileader Locality: 1.966543e-04 +Size 2 +PMPI: 3.369922e-04 +Pairwise: 5.133371e-03 +NonBlocking: 1.178787e-03 +Pairwise Hierarchical: 1.349374e-03 +Pairwise Multileader: 5.412381e-04 +Pairwise Node Aware: 3.357638e-04 +Pairwise Locality Aware: 3.653229e-04 +Pairwise Multileader Locality: 4.500884e-04 +Nonblocking Hierarchical: 1.087168e-03 +Nonblocking Multileader: 2.343747e-04 +Nonblocking Node Aware: 2.251697e-04 +Nonblocking Locality Aware: 2.431486e-04 +Nonblocking Multileader Locality: 2.718674e-04 +Size 4 +PMPI: 4.321816e-04 +Pairwise: 6.868810e-03 +NonBlocking: 1.609879e-01 +Pairwise Hierarchical: 2.068111e-03 +Pairwise Multileader: 7.400305e-04 +Pairwise Node Aware: 5.062287e-04 +Pairwise Locality Aware: 7.906349e-04 +Pairwise Multileader Locality: 6.324525e-04 +Nonblocking Hierarchical: 1.748320e-03 +Nonblocking Multileader: 4.374237e-04 +Nonblocking Node Aware: 3.109828e-04 +Nonblocking Locality Aware: 2.903090e-04 +Nonblocking Multileader Locality: 5.150527e-04 +Size 8 +PMPI: 6.464067e-04 +Pairwise: 7.588703e-03 +NonBlocking: 1.613709e-01 +Pairwise Hierarchical: 3.857115e-03 +Pairwise Multileader: 1.384285e-03 +Pairwise Node Aware: 6.027599e-04 +Pairwise Locality Aware: 5.814871e-04 +Pairwise Multileader Locality: 1.663954e-03 +Nonblocking Hierarchical: 3.463247e-03 +Nonblocking Multileader: 9.587975e-04 +Nonblocking Node Aware: 5.766569e-04 +Nonblocking Locality Aware: 5.860849e-04 +Nonblocking Multileader Locality: 1.164452e-03 +Size 16 +PMPI: 1.195457e-03 +Pairwise: 8.337727e-03 +NonBlocking: 1.156121e-01 +Pairwise Hierarchical: 8.532271e-03 +Pairwise Multileader: 2.553457e-03 +Pairwise Node Aware: 8.132202e-04 +Pairwise Locality Aware: 6.258255e-04 +Pairwise Multileader Locality: 2.952371e-03 +Nonblocking Hierarchical: 7.546165e-03 +Nonblocking Multileader: 1.862110e-03 +Nonblocking Node Aware: 5.961438e-04 +Nonblocking Locality Aware: 4.897437e-04 +Nonblocking Multileader Locality: 2.402794e-03 +Size 32 +PMPI: 2.243609e-03 +Pairwise: 7.326228e-03 +NonBlocking: 1.617722e-01 +Pairwise Hierarchical: 1.857671e-02 +Pairwise Multileader: 6.316310e-03 +Pairwise Node Aware: 1.526811e-03 +Pairwise Locality Aware: 1.268850e-03 +Pairwise Multileader Locality: 5.606962e-03 +Nonblocking Hierarchical: 1.572308e-02 +Nonblocking Multileader: 4.392090e-03 +Nonblocking Node Aware: 1.101129e-03 +Nonblocking Locality Aware: 8.903787e-04 +Nonblocking Multileader Locality: 4.565232e-03 +Size 64 +PMPI: 4.109957e-03 +Pairwise: 9.069228e-03 +NonBlocking: 1.166582e-01 +Pairwise Hierarchical: 3.857086e-02 +Pairwise Multileader: 1.252541e-02 +Pairwise Node Aware: 2.611681e-03 +Pairwise Locality Aware: 1.789491e-03 +Pairwise Multileader Locality: 1.018401e-02 +Nonblocking Hierarchical: 3.229497e-02 +Nonblocking Multileader: 8.094886e-03 +Nonblocking Node Aware: 1.756383e-03 +Nonblocking Locality Aware: 2.148325e-03 +Nonblocking Multileader Locality: 8.440486e-03 +Size 128 +PMPI: 7.654764e-03 +Pairwise: 9.615088e-03 +NonBlocking: 1.174640e-01 +Pairwise Hierarchical: 7.627115e-02 +Pairwise Multileader: 2.384241e-02 +Pairwise Node Aware: 4.882698e-03 +Pairwise Locality Aware: 3.600137e-03 +Pairwise Multileader Locality: 2.063456e-02 +Nonblocking Hierarchical: 6.349325e-02 +Nonblocking Multileader: 1.567557e-02 +Nonblocking Node Aware: 3.453122e-03 +Nonblocking Locality Aware: 4.730939e-03 +Nonblocking Multileader Locality: 1.697951e-02 +Size 256 +PMPI: 1.303230e-02 +Pairwise: 1.565167e-02 +NonBlocking: 1.204689e-01 +Pairwise Hierarchical: 1.404795e-01 +Pairwise Multileader: 5.212524e-02 +Pairwise Node Aware: 7.959539e-03 +Pairwise Locality Aware: 8.058954e-03 +Pairwise Multileader Locality: 5.313063e-02 +Nonblocking Hierarchical: 1.305548e-01 +Nonblocking Multileader: 3.635486e-02 +Nonblocking Node Aware: 5.999253e-03 +Nonblocking Locality Aware: 7.620202e-03 +Nonblocking Multileader Locality: 3.894612e-02 +Size 512 +PMPI: 2.812744e-02 +Pairwise: 2.217927e-02 +NonBlocking: 1.190053e-01 +Pairwise Hierarchical: 2.659945e-01 +Pairwise Multileader: 9.026120e-02 +Pairwise Node Aware: 1.568987e-02 +Pairwise Locality Aware: 1.849496e-02 +Pairwise Multileader Locality: 9.883660e-02 +Nonblocking Hierarchical: 2.412789e-01 +Nonblocking Multileader: 6.688363e-02 +Nonblocking Node Aware: 1.427083e-02 +Nonblocking Locality Aware: 1.469256e-02 +Nonblocking Multileader Locality: 7.833993e-02 +Size 1024 +PMPI: 3.931971e-02 +Pairwise: 3.348190e-02 +NonBlocking: 1.259071e-01 +Pairwise Hierarchical: 5.073235e-01 +Pairwise Multileader: 1.647430e-01 +Pairwise Node Aware: 2.836478e-02 +Pairwise Locality Aware: 2.863005e-02 +Pairwise Multileader Locality: 1.780058e-01 +Nonblocking Hierarchical: 4.856563e-01 +Nonblocking Multileader: 1.237574e-01 +Nonblocking Node Aware: 2.489943e-02 +Nonblocking Locality Aware: 2.046283e-02 +Nonblocking Multileader Locality: 1.479246e-01 +Size 2048 +PMPI: 7.429787e-02 +Pairwise: 7.509024e-02 +NonBlocking: 1.180462e-01 +Pairwise Hierarchical: 1.129799e+00 +Pairwise Multileader: 3.157446e-01 +Pairwise Node Aware: 5.473916e-02 +Pairwise Locality Aware: 5.975663e-02 +Pairwise Multileader Locality: 3.477949e-01 +Nonblocking Hierarchical: 1.067802e+00 +Nonblocking Multileader: 2.420835e-01 +Nonblocking Node Aware: 4.558751e-02 +Nonblocking Locality Aware: 4.156190e-02 +Nonblocking Multileader Locality: 2.970451e-01 +Size 4096 +PMPI: 1.497327e-01 +Pairwise: 1.283908e-01 +NonBlocking: 1.754496e-01 +Pairwise Hierarchical: 2.233595e+00 +Pairwise Multileader: 6.125226e-01 +Pairwise Node Aware: 1.047655e-01 +Pairwise Locality Aware: 1.168447e-01 +Pairwise Multileader Locality: 7.055160e-01 +Nonblocking Hierarchical: 2.140507e+00 +Nonblocking Multileader: 5.251436e-01 +Nonblocking Node Aware: 9.071829e-02 +Nonblocking Locality Aware: 9.008356e-02 +Nonblocking Multileader Locality: 6.473633e-01 +Size 8192 +PMPI: 2.481554e-01 +Pairwise: 2.339235e-01 +NonBlocking: 2.049125e-01 diff --git a/runscripts/dane_results/alltoall_N16.2373828.err b/runscripts/dane_results/alltoall_N16.2373828.err new file mode 100644 index 000000000..8329de3ba --- /dev/null +++ b/runscripts/dane_results/alltoall_N16.2373828.err @@ -0,0 +1,264 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2373828.0 ON dane369 CANCELLED AT 2025-04-29T18:36:02 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane369: task 5: Killed +srun: error: dane896: tasks 1300-1399: Killed +srun: error: dane862: tasks 1100-1199: Killed +srun: error: dane890: tasks 1200-1299: Killed +srun: error: dane1033: tasks 1400-1499: Killed +srun: error: dane1061: tasks 1500-1599: Killed +srun: error: dane830: tasks 1000-1099: Killed +srun: error: dane825: tasks 900-999: Killed +srun: error: dane525: tasks 300-399: Killed +srun: error: dane726: tasks 700-799: Killed +srun: error: dane535: tasks 400-499: Killed +srun: error: dane775: tasks 800-899: Killed +srun: error: dane605: tasks 500-599: Killed +srun: error: dane668: tasks 600-699: Killed +srun: error: dane375: tasks 100-199: Killed +srun: error: dane504: tasks 200-299: Killed +srun: error: dane369: tasks 0-4,6-99: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2373828.1 ON dane369 CANCELLED AT 2025-04-29T18:45:55 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane896: tasks 1300-1399: Killed +srun: error: dane862: tasks 1100-1199: Killed +srun: error: dane890: tasks 1200-1299: Killed +srun: error: dane1033: tasks 1400-1499: Killed +srun: error: dane1061: tasks 1500-1599: Killed +srun: error: dane830: tasks 1000-1099: Killed +srun: error: dane825: tasks 900-999: Killed +srun: error: dane525: tasks 300-399: Killed +srun: error: dane726: tasks 700-799: Killed +srun: error: dane535: tasks 400-499: Killed +srun: error: dane775: tasks 800-899: Killed +srun: error: dane605: tasks 500-599: Killed +srun: error: dane375: tasks 100-199: Killed +srun: error: dane668: tasks 600-699: Killed +srun: error: dane504: tasks 200-299: Killed +srun: error: dane369: tasks 0-99: Killed +slurmstepd: error: *** JOB 2373828 ON dane369 CANCELLED AT 2025-04-29T18:55:07 DUE TO TIME LIMIT *** +slurmstepd: error: *** STEP 2373828.2 ON dane369 CANCELLED AT 2025-04-29T18:55:07 DUE TO TIME LIMIT *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. diff --git a/runscripts/dane_results/alltoall_N16.2373828.out b/runscripts/dane_results/alltoall_N16.2373828.out new file mode 100644 index 000000000..2fe445901 --- /dev/null +++ b/runscripts/dane_results/alltoall_N16.2373828.out @@ -0,0 +1,927 @@ +Test 1 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 3.794489e-04 +Pairwise: 6.504499e-03 +NonBlocking: 9.730093e-02 +Pairwise Hierarchical: 1.129899e-03 +Pairwise Node Aware: 4.192500e-04 +Nonblocking Hierarchical: 9.447876e-04 +Nonblocking Node Aware: 2.405383e-04 +Pairwise Multileader, 4 leaders: 7.740759e-04 +Pairwise Locality Aware, 4 leaders: 5.902961e-04 +Pairwise Multileader Locality, 4 leaders: 4.833944e-04 +Nonblocking Multileader, 4 leaders: 3.040673e-04 +Nonblocking Locality Aware, 4 leaders: 2.584775e-04 +Nonblocking Multileader Locality, 4 leaders: 3.544795e-04 +Pairwise Multileader, 10 leaders: 1.096958e-03 +Pairwise Locality Aware, 10 leaders: 1.433433e-03 +Pairwise Multileader Locality, 10 leaders: 2.535221e-04 +Nonblocking Multileader, 10 leaders: 2.420682e-04 +Nonblocking Locality Aware, 10 leaders: 3.737941e-02 +Nonblocking Multileader Locality, 10 leaders: 1.789479e-04 +Pairwise Multileader, 20 leaders: 1.966946e-03 +Pairwise Locality Aware, 20 leaders: 2.620055e-03 +Pairwise Multileader Locality, 20 leaders: 2.675920e-04 +Nonblocking Multileader, 20 leaders: 2.086540e-03 +Nonblocking Locality Aware, 20 leaders: 6.804342e-02 +Nonblocking Multileader Locality, 20 leaders: 1.374062e-04 +Size 2 +PMPI: 4.603319e-04 +Pairwise: 4.658378e-03 +NonBlocking: 5.639409e-02 +Pairwise Hierarchical: 2.594213e-03 +Pairwise Node Aware: 4.226319e-04 +Nonblocking Hierarchical: 2.158167e-03 +Nonblocking Node Aware: 3.169875e-04 +Pairwise Multileader, 4 leaders: 9.859710e-04 +Pairwise Locality Aware, 4 leaders: 5.301274e-04 +Pairwise Multileader Locality, 4 leaders: 8.602116e-04 +Nonblocking Multileader, 4 leaders: 5.298458e-04 +Nonblocking Locality Aware, 4 leaders: 2.904189e-04 +Nonblocking Multileader Locality, 4 leaders: 6.234984e-04 +Pairwise Multileader, 10 leaders: 1.400882e-03 +Pairwise Locality Aware, 10 leaders: 1.333186e-03 +Pairwise Multileader Locality, 10 leaders: 3.576020e-04 +Nonblocking Multileader, 10 leaders: 1.124575e-03 +Nonblocking Locality Aware, 10 leaders: 6.190986e-02 +Nonblocking Multileader Locality, 10 leaders: 2.583793e-04 +Pairwise Multileader, 20 leaders: 3.114344e-03 +Pairwise Locality Aware, 20 leaders: 2.006191e-03 +Pairwise Multileader Locality, 20 leaders: 3.260864e-04 +Nonblocking Multileader, 20 leaders: 3.770052e-03 +Nonblocking Locality Aware, 20 leaders: 1.149314e-01 +Nonblocking Multileader Locality, 20 leaders: 1.991059e-04 +Size 4 +PMPI: 7.753472e-04 +Pairwise: 1.270843e-02 +NonBlocking: 1.172242e-01 +Pairwise Hierarchical: 4.032885e-03 +Pairwise Node Aware: 6.232318e-04 +Nonblocking Hierarchical: 3.429688e-03 +Nonblocking Node Aware: 5.471381e-04 +Pairwise Multileader, 4 leaders: 1.370562e-03 +Pairwise Locality Aware, 4 leaders: 7.221339e-04 +Pairwise Multileader Locality, 4 leaders: 1.306801e-03 +Nonblocking Multileader, 4 leaders: 9.157774e-04 +Nonblocking Locality Aware, 4 leaders: 4.592083e-04 +Nonblocking Multileader Locality, 4 leaders: 1.065450e-03 +Pairwise Multileader, 10 leaders: 1.600975e-03 +Pairwise Locality Aware, 10 leaders: 1.568594e-03 +Pairwise Multileader Locality, 10 leaders: 5.800734e-04 +Nonblocking Multileader, 10 leaders: 1.431100e-03 +Nonblocking Locality Aware, 10 leaders: 9.205127e-02 +Nonblocking Multileader Locality, 10 leaders: 4.873458e-04 +Pairwise Multileader, 20 leaders: 2.617664e-03 +Pairwise Locality Aware, 20 leaders: 2.745192e-03 +Pairwise Multileader Locality, 20 leaders: 4.438932e-04 +Nonblocking Multileader, 20 leaders: 1.309478e-02 +Nonblocking Locality Aware, 20 leaders: 8.664615e-02 +Nonblocking Multileader Locality, 20 leaders: 3.594214e-04 +Size 8 +PMPI: 1.301582e-03 +Pairwise: 1.330010e-02 +NonBlocking: 1.394423e-01 +Pairwise Hierarchical: 9.432171e-03 +Pairwise Node Aware: 8.522373e-04 +Nonblocking Hierarchical: 7.978889e-03 +Nonblocking Node Aware: 8.085175e-04 +Pairwise Multileader, 4 leaders: 3.197559e-03 +Pairwise Locality Aware, 4 leaders: 8.662855e-04 +Pairwise Multileader Locality, 4 leaders: 3.281462e-03 +Nonblocking Multileader, 4 leaders: 2.009799e-03 +Nonblocking Locality Aware, 4 leaders: 5.342938e-04 +Nonblocking Multileader Locality, 4 leaders: 2.557685e-03 +Pairwise Multileader, 10 leaders: 1.834636e-03 +Pairwise Locality Aware, 10 leaders: 1.899096e-03 +Pairwise Multileader Locality, 10 leaders: 1.185562e-03 +Nonblocking Multileader, 10 leaders: 1.454721e-03 +Nonblocking Locality Aware, 10 leaders: 3.748731e-02 +Nonblocking Multileader Locality, 10 leaders: 9.231060e-04 +Pairwise Multileader, 20 leaders: 3.133324e-03 +Pairwise Locality Aware, 20 leaders: 2.559248e-03 +Pairwise Multileader Locality, 20 leaders: 7.180245e-04 +Nonblocking Multileader, 20 leaders: 2.301150e-02 +Nonblocking Locality Aware, 20 leaders: 1.116800e-01 +Nonblocking Multileader Locality, 20 leaders: 5.054990e-04 +Size 16 +PMPI: 2.786904e-03 +Pairwise: 1.240259e-02 +NonBlocking: 1.176610e-01 +Pairwise Hierarchical: 2.012975e-02 +Pairwise Node Aware: 1.237249e-03 +Nonblocking Hierarchical: 1.630929e-02 +Nonblocking Node Aware: 1.145786e-03 +Pairwise Multileader, 4 leaders: 4.984530e-03 +Pairwise Locality Aware, 4 leaders: 1.201736e-03 +Pairwise Multileader Locality, 4 leaders: 5.751385e-03 +Nonblocking Multileader, 4 leaders: 3.995561e-03 +Nonblocking Locality Aware, 4 leaders: 1.008395e-03 +Nonblocking Multileader Locality, 4 leaders: 4.630736e-03 +Pairwise Multileader, 10 leaders: 3.360594e-03 +Pairwise Locality Aware, 10 leaders: 1.711519e-03 +Pairwise Multileader Locality, 10 leaders: 2.206352e-03 +Nonblocking Multileader, 10 leaders: 2.238942e-03 +Nonblocking Locality Aware, 10 leaders: 8.739717e-02 +Nonblocking Multileader Locality, 10 leaders: 1.887174e-03 +Pairwise Multileader, 20 leaders: 3.379525e-03 +Pairwise Locality Aware, 20 leaders: 2.919760e-03 +Pairwise Multileader Locality, 20 leaders: 1.443105e-03 +Nonblocking Multileader, 20 leaders: 2.012190e-02 +Nonblocking Locality Aware, 20 leaders: 9.964312e-02 +Nonblocking Multileader Locality, 20 leaders: 1.266910e-03 +Size 32 +PMPI: 4.642448e-03 +Pairwise: 1.351014e-02 +NonBlocking: 1.174015e-01 +Pairwise Hierarchical: 4.185728e-02 +Pairwise Node Aware: 2.126635e-03 +Nonblocking Hierarchical: 3.330918e-02 +Nonblocking Node Aware: 1.959215e-03 +Pairwise Multileader, 4 leaders: 1.355883e-02 +Pairwise Locality Aware, 4 leaders: 2.109951e-03 +Pairwise Multileader Locality, 4 leaders: 1.090502e-02 +Nonblocking Multileader, 4 leaders: 8.560131e-03 +Nonblocking Locality Aware, 4 leaders: 2.034394e-03 +Nonblocking Multileader Locality, 4 leaders: 8.703474e-03 +Pairwise Multileader, 10 leaders: 5.733673e-03 +Pairwise Locality Aware, 10 leaders: 2.854156e-03 +Pairwise Multileader Locality, 10 leaders: 4.755022e-03 +Nonblocking Multileader, 10 leaders: 5.029582e-03 +Nonblocking Locality Aware, 10 leaders: 9.287855e-02 +Nonblocking Multileader Locality, 10 leaders: 3.833719e-03 +Pairwise Multileader, 20 leaders: 4.859372e-03 +Pairwise Locality Aware, 20 leaders: 2.900167e-03 +Pairwise Multileader Locality, 20 leaders: 2.660712e-03 +Nonblocking Multileader, 20 leaders: 1.300142e-02 +Nonblocking Locality Aware, 20 leaders: 1.160612e-01 +Nonblocking Multileader Locality, 20 leaders: 2.239605e-03 +Size 64 +PMPI: 9.106206e-03 +Pairwise: 1.127028e-02 +NonBlocking: 1.639123e-01 +Pairwise Hierarchical: 8.223226e-02 +Pairwise Node Aware: 4.250436e-03 +Nonblocking Hierarchical: 6.424488e-02 +Nonblocking Node Aware: 3.184991e-03 +Pairwise Multileader, 4 leaders: 2.597364e-02 +Pairwise Locality Aware, 4 leaders: 4.341806e-03 +Pairwise Multileader Locality, 4 leaders: 2.291162e-02 +Nonblocking Multileader, 4 leaders: 1.633084e-02 +Nonblocking Locality Aware, 4 leaders: 5.208865e-03 +Nonblocking Multileader Locality, 4 leaders: 1.748405e-02 +Pairwise Multileader, 10 leaders: 1.211540e-02 +Pairwise Locality Aware, 10 leaders: 4.729295e-03 +Pairwise Multileader Locality, 10 leaders: 1.003302e-02 +Nonblocking Multileader, 10 leaders: 7.951839e-03 +Nonblocking Locality Aware, 10 leaders: 8.880017e-02 +Nonblocking Multileader Locality, 10 leaders: 8.465270e-03 +Pairwise Multileader, 20 leaders: 8.295750e-03 +Pairwise Locality Aware, 20 leaders: 4.166582e-03 +Pairwise Multileader Locality, 20 leaders: 5.599063e-03 +Nonblocking Multileader, 20 leaders: 4.873910e-03 +Nonblocking Locality Aware, 20 leaders: 9.390838e-02 +Nonblocking Multileader Locality, 20 leaders: 4.295789e-03 +Size 128 +PMPI: 1.603781e-02 +Pairwise: 1.827540e-02 +NonBlocking: 1.134964e-01 +Pairwise Hierarchical: 1.571414e-01 +Pairwise Node Aware: 9.310894e-03 +Nonblocking Hierarchical: 1.246357e-01 +Nonblocking Node Aware: 6.802084e-03 +Pairwise Multileader, 4 leaders: 5.110590e-02 +Pairwise Locality Aware, 4 leaders: 7.593248e-03 +Pairwise Multileader Locality, 4 leaders: 5.452274e-02 +Nonblocking Multileader, 4 leaders: 3.699235e-02 +Nonblocking Locality Aware, 4 leaders: 9.556994e-03 +Nonblocking Multileader Locality, 4 leaders: 4.148698e-02 +Pairwise Multileader, 10 leaders: 2.330106e-02 +Pairwise Locality Aware, 10 leaders: 8.560640e-03 +Pairwise Multileader Locality, 10 leaders: 2.396577e-02 +Nonblocking Multileader, 10 leaders: 1.734782e-02 +Nonblocking Locality Aware, 10 leaders: 1.065017e-02 +Nonblocking Multileader Locality, 10 leaders: 1.949841e-02 +Pairwise Multileader, 20 leaders: 1.455032e-02 +Pairwise Locality Aware, 20 leaders: 9.661585e-03 +Pairwise Multileader Locality, 20 leaders: 1.163652e-02 +Nonblocking Multileader, 20 leaders: 1.159571e-02 +Nonblocking Locality Aware, 20 leaders: 1.187901e-01 +Nonblocking Multileader Locality, 20 leaders: 9.515362e-03 +Size 256 +PMPI: 3.358322e-02 +Pairwise: 2.646945e-02 +NonBlocking: 1.441643e-01 +Pairwise Hierarchical: 3.045880e-01 +Pairwise Node Aware: 1.500030e-02 +Nonblocking Hierarchical: 2.475696e-01 +Nonblocking Node Aware: 1.364742e-02 +Pairwise Multileader, 4 leaders: 9.585916e-02 +Pairwise Locality Aware, 4 leaders: 1.487893e-02 +Pairwise Multileader Locality, 4 leaders: 1.057842e-01 +Nonblocking Multileader, 4 leaders: 6.976231e-02 +Nonblocking Locality Aware, 4 leaders: 1.232968e-02 +Nonblocking Multileader Locality, 4 leaders: 8.032338e-02 +Pairwise Multileader, 10 leaders: 5.470154e-02 +Pairwise Locality Aware, 10 leaders: 1.631139e-02 +Pairwise Multileader Locality, 10 leaders: 5.146596e-02 +Nonblocking Multileader, 10 leaders: 3.685434e-02 +Nonblocking Locality Aware, 10 leaders: 2.321564e-02 +Nonblocking Multileader Locality, 10 leaders: 4.180292e-02 +Pairwise Multileader, 20 leaders: 2.938131e-02 +Pairwise Locality Aware, 20 leaders: 1.492818e-02 +Pairwise Multileader Locality, 20 leaders: 2.602605e-02 +Nonblocking Multileader, 20 leaders: 1.750589e-02 +Nonblocking Locality Aware, 20 leaders: 5.938290e-02 +Nonblocking Multileader Locality, 20 leaders: 2.072122e-02 +Size 512 +PMPI: 7.989936e-02 +Pairwise: 3.932608e-02 +NonBlocking: 1.585077e-01 +Pairwise Hierarchical: 6.107655e-01 +Pairwise Node Aware: 2.957187e-02 +Nonblocking Hierarchical: 5.434065e-01 +Nonblocking Node Aware: 2.762198e-02 +Pairwise Multileader, 4 leaders: 1.774822e-01 +Pairwise Locality Aware, 4 leaders: 2.939513e-02 +Pairwise Multileader Locality, 4 leaders: 2.007024e-01 +Nonblocking Multileader, 4 leaders: 1.313584e-01 +Nonblocking Locality Aware, 4 leaders: 2.325997e-02 +Nonblocking Multileader Locality, 4 leaders: 1.561555e-01 +Pairwise Multileader, 10 leaders: 1.051114e-01 +Pairwise Locality Aware, 10 leaders: 3.553133e-02 +Pairwise Multileader Locality, 10 leaders: 1.053564e-01 +Nonblocking Multileader, 10 leaders: 7.291320e-02 +Nonblocking Locality Aware, 10 leaders: 2.779764e-02 +Nonblocking Multileader Locality, 10 leaders: 8.664382e-02 +Pairwise Multileader, 20 leaders: 6.017251e-02 +Pairwise Locality Aware, 20 leaders: 3.463809e-02 +Pairwise Multileader Locality, 20 leaders: 5.428480e-02 +Nonblocking Multileader, 20 leaders: 3.653350e-02 +Nonblocking Locality Aware, 20 leaders: 5.895685e-02 +Nonblocking Multileader Locality, 20 leaders: 4.348569e-02 +Size 1024 +PMPI: 7.987664e-02 +Pairwise: 7.830828e-02 +NonBlocking: 2.049396e-01 +Pairwise Hierarchical: 1.217501e+00 +Pairwise Node Aware: 6.205361e-02 +Nonblocking Hierarchical: 1.075471e+00 +Nonblocking Node Aware: 4.937102e-02 +Pairwise Multileader, 4 leaders: 3.330059e-01 +Pairwise Locality Aware, 4 leaders: 6.690090e-02 +Pairwise Multileader Locality, 4 leaders: 3.755594e-01 +Nonblocking Multileader, 4 leaders: 2.489275e-01 +Nonblocking Locality Aware, 4 leaders: 4.563160e-02 +Nonblocking Multileader Locality, 4 leaders: 2.988085e-01 +Pairwise Multileader, 10 leaders: 1.918758e-01 +Pairwise Locality Aware, 10 leaders: 7.673775e-02 +Pairwise Multileader Locality, 10 leaders: 2.045379e-01 +Nonblocking Multileader, 10 leaders: 1.381898e-01 +Nonblocking Locality Aware, 10 leaders: 6.298257e-02 +Nonblocking Multileader Locality, 10 leaders: 1.669538e-01 +Pairwise Multileader, 20 leaders: 1.341792e-01 +Pairwise Locality Aware, 20 leaders: 6.873469e-02 +Pairwise Multileader Locality, 20 leaders: 1.194109e-01 +Nonblocking Multileader, 20 leaders: 8.544964e-02 +Nonblocking Locality Aware, 20 leaders: 4.423573e-02 +Nonblocking Multileader Locality, 20 leaders: 1.036093e-01 +Size 2048 +PMPI: 1.426142e-01 +Pairwise: 1.347378e-01 +NonBlocking: 2.122605e-01 +Pairwise Hierarchical: 2.515640e+00 +Pairwise Node Aware: 1.155624e-01 +Nonblocking Hierarchical: 2.235202e+00 +Nonblocking Node Aware: 9.640245e-02 +Pairwise Multileader, 4 leaders: 6.694498e-01 +Pairwise Locality Aware, 4 leaders: 1.279467e-01 +Pairwise Multileader Locality, 4 leaders: 7.409320e-01 +Nonblocking Multileader, 4 leaders: 5.455441e-01 +Nonblocking Locality Aware, 4 leaders: 9.114183e-02 +Nonblocking Multileader Locality, 4 leaders: 6.573869e-01 +Pairwise Multileader, 10 leaders: 3.892892e-01 +Pairwise Locality Aware, 10 leaders: 1.436089e-01 +Pairwise Multileader Locality, 10 leaders: 4.314356e-01 +Nonblocking Multileader, 10 leaders: 2.775093e-01 +Nonblocking Locality Aware, 10 leaders: 9.290450e-02 +Nonblocking Multileader Locality, 10 leaders: 3.377486e-01 +Pairwise Multileader, 20 leaders: 2.409244e-01 +Pairwise Locality Aware, 20 leaders: 1.366682e-01 +Pairwise Multileader Locality, 20 leaders: 2.327778e-01 +Nonblocking Multileader, 20 leaders: 1.632407e-01 +Nonblocking Locality Aware, 20 leaders: 1.033913e-01 +Nonblocking Multileader Locality, 20 leaders: 2.027746e-01 +Size 4096 +PMPI: 3.225837e-01 +Pairwise: 2.956063e-01 +NonBlocking: 5.146171e-01 +Test 2 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 3.885212e-04 +Pairwise: 7.478004e-03 +NonBlocking: 8.186877e-02 +Pairwise Hierarchical: 1.147187e-03 +Pairwise Node Aware: 3.683327e-04 +Nonblocking Hierarchical: 9.438163e-04 +Nonblocking Node Aware: 2.506517e-04 +Pairwise Multileader, 4 leaders: 6.767826e-04 +Pairwise Locality Aware, 4 leaders: 5.659498e-04 +Pairwise Multileader Locality, 4 leaders: 5.152985e-04 +Nonblocking Multileader, 4 leaders: 3.041156e-04 +Nonblocking Locality Aware, 4 leaders: 2.234121e-04 +Nonblocking Multileader Locality, 4 leaders: 3.574159e-04 +Pairwise Multileader, 10 leaders: 1.268406e-03 +Pairwise Locality Aware, 10 leaders: 1.280693e-03 +Pairwise Multileader Locality, 10 leaders: 2.793070e-04 +Nonblocking Multileader, 10 leaders: 8.237706e-04 +Nonblocking Locality Aware, 10 leaders: 2.785961e-02 +Nonblocking Multileader Locality, 10 leaders: 1.842829e-04 +Pairwise Multileader, 20 leaders: 2.540250e-03 +Pairwise Locality Aware, 20 leaders: 2.504927e-03 +Pairwise Multileader Locality, 20 leaders: 2.621516e-04 +Nonblocking Multileader, 20 leaders: 4.082078e-03 +Nonblocking Locality Aware, 20 leaders: 9.201537e-02 +Nonblocking Multileader Locality, 20 leaders: 1.596912e-04 +Size 2 +PMPI: 4.656626e-04 +Pairwise: 4.870995e-03 +NonBlocking: 2.097123e-02 +Pairwise Hierarchical: 2.592702e-03 +Pairwise Node Aware: 4.541933e-04 +Nonblocking Hierarchical: 2.149293e-03 +Nonblocking Node Aware: 3.277810e-04 +Pairwise Multileader, 4 leaders: 9.331829e-04 +Pairwise Locality Aware, 4 leaders: 7.161936e-04 +Pairwise Multileader Locality, 4 leaders: 8.415201e-04 +Nonblocking Multileader, 4 leaders: 5.533841e-04 +Nonblocking Locality Aware, 4 leaders: 4.942686e-04 +Nonblocking Multileader Locality, 4 leaders: 6.366438e-04 +Pairwise Multileader, 10 leaders: 1.545796e-03 +Pairwise Locality Aware, 10 leaders: 1.363522e-03 +Pairwise Multileader Locality, 10 leaders: 3.638035e-04 +Nonblocking Multileader, 10 leaders: 1.223131e-03 +Nonblocking Locality Aware, 10 leaders: 6.193997e-02 +Nonblocking Multileader Locality, 10 leaders: 2.601183e-04 +Pairwise Multileader, 20 leaders: 2.449923e-03 +Pairwise Locality Aware, 20 leaders: 2.707230e-03 +Pairwise Multileader Locality, 20 leaders: 2.980007e-04 +Nonblocking Multileader, 20 leaders: 5.226773e-03 +Nonblocking Locality Aware, 20 leaders: 9.893999e-02 +Nonblocking Multileader Locality, 20 leaders: 2.058589e-04 +Size 4 +PMPI: 7.827964e-04 +Pairwise: 1.391212e-02 +NonBlocking: 1.240854e-01 +Pairwise Hierarchical: 4.021125e-03 +Pairwise Node Aware: 6.273995e-04 +Nonblocking Hierarchical: 3.442046e-03 +Nonblocking Node Aware: 5.143027e-04 +Pairwise Multileader, 4 leaders: 1.545447e-03 +Pairwise Locality Aware, 4 leaders: 6.597042e-04 +Pairwise Multileader Locality, 4 leaders: 1.315001e-03 +Nonblocking Multileader, 4 leaders: 9.134496e-04 +Nonblocking Locality Aware, 4 leaders: 4.065969e-04 +Nonblocking Multileader Locality, 4 leaders: 1.074633e-03 +Pairwise Multileader, 10 leaders: 1.507272e-03 +Pairwise Locality Aware, 10 leaders: 1.658640e-03 +Pairwise Multileader Locality, 10 leaders: 5.839491e-04 +Nonblocking Multileader, 10 leaders: 4.351689e-04 +Nonblocking Locality Aware, 10 leaders: 8.660636e-02 +Nonblocking Multileader Locality, 10 leaders: 4.643775e-04 +Pairwise Multileader, 20 leaders: 3.226677e-03 +Pairwise Locality Aware, 20 leaders: 2.384344e-03 +Pairwise Multileader Locality, 20 leaders: 4.048493e-04 +Nonblocking Multileader, 20 leaders: 8.780050e-03 +Nonblocking Locality Aware, 20 leaders: 8.665659e-02 +Nonblocking Multileader Locality, 20 leaders: 2.728608e-04 +Size 8 +PMPI: 1.312255e-03 +Pairwise: 1.374306e-02 +NonBlocking: 1.630498e-01 +Pairwise Hierarchical: 9.667063e-03 +Pairwise Node Aware: 1.056985e-03 +Nonblocking Hierarchical: 8.087162e-03 +Nonblocking Node Aware: 8.737512e-04 +Pairwise Multileader, 4 leaders: 2.807439e-03 +Pairwise Locality Aware, 4 leaders: 8.278259e-04 +Pairwise Multileader Locality, 4 leaders: 3.284574e-03 +Nonblocking Multileader, 4 leaders: 2.032054e-03 +Nonblocking Locality Aware, 4 leaders: 5.430106e-04 +Nonblocking Multileader Locality, 4 leaders: 2.596942e-03 +Pairwise Multileader, 10 leaders: 2.312205e-03 +Pairwise Locality Aware, 10 leaders: 1.361125e-03 +Pairwise Multileader Locality, 10 leaders: 1.233653e-03 +Nonblocking Multileader, 10 leaders: 1.297380e-03 +Nonblocking Locality Aware, 10 leaders: 6.204471e-02 +Nonblocking Multileader Locality, 10 leaders: 7.952531e-04 +Pairwise Multileader, 20 leaders: 2.694299e-03 +Pairwise Locality Aware, 20 leaders: 2.084446e-03 +Pairwise Multileader Locality, 20 leaders: 7.664729e-04 +Nonblocking Multileader, 20 leaders: 2.019404e-02 +Nonblocking Locality Aware, 20 leaders: 6.213359e-02 +Nonblocking Multileader Locality, 20 leaders: 6.031993e-04 +Size 16 +PMPI: 2.814241e-03 +Pairwise: 9.697412e-03 +NonBlocking: 1.311970e-01 +Pairwise Hierarchical: 2.018730e-02 +Pairwise Node Aware: 1.293413e-03 +Nonblocking Hierarchical: 1.639659e-02 +Nonblocking Node Aware: 1.166981e-03 +Pairwise Multileader, 4 leaders: 4.812966e-03 +Pairwise Locality Aware, 4 leaders: 1.203516e-03 +Pairwise Multileader Locality, 4 leaders: 5.966457e-03 +Nonblocking Multileader, 4 leaders: 3.883115e-03 +Nonblocking Locality Aware, 4 leaders: 8.011733e-04 +Nonblocking Multileader Locality, 4 leaders: 4.708137e-03 +Pairwise Multileader, 10 leaders: 3.239743e-03 +Pairwise Locality Aware, 10 leaders: 2.037768e-03 +Pairwise Multileader Locality, 10 leaders: 2.180650e-03 +Nonblocking Multileader, 10 leaders: 2.169439e-03 +Nonblocking Locality Aware, 10 leaders: 8.717671e-02 +Nonblocking Multileader Locality, 10 leaders: 1.904819e-03 +Pairwise Multileader, 20 leaders: 3.979381e-03 +Pairwise Locality Aware, 20 leaders: 2.558862e-03 +Pairwise Multileader Locality, 20 leaders: 1.410015e-03 +Nonblocking Multileader, 20 leaders: 6.235997e-02 +Nonblocking Locality Aware, 20 leaders: 9.230522e-02 +Nonblocking Multileader Locality, 20 leaders: 1.213653e-03 +Size 32 +PMPI: 4.577496e-03 +Pairwise: 1.253456e-02 +NonBlocking: 1.631845e-01 +Pairwise Hierarchical: 4.242538e-02 +Pairwise Node Aware: 2.095180e-03 +Nonblocking Hierarchical: 3.355638e-02 +Nonblocking Node Aware: 2.015657e-03 +Pairwise Multileader, 4 leaders: 1.340231e-02 +Pairwise Locality Aware, 4 leaders: 1.848645e-03 +Pairwise Multileader Locality, 4 leaders: 1.103154e-02 +Nonblocking Multileader, 4 leaders: 8.614264e-03 +Nonblocking Locality Aware, 4 leaders: 1.575594e-03 +Nonblocking Multileader Locality, 4 leaders: 8.768141e-03 +Pairwise Multileader, 10 leaders: 5.385581e-03 +Pairwise Locality Aware, 10 leaders: 2.925347e-03 +Pairwise Multileader Locality, 10 leaders: 4.783425e-03 +Nonblocking Multileader, 10 leaders: 4.963975e-03 +Nonblocking Locality Aware, 10 leaders: 9.313604e-02 +Nonblocking Multileader Locality, 10 leaders: 3.825425e-03 +Pairwise Multileader, 20 leaders: 5.503902e-03 +Pairwise Locality Aware, 20 leaders: 2.810522e-03 +Pairwise Multileader Locality, 20 leaders: 2.586881e-03 +Nonblocking Multileader, 20 leaders: 2.003455e-02 +Nonblocking Locality Aware, 20 leaders: 9.325453e-02 +Nonblocking Multileader Locality, 20 leaders: 1.749263e-03 +Size 64 +PMPI: 9.048731e-03 +Pairwise: 1.518943e-02 +NonBlocking: 1.636417e-01 +Pairwise Hierarchical: 8.273521e-02 +Pairwise Node Aware: 4.388383e-03 +Nonblocking Hierarchical: 6.424360e-02 +Nonblocking Node Aware: 3.319913e-03 +Pairwise Multileader, 4 leaders: 2.598608e-02 +Pairwise Locality Aware, 4 leaders: 3.476691e-03 +Pairwise Multileader Locality, 4 leaders: 2.231037e-02 +Nonblocking Multileader, 4 leaders: 1.636491e-02 +Nonblocking Locality Aware, 4 leaders: 3.842887e-03 +Nonblocking Multileader Locality, 4 leaders: 1.765232e-02 +Pairwise Multileader, 10 leaders: 1.202461e-02 +Pairwise Locality Aware, 10 leaders: 3.996800e-03 +Pairwise Multileader Locality, 10 leaders: 1.024144e-02 +Nonblocking Multileader, 10 leaders: 7.876929e-03 +Nonblocking Locality Aware, 10 leaders: 7.632976e-02 +Nonblocking Multileader Locality, 10 leaders: 8.509270e-03 +Pairwise Multileader, 20 leaders: 8.095084e-03 +Pairwise Locality Aware, 20 leaders: 5.453418e-03 +Pairwise Multileader Locality, 20 leaders: 5.661901e-03 +Nonblocking Multileader, 20 leaders: 4.303389e-03 +Nonblocking Locality Aware, 20 leaders: 1.397203e-01 +Nonblocking Multileader Locality, 20 leaders: 4.377829e-03 +Size 128 +PMPI: 1.610684e-02 +Pairwise: 1.904374e-02 +NonBlocking: 1.387384e-01 +Pairwise Hierarchical: 1.580213e-01 +Pairwise Node Aware: 9.452177e-03 +Nonblocking Hierarchical: 1.240422e-01 +Nonblocking Node Aware: 7.095227e-03 +Pairwise Multileader, 4 leaders: 5.310162e-02 +Pairwise Locality Aware, 4 leaders: 6.831349e-03 +Pairwise Multileader Locality, 4 leaders: 5.481982e-02 +Nonblocking Multileader, 4 leaders: 3.714897e-02 +Nonblocking Locality Aware, 4 leaders: 7.818360e-03 +Nonblocking Multileader Locality, 4 leaders: 4.196103e-02 +Pairwise Multileader, 10 leaders: 2.265429e-02 +Pairwise Locality Aware, 10 leaders: 7.908054e-03 +Pairwise Multileader Locality, 10 leaders: 2.424963e-02 +Nonblocking Multileader, 10 leaders: 1.727127e-02 +Nonblocking Locality Aware, 10 leaders: 9.967747e-03 +Nonblocking Multileader Locality, 10 leaders: 1.958599e-02 +Pairwise Multileader, 20 leaders: 1.500100e-02 +Pairwise Locality Aware, 20 leaders: 9.076166e-03 +Pairwise Multileader Locality, 20 leaders: 1.169397e-02 +Nonblocking Multileader, 20 leaders: 1.035984e-02 +Nonblocking Locality Aware, 20 leaders: 9.717481e-02 +Nonblocking Multileader Locality, 20 leaders: 9.618162e-03 +Size 256 +PMPI: 3.377285e-02 +Pairwise: 2.100875e-02 +NonBlocking: 1.427803e-01 +Pairwise Hierarchical: 3.083426e-01 +Pairwise Node Aware: 1.464036e-02 +Nonblocking Hierarchical: 2.479594e-01 +Nonblocking Node Aware: 1.356602e-02 +Pairwise Multileader, 4 leaders: 9.493639e-02 +Pairwise Locality Aware, 4 leaders: 1.770449e-02 +Pairwise Multileader Locality, 4 leaders: 1.046179e-01 +Nonblocking Multileader, 4 leaders: 6.864804e-02 +Nonblocking Locality Aware, 4 leaders: 1.495009e-02 +Nonblocking Multileader Locality, 4 leaders: 7.953759e-02 +Pairwise Multileader, 10 leaders: 5.521455e-02 +Pairwise Locality Aware, 10 leaders: 1.522211e-02 +Pairwise Multileader Locality, 10 leaders: 5.167622e-02 +Nonblocking Multileader, 10 leaders: 3.706520e-02 +Nonblocking Locality Aware, 10 leaders: 2.401096e-02 +Nonblocking Multileader Locality, 10 leaders: 4.224329e-02 +Pairwise Multileader, 20 leaders: 2.888506e-02 +Pairwise Locality Aware, 20 leaders: 1.610305e-02 +Pairwise Multileader Locality, 20 leaders: 2.598425e-02 +Nonblocking Multileader, 20 leaders: 1.697754e-02 +Nonblocking Locality Aware, 20 leaders: 3.844738e-02 +Nonblocking Multileader Locality, 20 leaders: 2.056405e-02 +Size 512 +PMPI: 7.929235e-02 +Pairwise: 3.489512e-02 +NonBlocking: 1.765847e-01 +Pairwise Hierarchical: 5.963804e-01 +Pairwise Node Aware: 3.301239e-02 +Nonblocking Hierarchical: 5.446209e-01 +Nonblocking Node Aware: 3.038346e-02 +Pairwise Multileader, 4 leaders: 1.811619e-01 +Pairwise Locality Aware, 4 leaders: 3.044264e-02 +Pairwise Multileader Locality, 4 leaders: 2.025514e-01 +Nonblocking Multileader, 4 leaders: 1.316438e-01 +Nonblocking Locality Aware, 4 leaders: 2.284937e-02 +Nonblocking Multileader Locality, 4 leaders: 1.569969e-01 +Pairwise Multileader, 10 leaders: 1.060530e-01 +Pairwise Locality Aware, 10 leaders: 3.355888e-02 +Pairwise Multileader Locality, 10 leaders: 1.058262e-01 +Nonblocking Multileader, 10 leaders: 7.348405e-02 +Nonblocking Locality Aware, 10 leaders: 2.658681e-02 +Nonblocking Multileader Locality, 10 leaders: 8.670894e-02 +Pairwise Multileader, 20 leaders: 5.773434e-02 +Pairwise Locality Aware, 20 leaders: 3.492593e-02 +Pairwise Multileader Locality, 20 leaders: 5.420593e-02 +Nonblocking Multileader, 20 leaders: 3.384803e-02 +Nonblocking Locality Aware, 20 leaders: 4.842403e-02 +Nonblocking Multileader Locality, 20 leaders: 4.353154e-02 +Size 1024 +PMPI: 7.691317e-02 +Pairwise: 6.977977e-02 +NonBlocking: 2.211700e-01 +Pairwise Hierarchical: 1.182616e+00 +Pairwise Node Aware: 5.819379e-02 +Nonblocking Hierarchical: 1.071141e+00 +Nonblocking Node Aware: 4.806296e-02 +Pairwise Multileader, 4 leaders: 3.305948e-01 +Pairwise Locality Aware, 4 leaders: 6.361833e-02 +Pairwise Multileader Locality, 4 leaders: 3.749542e-01 +Nonblocking Multileader, 4 leaders: 2.469162e-01 +Nonblocking Locality Aware, 4 leaders: 4.624636e-02 +Nonblocking Multileader Locality, 4 leaders: 3.008025e-01 +Pairwise Multileader, 10 leaders: 1.944837e-01 +Pairwise Locality Aware, 10 leaders: 6.729415e-02 +Pairwise Multileader Locality, 10 leaders: 2.045022e-01 +Nonblocking Multileader, 10 leaders: 1.387563e-01 +Nonblocking Locality Aware, 10 leaders: 5.945051e-02 +Nonblocking Multileader Locality, 10 leaders: 1.678692e-01 +Pairwise Multileader, 20 leaders: 1.356436e-01 +Pairwise Locality Aware, 20 leaders: 7.375354e-02 +Pairwise Multileader Locality, 20 leaders: 1.199923e-01 +Nonblocking Multileader, 20 leaders: 8.477018e-02 +Nonblocking Locality Aware, 20 leaders: 6.095028e-02 +Nonblocking Multileader Locality, 20 leaders: 1.045430e-01 +Size 2048 +PMPI: 1.402930e-01 +Pairwise: 1.322982e-01 +NonBlocking: 2.110277e-01 +Pairwise Hierarchical: 2.505509e+00 +Pairwise Node Aware: 1.170062e-01 +Nonblocking Hierarchical: 2.261080e+00 +Nonblocking Node Aware: 9.799649e-02 +Pairwise Multileader, 4 leaders: 6.720463e-01 +Pairwise Locality Aware, 4 leaders: 1.294366e-01 +Pairwise Multileader Locality, 4 leaders: 7.567576e-01 +Nonblocking Multileader, 4 leaders: 5.598694e-01 +Nonblocking Locality Aware, 4 leaders: 9.004450e-02 +Nonblocking Multileader Locality, 4 leaders: 6.594793e-01 +Pairwise Multileader, 10 leaders: 3.953671e-01 +Pairwise Locality Aware, 10 leaders: 1.430984e-01 +Pairwise Multileader Locality, 10 leaders: 4.329844e-01 +Nonblocking Multileader, 10 leaders: 2.757359e-01 +Nonblocking Locality Aware, 10 leaders: 9.285788e-02 +Nonblocking Multileader Locality, 10 leaders: 3.401643e-01 +Pairwise Multileader, 20 leaders: 2.414140e-01 +Pairwise Locality Aware, 20 leaders: 1.372615e-01 +Pairwise Multileader Locality, 20 leaders: 2.327329e-01 +Nonblocking Multileader, 20 leaders: 1.641776e-01 +Nonblocking Locality Aware, 20 leaders: 1.040875e-01 +Nonblocking Multileader Locality, 20 leaders: 2.038472e-01 +Size 4096 +PMPI: 3.045753e-01 +Pairwise: 2.977667e-01 +NonBlocking: 4.818090e-01 +Test 3 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 3.960546e-04 +Pairwise: 5.474995e-03 +NonBlocking: 9.639122e-02 +Pairwise Hierarchical: 1.116929e-03 +Pairwise Node Aware: 3.941240e-04 +Nonblocking Hierarchical: 9.495636e-04 +Nonblocking Node Aware: 2.523344e-04 +Pairwise Multileader, 4 leaders: 7.216885e-04 +Pairwise Locality Aware, 4 leaders: 7.156528e-04 +Pairwise Multileader Locality, 4 leaders: 4.765839e-04 +Nonblocking Multileader, 4 leaders: 3.064689e-04 +Nonblocking Locality Aware, 4 leaders: 3.103908e-04 +Nonblocking Multileader Locality, 4 leaders: 3.495175e-04 +Pairwise Multileader, 10 leaders: 1.382130e-03 +Pairwise Locality Aware, 10 leaders: 1.073391e-03 +Pairwise Multileader Locality, 10 leaders: 2.910807e-04 +Nonblocking Multileader, 10 leaders: 1.236408e-03 +Nonblocking Locality Aware, 10 leaders: 2.494663e-02 +Nonblocking Multileader Locality, 10 leaders: 1.802727e-04 +Pairwise Multileader, 20 leaders: 3.319134e-03 +Pairwise Locality Aware, 20 leaders: 2.100537e-03 +Pairwise Multileader Locality, 20 leaders: 2.305420e-04 +Nonblocking Multileader, 20 leaders: 1.075209e-03 +Nonblocking Locality Aware, 20 leaders: 5.507285e-02 +Nonblocking Multileader Locality, 20 leaders: 1.560742e-04 +Size 2 +PMPI: 4.862744e-04 +Pairwise: 6.364438e-03 +NonBlocking: 4.252438e-02 +Pairwise Hierarchical: 2.648995e-03 +Pairwise Node Aware: 4.620677e-04 +Nonblocking Hierarchical: 2.147210e-03 +Nonblocking Node Aware: 3.576659e-04 +Pairwise Multileader, 4 leaders: 9.856562e-04 +Pairwise Locality Aware, 4 leaders: 6.840954e-04 +Pairwise Multileader Locality, 4 leaders: 1.328916e-03 +Nonblocking Multileader, 4 leaders: 4.510802e-04 +Nonblocking Locality Aware, 4 leaders: 3.886157e-04 +Nonblocking Multileader Locality, 4 leaders: 4.860972e-04 +Pairwise Multileader, 10 leaders: 1.096833e-03 +Pairwise Locality Aware, 10 leaders: 1.657395e-03 +Pairwise Multileader Locality, 10 leaders: 3.680258e-04 +Nonblocking Multileader, 10 leaders: 1.513308e-03 +Nonblocking Locality Aware, 10 leaders: 9.181980e-02 +Nonblocking Multileader Locality, 10 leaders: 2.680016e-04 +Pairwise Multileader, 20 leaders: 3.365415e-03 +Pairwise Locality Aware, 20 leaders: 2.703683e-03 +Pairwise Multileader Locality, 20 leaders: 3.352101e-04 +Nonblocking Multileader, 20 leaders: 2.553489e-03 +Nonblocking Locality Aware, 20 leaders: 7.451855e-02 +Nonblocking Multileader Locality, 20 leaders: 2.238203e-04 +Size 4 +PMPI: 8.205711e-04 +Pairwise: 1.106186e-02 +NonBlocking: 1.400955e-01 +Pairwise Hierarchical: 3.980453e-03 +Pairwise Node Aware: 6.022592e-04 +Nonblocking Hierarchical: 3.461640e-03 +Nonblocking Node Aware: 5.526901e-04 +Pairwise Multileader, 4 leaders: 1.564894e-03 +Pairwise Locality Aware, 4 leaders: 5.753050e-04 +Pairwise Multileader Locality, 4 leaders: 1.324541e-03 +Nonblocking Multileader, 4 leaders: 9.103165e-04 +Nonblocking Locality Aware, 4 leaders: 3.570327e-04 +Nonblocking Multileader Locality, 4 leaders: 1.073172e-03 +Pairwise Multileader, 10 leaders: 1.568238e-03 +Pairwise Locality Aware, 10 leaders: 1.478648e-03 +Pairwise Multileader Locality, 10 leaders: 5.863741e-04 +Nonblocking Multileader, 10 leaders: 3.252252e-03 +Nonblocking Locality Aware, 10 leaders: 6.210261e-02 +Nonblocking Multileader Locality, 10 leaders: 4.540004e-04 +Pairwise Multileader, 20 leaders: 3.094871e-03 +Pairwise Locality Aware, 20 leaders: 3.454732e-03 +Pairwise Multileader Locality, 20 leaders: 4.655524e-04 +Nonblocking Multileader, 20 leaders: 1.276040e-02 +Nonblocking Locality Aware, 20 leaders: 9.180171e-02 +Nonblocking Multileader Locality, 20 leaders: 2.765709e-04 +Size 8 +PMPI: 1.405421e-03 +Pairwise: 1.022192e-02 +NonBlocking: 1.849372e-01 +Pairwise Hierarchical: 9.842139e-03 +Pairwise Node Aware: 8.929282e-04 +Nonblocking Hierarchical: 8.127924e-03 +Nonblocking Node Aware: 8.828257e-04 +Pairwise Multileader, 4 leaders: 4.197558e-03 +Pairwise Locality Aware, 4 leaders: 1.019653e-03 +Pairwise Multileader Locality, 4 leaders: 3.449939e-03 +Nonblocking Multileader, 4 leaders: 2.008045e-03 +Nonblocking Locality Aware, 4 leaders: 6.432198e-04 +Nonblocking Multileader Locality, 4 leaders: 2.618099e-03 +Pairwise Multileader, 10 leaders: 2.710071e-03 +Pairwise Locality Aware, 10 leaders: 2.087068e-03 +Pairwise Multileader Locality, 10 leaders: 1.344178e-03 +Nonblocking Multileader, 10 leaders: 1.161001e-03 +Nonblocking Locality Aware, 10 leaders: 6.931186e-02 +Nonblocking Multileader Locality, 10 leaders: 9.292750e-04 +Pairwise Multileader, 20 leaders: 3.457649e-03 +Pairwise Locality Aware, 20 leaders: 2.952652e-03 +Pairwise Multileader Locality, 20 leaders: 7.856568e-04 +Nonblocking Multileader, 20 leaders: 2.045112e-02 +Nonblocking Locality Aware, 20 leaders: 8.681970e-02 +Nonblocking Multileader Locality, 20 leaders: 6.722649e-04 +Size 16 +PMPI: 2.838411e-03 +Pairwise: 9.943719e-03 +NonBlocking: 1.404253e-01 +Pairwise Hierarchical: 2.090883e-02 +Pairwise Node Aware: 1.401175e-03 +Nonblocking Hierarchical: 1.655671e-02 +Nonblocking Node Aware: 1.207182e-03 +Pairwise Multileader, 4 leaders: 4.912534e-03 +Pairwise Locality Aware, 4 leaders: 1.214650e-03 +Pairwise Multileader Locality, 4 leaders: 6.027324e-03 +Nonblocking Multileader, 4 leaders: 3.965592e-03 +Nonblocking Locality Aware, 4 leaders: 9.778685e-04 +Nonblocking Multileader Locality, 4 leaders: 4.842933e-03 +Pairwise Multileader, 10 leaders: 3.260387e-03 +Pairwise Locality Aware, 10 leaders: 1.830816e-03 +Pairwise Multileader Locality, 10 leaders: 2.225314e-03 +Nonblocking Multileader, 10 leaders: 1.873477e-03 +Nonblocking Locality Aware, 10 leaders: 9.270157e-02 +Nonblocking Multileader Locality, 10 leaders: 1.914955e-03 +Pairwise Multileader, 20 leaders: 4.564122e-03 +Pairwise Locality Aware, 20 leaders: 3.169893e-03 +Pairwise Multileader Locality, 20 leaders: 1.601558e-03 +Nonblocking Multileader, 20 leaders: 3.570014e-02 +Nonblocking Locality Aware, 20 leaders: 1.118160e-01 +Nonblocking Multileader Locality, 20 leaders: 1.200267e-03 +Size 32 +PMPI: 4.705470e-03 +Pairwise: 1.413085e-02 +NonBlocking: 1.397427e-01 +Pairwise Hierarchical: 4.255240e-02 +Pairwise Node Aware: 2.121627e-03 +Nonblocking Hierarchical: 3.374974e-02 +Nonblocking Node Aware: 2.020638e-03 +Pairwise Multileader, 4 leaders: 1.350621e-02 +Pairwise Locality Aware, 4 leaders: 2.048482e-03 +Pairwise Multileader Locality, 4 leaders: 1.091914e-02 +Nonblocking Multileader, 4 leaders: 8.584576e-03 +Nonblocking Locality Aware, 4 leaders: 1.849526e-03 +Nonblocking Multileader Locality, 4 leaders: 8.756973e-03 +Pairwise Multileader, 10 leaders: 5.574793e-03 +Pairwise Locality Aware, 10 leaders: 2.766874e-03 +Pairwise Multileader Locality, 10 leaders: 4.734075e-03 +Nonblocking Multileader, 10 leaders: 3.852127e-03 +Nonblocking Locality Aware, 10 leaders: 9.343866e-02 +Nonblocking Multileader Locality, 10 leaders: 3.893754e-03 +Pairwise Multileader, 20 leaders: 5.233732e-03 +Pairwise Locality Aware, 20 leaders: 3.645739e-03 +Pairwise Multileader Locality, 20 leaders: 2.567764e-03 +Nonblocking Multileader, 20 leaders: 2.627530e-02 +Nonblocking Locality Aware, 20 leaders: 9.302013e-02 +Nonblocking Multileader Locality, 20 leaders: 2.215860e-03 +Size 64 +PMPI: 9.066405e-03 +Pairwise: 1.198786e-02 +NonBlocking: 1.414897e-01 +Pairwise Hierarchical: 8.173402e-02 +Pairwise Node Aware: 4.550808e-03 +Nonblocking Hierarchical: 6.494452e-02 +Nonblocking Node Aware: 3.514778e-03 +Pairwise Multileader, 4 leaders: 2.611272e-02 +Pairwise Locality Aware, 4 leaders: 3.428543e-03 +Pairwise Multileader Locality, 4 leaders: 2.219968e-02 +Nonblocking Multileader, 4 leaders: 1.647681e-02 +Nonblocking Locality Aware, 4 leaders: 3.348759e-03 +Nonblocking Multileader Locality, 4 leaders: 1.759053e-02 +Pairwise Multileader, 10 leaders: 1.176387e-02 +Pairwise Locality Aware, 10 leaders: 3.428574e-03 +Pairwise Multileader Locality, 10 leaders: 1.019865e-02 +Nonblocking Multileader, 10 leaders: 7.853390e-03 +Nonblocking Locality Aware, 10 leaders: 7.619476e-02 +Nonblocking Multileader Locality, 10 leaders: 8.558144e-03 +Pairwise Multileader, 20 leaders: 7.899463e-03 +Pairwise Locality Aware, 20 leaders: 4.789286e-03 +Pairwise Multileader Locality, 20 leaders: 5.488634e-03 +Nonblocking Multileader, 20 leaders: 4.857965e-03 +Nonblocking Locality Aware, 20 leaders: 1.177537e-01 +Nonblocking Multileader Locality, 20 leaders: 4.318849e-03 +Size 128 +PMPI: 1.575609e-02 +Pairwise: 1.855862e-02 +NonBlocking: 1.146538e-01 +Pairwise Hierarchical: 1.581138e-01 +Pairwise Node Aware: 9.241728e-03 +Nonblocking Hierarchical: 1.238153e-01 +Nonblocking Node Aware: 6.616641e-03 +Pairwise Multileader, 4 leaders: 5.252088e-02 +Pairwise Locality Aware, 4 leaders: 7.693637e-03 +Pairwise Multileader Locality, 4 leaders: 5.529461e-02 +Nonblocking Multileader, 4 leaders: 3.705402e-02 +Nonblocking Locality Aware, 4 leaders: 1.046316e-02 +Nonblocking Multileader Locality, 4 leaders: 4.250655e-02 +Pairwise Multileader, 10 leaders: 2.351790e-02 +Pairwise Locality Aware, 10 leaders: 8.337354e-03 +Pairwise Multileader Locality, 10 leaders: 2.411577e-02 +Nonblocking Multileader, 10 leaders: 1.738704e-02 +Nonblocking Locality Aware, 10 leaders: 1.072018e-02 +Nonblocking Multileader Locality, 10 leaders: 1.958778e-02 +Pairwise Multileader, 20 leaders: 1.505632e-02 +Pairwise Locality Aware, 20 leaders: 8.647552e-03 +Pairwise Multileader Locality, 20 leaders: 1.166353e-02 +Nonblocking Multileader, 20 leaders: 8.845706e-03 +Nonblocking Locality Aware, 20 leaders: 9.670877e-02 +Nonblocking Multileader Locality, 20 leaders: 9.580278e-03 +Size 256 +PMPI: 3.097310e-02 +Pairwise: 2.905423e-02 +NonBlocking: 1.430299e-01 +Pairwise Hierarchical: 3.068532e-01 +Pairwise Node Aware: 1.459277e-02 +Nonblocking Hierarchical: 2.492509e-01 +Nonblocking Node Aware: 1.365241e-02 +Pairwise Multileader, 4 leaders: 9.575482e-02 +Pairwise Locality Aware, 4 leaders: 1.317357e-02 +Pairwise Multileader Locality, 4 leaders: 1.051971e-01 +Nonblocking Multileader, 4 leaders: 6.875683e-02 +Nonblocking Locality Aware, 4 leaders: 1.139220e-02 +Nonblocking Multileader Locality, 4 leaders: 8.054341e-02 +Pairwise Multileader, 10 leaders: 5.531116e-02 +Pairwise Locality Aware, 10 leaders: 1.590088e-02 +Pairwise Multileader Locality, 10 leaders: 5.114276e-02 +Nonblocking Multileader, 10 leaders: 3.707978e-02 +Nonblocking Locality Aware, 10 leaders: 2.510372e-02 +Nonblocking Multileader Locality, 10 leaders: 4.188994e-02 +Pairwise Multileader, 20 leaders: 3.019336e-02 +Pairwise Locality Aware, 20 leaders: 1.555240e-02 +Pairwise Multileader Locality, 20 leaders: 2.615888e-02 +Nonblocking Multileader, 20 leaders: 1.719047e-02 +Nonblocking Locality Aware, 20 leaders: 2.997216e-02 +Nonblocking Multileader Locality, 20 leaders: 2.078194e-02 +Size 512 +PMPI: 5.732195e-02 +Pairwise: 3.929766e-02 +NonBlocking: 1.573110e-01 +Pairwise Hierarchical: 6.061261e-01 +Pairwise Node Aware: 3.256360e-02 +Nonblocking Hierarchical: 5.331022e-01 +Nonblocking Node Aware: 2.984250e-02 +Pairwise Multileader, 4 leaders: 1.798160e-01 +Pairwise Locality Aware, 4 leaders: 3.106555e-02 +Pairwise Multileader Locality, 4 leaders: 2.012846e-01 +Nonblocking Multileader, 4 leaders: 1.329920e-01 +Nonblocking Locality Aware, 4 leaders: 2.626152e-02 +Nonblocking Multileader Locality, 4 leaders: 1.580197e-01 +Pairwise Multileader, 10 leaders: 1.062948e-01 +Pairwise Locality Aware, 10 leaders: 3.419549e-02 +Pairwise Multileader Locality, 10 leaders: 1.058527e-01 +Nonblocking Multileader, 10 leaders: 7.315307e-02 +Nonblocking Locality Aware, 10 leaders: 2.836180e-02 +Nonblocking Multileader Locality, 10 leaders: 8.659736e-02 +Pairwise Multileader, 20 leaders: 5.892080e-02 +Pairwise Locality Aware, 20 leaders: 3.322558e-02 +Pairwise Multileader Locality, 20 leaders: 5.437122e-02 +Nonblocking Multileader, 20 leaders: 3.439948e-02 +Nonblocking Locality Aware, 20 leaders: 4.324607e-02 +Nonblocking Multileader Locality, 20 leaders: 4.358004e-02 +Size 1024 +PMPI: 7.635907e-02 +Pairwise: 7.154243e-02 +NonBlocking: 1.985505e-01 +Pairwise Hierarchical: 1.214027e+00 +Pairwise Node Aware: 6.054313e-02 +Nonblocking Hierarchical: 1.094687e+00 +Nonblocking Node Aware: 4.814290e-02 +Pairwise Multileader, 4 leaders: 3.339105e-01 +Pairwise Locality Aware, 4 leaders: 6.332949e-02 +Pairwise Multileader Locality, 4 leaders: 3.714774e-01 +Nonblocking Multileader, 4 leaders: 2.504473e-01 +Nonblocking Locality Aware, 4 leaders: 4.599229e-02 +Nonblocking Multileader Locality, 4 leaders: 2.966327e-01 +Pairwise Multileader, 10 leaders: 1.939747e-01 +Pairwise Locality Aware, 10 leaders: 7.147648e-02 +Pairwise Multileader Locality, 10 leaders: 2.049406e-01 +Nonblocking Multileader, 10 leaders: 1.387262e-01 +Nonblocking Locality Aware, 10 leaders: 9.969333e-02 +Nonblocking Multileader Locality, 10 leaders: 1.686026e-01 +Pairwise Multileader, 20 leaders: 1.409243e-01 +Pairwise Locality Aware, 20 leaders: 7.024910e-02 +Pairwise Multileader Locality, 20 leaders: 1.196581e-01 +Nonblocking Multileader, 20 leaders: 8.495365e-02 +Nonblocking Locality Aware, 20 leaders: 4.496643e-02 +Nonblocking Multileader Locality, 20 leaders: 1.039970e-01 +Size 2048 +PMPI: 1.461020e-01 +Pairwise: 1.400693e-01 +NonBlocking: 2.126853e-01 +Pairwise Hierarchical: 2.531030e+00 +Pairwise Node Aware: 1.163156e-01 diff --git a/runscripts/dane_results/alltoall_N16.2484416.err b/runscripts/dane_results/alltoall_N16.2484416.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/dane_results/alltoall_N16.2484416.out b/runscripts/dane_results/alltoall_N16.2484416.out new file mode 100644 index 000000000..46e3b0cc4 --- /dev/null +++ b/runscripts/dane_results/alltoall_N16.2484416.out @@ -0,0 +1,783 @@ +Test 1 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 4.330241e-04 +Pairwise: 5.950948e-03 +NonBlocking: 4.418651e-02 +Pairwise Hierarchical: 1.231506e-03 +Pairwise Node Aware: 4.213355e-04 +Nonblocking Hierarchical: 1.163910e-03 +Nonblocking Node Aware: 3.396008e-04 +Pairwise Multileader, 4 leaders: 6.823751e-04 +Pairwise Locality Aware, 4 leaders: 5.584483e-04 +Pairwise Multileader Locality, 4 leaders: 5.086404e-04 +Nonblocking Multileader, 4 leaders: 3.027484e-04 +Nonblocking Locality Aware, 4 leaders: 2.812514e-04 +Nonblocking Multileader Locality, 4 leaders: 3.608931e-04 +Pairwise Multileader, 10 leaders: 1.326520e-03 +Pairwise Locality Aware, 10 leaders: 9.543917e-04 +Pairwise Multileader Locality, 10 leaders: 2.778076e-04 +Nonblocking Multileader, 10 leaders: 7.781402e-04 +Nonblocking Locality Aware, 10 leaders: 1.262865e-02 +Nonblocking Multileader Locality, 10 leaders: 1.859482e-04 +Pairwise Multileader, 20 leaders: 2.424050e-03 +Pairwise Locality Aware, 20 leaders: 2.391027e-03 +Pairwise Multileader Locality, 20 leaders: 3.088969e-04 +Nonblocking Multileader, 20 leaders: 7.990173e-04 +Nonblocking Locality Aware, 20 leaders: 4.976892e-02 +Nonblocking Multileader Locality, 20 leaders: 1.591567e-04 +Size 2 +PMPI: 5.978510e-04 +Pairwise: 4.568059e-03 +NonBlocking: 4.187781e-02 +Pairwise Hierarchical: 3.042974e-03 +Pairwise Node Aware: 4.450407e-04 +Nonblocking Hierarchical: 2.402693e-03 +Nonblocking Node Aware: 3.723002e-04 +Pairwise Multileader, 4 leaders: 1.008550e-03 +Pairwise Locality Aware, 4 leaders: 7.492315e-04 +Pairwise Multileader Locality, 4 leaders: 1.245089e-03 +Nonblocking Multileader, 4 leaders: 5.444290e-04 +Nonblocking Locality Aware, 4 leaders: 3.572208e-04 +Nonblocking Multileader Locality, 4 leaders: 8.786531e-04 +Pairwise Multileader, 10 leaders: 1.103616e-03 +Pairwise Locality Aware, 10 leaders: 1.341004e-03 +Pairwise Multileader Locality, 10 leaders: 3.493382e-04 +Nonblocking Multileader, 10 leaders: 9.601956e-04 +Nonblocking Locality Aware, 10 leaders: 7.422236e-02 +Nonblocking Multileader Locality, 10 leaders: 2.747619e-04 +Pairwise Multileader, 20 leaders: 1.996505e-03 +Pairwise Locality Aware, 20 leaders: 2.385028e-03 +Pairwise Multileader Locality, 20 leaders: 3.188455e-04 +Nonblocking Multileader, 20 leaders: 1.982868e-03 +Nonblocking Locality Aware, 20 leaders: 6.192662e-02 +Nonblocking Multileader Locality, 20 leaders: 2.366972e-04 +Size 4 +PMPI: 8.139450e-04 +Pairwise: 9.111973e-03 +NonBlocking: 1.626817e-01 +Pairwise Hierarchical: 4.891617e-03 +Pairwise Node Aware: 5.751103e-04 +Nonblocking Hierarchical: 3.841045e-03 +Nonblocking Node Aware: 5.048780e-04 +Pairwise Multileader, 4 leaders: 1.436506e-03 +Pairwise Locality Aware, 4 leaders: 6.993173e-04 +Pairwise Multileader Locality, 4 leaders: 1.324855e-03 +Nonblocking Multileader, 4 leaders: 8.961939e-04 +Nonblocking Locality Aware, 4 leaders: 3.404878e-04 +Nonblocking Multileader Locality, 4 leaders: 1.129216e-03 +Pairwise Multileader, 10 leaders: 1.656599e-03 +Pairwise Locality Aware, 10 leaders: 1.299851e-03 +Pairwise Multileader Locality, 10 leaders: 5.777706e-04 +Nonblocking Multileader, 10 leaders: 3.791962e-03 +Nonblocking Locality Aware, 10 leaders: 8.670521e-02 +Nonblocking Multileader Locality, 10 leaders: 4.640104e-04 +Pairwise Multileader, 20 leaders: 2.556104e-03 +Pairwise Locality Aware, 20 leaders: 2.283212e-03 +Pairwise Multileader Locality, 20 leaders: 4.955640e-04 +Nonblocking Multileader, 20 leaders: 3.979302e-03 +Nonblocking Locality Aware, 20 leaders: 6.699171e-02 +Nonblocking Multileader Locality, 20 leaders: 3.827473e-04 +Size 8 +PMPI: 1.609922e-03 +Pairwise: 1.049803e-02 +NonBlocking: 1.637957e-01 +Pairwise Hierarchical: 1.273378e-02 +Pairwise Node Aware: 1.010801e-03 +Nonblocking Hierarchical: 8.355523e-03 +Nonblocking Node Aware: 7.515149e-04 +Pairwise Multileader, 4 leaders: 3.210506e-03 +Pairwise Locality Aware, 4 leaders: 1.143823e-03 +Pairwise Multileader Locality, 4 leaders: 3.507109e-03 +Nonblocking Multileader, 4 leaders: 2.082108e-03 +Nonblocking Locality Aware, 4 leaders: 5.088995e-04 +Nonblocking Multileader Locality, 4 leaders: 2.511954e-03 +Pairwise Multileader, 10 leaders: 2.340936e-03 +Pairwise Locality Aware, 10 leaders: 1.323712e-03 +Pairwise Multileader Locality, 10 leaders: 1.205298e-03 +Nonblocking Multileader, 10 leaders: 3.088389e-03 +Nonblocking Locality Aware, 10 leaders: 7.453733e-02 +Nonblocking Multileader Locality, 10 leaders: 7.744383e-04 +Pairwise Multileader, 20 leaders: 2.433188e-03 +Pairwise Locality Aware, 20 leaders: 2.912048e-03 +Pairwise Multileader Locality, 20 leaders: 8.064673e-04 +Nonblocking Multileader, 20 leaders: 2.310288e-02 +Nonblocking Locality Aware, 20 leaders: 8.675227e-02 +Nonblocking Multileader Locality, 20 leaders: 5.204873e-04 +Size 16 +PMPI: 2.844742e-03 +Pairwise: 1.307978e-02 +NonBlocking: 1.184554e-01 +Pairwise Hierarchical: 2.102447e-02 +Pairwise Node Aware: 1.320471e-03 +Nonblocking Hierarchical: 1.668942e-02 +Nonblocking Node Aware: 1.175672e-03 +Pairwise Multileader, 4 leaders: 4.692157e-03 +Pairwise Locality Aware, 4 leaders: 1.235443e-03 +Pairwise Multileader Locality, 4 leaders: 5.867382e-03 +Nonblocking Multileader, 4 leaders: 3.686511e-03 +Nonblocking Locality Aware, 4 leaders: 8.905253e-04 +Nonblocking Multileader Locality, 4 leaders: 4.951118e-03 +Pairwise Multileader, 10 leaders: 4.729464e-03 +Pairwise Locality Aware, 10 leaders: 1.999899e-03 +Pairwise Multileader Locality, 10 leaders: 2.211262e-03 +Nonblocking Multileader, 10 leaders: 1.596786e-03 +Nonblocking Locality Aware, 10 leaders: 8.725084e-02 +Nonblocking Multileader Locality, 10 leaders: 1.925897e-03 +Pairwise Multileader, 20 leaders: 2.690617e-03 +Pairwise Locality Aware, 20 leaders: 2.595551e-03 +Pairwise Multileader Locality, 20 leaders: 1.384101e-03 +Nonblocking Multileader, 20 leaders: 7.701969e-03 +Nonblocking Locality Aware, 20 leaders: 8.692606e-02 +Nonblocking Multileader Locality, 20 leaders: 1.086531e-03 +Size 32 +PMPI: 4.399972e-03 +Pairwise: 1.266790e-02 +NonBlocking: 1.176694e-01 +Pairwise Hierarchical: 4.236802e-02 +Pairwise Node Aware: 2.304586e-03 +Nonblocking Hierarchical: 3.376797e-02 +Nonblocking Node Aware: 2.067517e-03 +Pairwise Multileader, 4 leaders: 1.381995e-02 +Pairwise Locality Aware, 4 leaders: 1.816843e-03 +Pairwise Multileader Locality, 4 leaders: 1.101817e-02 +Nonblocking Multileader, 4 leaders: 8.668375e-03 +Nonblocking Locality Aware, 4 leaders: 1.459174e-03 +Nonblocking Multileader Locality, 4 leaders: 8.857145e-03 +Pairwise Multileader, 10 leaders: 5.335817e-03 +Pairwise Locality Aware, 10 leaders: 2.823025e-03 +Pairwise Multileader Locality, 10 leaders: 4.740068e-03 +Nonblocking Multileader, 10 leaders: 3.349808e-03 +Nonblocking Locality Aware, 10 leaders: 9.355382e-02 +Nonblocking Multileader Locality, 10 leaders: 3.800382e-03 +Pairwise Multileader, 20 leaders: 4.162088e-03 +Pairwise Locality Aware, 20 leaders: 3.315167e-03 +Pairwise Multileader Locality, 20 leaders: 2.562703e-03 +Nonblocking Multileader, 20 leaders: 1.647736e-02 +Nonblocking Locality Aware, 20 leaders: 9.347074e-02 +Nonblocking Multileader Locality, 20 leaders: 1.992682e-03 +Size 64 +PMPI: 8.697242e-03 +Pairwise: 1.214465e-02 +NonBlocking: 1.185682e-01 +Pairwise Hierarchical: 8.212852e-02 +Pairwise Node Aware: 4.213681e-03 +Nonblocking Hierarchical: 6.453548e-02 +Nonblocking Node Aware: 3.513836e-03 +Pairwise Multileader, 4 leaders: 2.539389e-02 +Pairwise Locality Aware, 4 leaders: 3.505002e-03 +Pairwise Multileader Locality, 4 leaders: 2.212202e-02 +Nonblocking Multileader, 4 leaders: 1.633886e-02 +Nonblocking Locality Aware, 4 leaders: 3.831837e-03 +Nonblocking Multileader Locality, 4 leaders: 1.741961e-02 +Pairwise Multileader, 10 leaders: 1.124783e-02 +Pairwise Locality Aware, 10 leaders: 4.324005e-03 +Pairwise Multileader Locality, 10 leaders: 1.006150e-02 +Nonblocking Multileader, 10 leaders: 7.839897e-03 +Nonblocking Locality Aware, 10 leaders: 8.856915e-02 +Nonblocking Multileader Locality, 10 leaders: 8.473225e-03 +Pairwise Multileader, 20 leaders: 7.872161e-03 +Pairwise Locality Aware, 20 leaders: 4.152626e-03 +Pairwise Multileader Locality, 20 leaders: 5.436173e-03 +Nonblocking Multileader, 20 leaders: 6.187826e-03 +Nonblocking Locality Aware, 20 leaders: 1.161806e-01 +Nonblocking Multileader Locality, 20 leaders: 4.264051e-03 +Size 128 +PMPI: 1.642707e-02 +Pairwise: 1.136302e-02 +NonBlocking: 1.395690e-01 +Pairwise Hierarchical: 1.565105e-01 +Pairwise Node Aware: 8.927714e-03 +Nonblocking Hierarchical: 1.234256e-01 +Nonblocking Node Aware: 7.085583e-03 +Pairwise Multileader, 4 leaders: 5.202922e-02 +Pairwise Locality Aware, 4 leaders: 8.246419e-03 +Pairwise Multileader Locality, 4 leaders: 5.490874e-02 +Nonblocking Multileader, 4 leaders: 3.647775e-02 +Nonblocking Locality Aware, 4 leaders: 8.119370e-03 +Nonblocking Multileader Locality, 4 leaders: 4.122743e-02 +Pairwise Multileader, 10 leaders: 2.487583e-02 +Pairwise Locality Aware, 10 leaders: 1.000314e-02 +Pairwise Multileader Locality, 10 leaders: 2.467004e-02 +Nonblocking Multileader, 10 leaders: 1.760585e-02 +Nonblocking Locality Aware, 10 leaders: 2.156532e-02 +Nonblocking Multileader Locality, 10 leaders: 1.968722e-02 +Pairwise Multileader, 20 leaders: 1.647741e-02 +Pairwise Locality Aware, 20 leaders: 1.100659e-02 +Pairwise Multileader Locality, 20 leaders: 1.264791e-02 +Nonblocking Multileader, 20 leaders: 8.168976e-03 +Nonblocking Locality Aware, 20 leaders: 9.524570e-02 +Nonblocking Multileader Locality, 20 leaders: 9.726965e-03 +Size 256 +PMPI: 3.620148e-02 +Pairwise: 2.166070e-02 +NonBlocking: 1.959571e-01 +Pairwise Hierarchical: 3.057154e-01 +Pairwise Node Aware: 1.344906e-02 +Nonblocking Hierarchical: 2.483800e-01 +Nonblocking Node Aware: 1.387136e-02 +Pairwise Multileader, 4 leaders: 9.456398e-02 +Pairwise Locality Aware, 4 leaders: 1.281594e-02 +Pairwise Multileader Locality, 4 leaders: 1.044530e-01 +Nonblocking Multileader, 4 leaders: 6.899244e-02 +Nonblocking Locality Aware, 4 leaders: 1.063059e-02 +Nonblocking Multileader Locality, 4 leaders: 7.988911e-02 +Pairwise Multileader, 10 leaders: 5.407502e-02 +Pairwise Locality Aware, 10 leaders: 1.529740e-02 +Pairwise Multileader Locality, 10 leaders: 5.091870e-02 +Nonblocking Multileader, 10 leaders: 3.679745e-02 +Nonblocking Locality Aware, 10 leaders: 2.304182e-02 +Nonblocking Multileader Locality, 10 leaders: 4.157429e-02 +Pairwise Multileader, 20 leaders: 2.792569e-02 +Pairwise Locality Aware, 20 leaders: 1.419149e-02 +Pairwise Multileader Locality, 20 leaders: 2.567481e-02 +Nonblocking Multileader, 20 leaders: 1.718773e-02 +Nonblocking Locality Aware, 20 leaders: 5.534666e-02 +Nonblocking Multileader Locality, 20 leaders: 2.063240e-02 +Size 512 +PMPI: 4.811985e-02 +Pairwise: 3.340172e-02 +NonBlocking: 2.054434e-01 +Pairwise Hierarchical: 5.915922e-01 +Pairwise Node Aware: 2.827268e-02 +Nonblocking Hierarchical: 5.317202e-01 +Nonblocking Node Aware: 2.786209e-02 +Pairwise Multileader, 4 leaders: 1.815433e-01 +Pairwise Locality Aware, 4 leaders: 3.183968e-02 +Pairwise Multileader Locality, 4 leaders: 2.015965e-01 +Nonblocking Multileader, 4 leaders: 1.310914e-01 +Nonblocking Locality Aware, 4 leaders: 2.594088e-02 +Nonblocking Multileader Locality, 4 leaders: 1.564615e-01 +Pairwise Multileader, 10 leaders: 1.031070e-01 +Pairwise Locality Aware, 10 leaders: 3.425219e-02 +Pairwise Multileader Locality, 10 leaders: 1.047621e-01 +Nonblocking Multileader, 10 leaders: 7.283069e-02 +Nonblocking Locality Aware, 10 leaders: 3.005283e-02 +Nonblocking Multileader Locality, 10 leaders: 8.608782e-02 +Pairwise Multileader, 20 leaders: 5.757395e-02 +Pairwise Locality Aware, 20 leaders: 3.269456e-02 +Pairwise Multileader Locality, 20 leaders: 5.477726e-02 +Nonblocking Multileader, 20 leaders: 3.674796e-02 +Nonblocking Locality Aware, 20 leaders: 5.508435e-02 +Nonblocking Multileader Locality, 20 leaders: 4.340753e-02 +Test 2 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 4.538737e-04 +Pairwise: 6.779058e-03 +NonBlocking: 5.530908e-02 +Pairwise Hierarchical: 2.487306e-03 +Pairwise Node Aware: 4.137996e-04 +Nonblocking Hierarchical: 9.475253e-04 +Nonblocking Node Aware: 2.495660e-04 +Pairwise Multileader, 4 leaders: 6.557347e-04 +Pairwise Locality Aware, 4 leaders: 6.526911e-04 +Pairwise Multileader Locality, 4 leaders: 4.849918e-04 +Nonblocking Multileader, 4 leaders: 3.014200e-04 +Nonblocking Locality Aware, 4 leaders: 2.336354e-04 +Nonblocking Multileader Locality, 4 leaders: 3.570558e-04 +Pairwise Multileader, 10 leaders: 9.615098e-04 +Pairwise Locality Aware, 10 leaders: 1.006921e-03 +Pairwise Multileader Locality, 10 leaders: 2.978437e-04 +Nonblocking Multileader, 10 leaders: 3.721328e-04 +Nonblocking Locality Aware, 10 leaders: 9.429862e-03 +Nonblocking Multileader Locality, 10 leaders: 1.869862e-04 +Pairwise Multileader, 20 leaders: 2.575303e-03 +Pairwise Locality Aware, 20 leaders: 2.264168e-03 +Pairwise Multileader Locality, 20 leaders: 2.273195e-04 +Nonblocking Multileader, 20 leaders: 6.519834e-04 +Nonblocking Locality Aware, 20 leaders: 7.434392e-02 +Nonblocking Multileader Locality, 20 leaders: 1.473408e-04 +Size 2 +PMPI: 5.428446e-04 +Pairwise: 6.024094e-03 +NonBlocking: 6.991065e-02 +Pairwise Hierarchical: 2.590141e-03 +Pairwise Node Aware: 4.662685e-04 +Nonblocking Hierarchical: 2.178634e-03 +Nonblocking Node Aware: 3.293290e-04 +Pairwise Multileader, 4 leaders: 1.128011e-03 +Pairwise Locality Aware, 4 leaders: 5.839151e-04 +Pairwise Multileader Locality, 4 leaders: 8.293025e-04 +Nonblocking Multileader, 4 leaders: 5.495449e-04 +Nonblocking Locality Aware, 4 leaders: 5.211003e-04 +Nonblocking Multileader Locality, 4 leaders: 6.160794e-04 +Pairwise Multileader, 10 leaders: 1.081497e-03 +Pairwise Locality Aware, 10 leaders: 1.769915e-03 +Pairwise Multileader Locality, 10 leaders: 3.548626e-04 +Nonblocking Multileader, 10 leaders: 2.833696e-03 +Nonblocking Locality Aware, 10 leaders: 6.191701e-02 +Nonblocking Multileader Locality, 10 leaders: 2.654992e-04 +Pairwise Multileader, 20 leaders: 3.294539e-03 +Pairwise Locality Aware, 20 leaders: 2.033174e-03 +Pairwise Multileader Locality, 20 leaders: 2.724131e-04 +Nonblocking Multileader, 20 leaders: 2.159896e-03 +Nonblocking Locality Aware, 20 leaders: 7.428232e-02 +Nonblocking Multileader Locality, 20 leaders: 1.834168e-04 +Size 4 +PMPI: 8.307984e-04 +Pairwise: 1.134325e-02 +NonBlocking: 1.177569e-01 +Pairwise Hierarchical: 4.133496e-03 +Pairwise Node Aware: 5.654517e-04 +Nonblocking Hierarchical: 3.427669e-03 +Nonblocking Node Aware: 4.832495e-04 +Pairwise Multileader, 4 leaders: 1.655393e-03 +Pairwise Locality Aware, 4 leaders: 7.086357e-04 +Pairwise Multileader Locality, 4 leaders: 1.323349e-03 +Nonblocking Multileader, 4 leaders: 9.078416e-04 +Nonblocking Locality Aware, 4 leaders: 3.390081e-04 +Nonblocking Multileader Locality, 4 leaders: 1.058624e-03 +Pairwise Multileader, 10 leaders: 1.727225e-03 +Pairwise Locality Aware, 10 leaders: 1.270533e-03 +Pairwise Multileader Locality, 10 leaders: 6.094714e-04 +Nonblocking Multileader, 10 leaders: 3.179114e-03 +Nonblocking Locality Aware, 10 leaders: 6.208688e-02 +Nonblocking Multileader Locality, 10 leaders: 4.832017e-04 +Pairwise Multileader, 20 leaders: 2.527178e-03 +Pairwise Locality Aware, 20 leaders: 3.305921e-03 +Pairwise Multileader Locality, 20 leaders: 4.762235e-04 +Nonblocking Multileader, 20 leaders: 2.082450e-02 +Nonblocking Locality Aware, 20 leaders: 1.148664e-01 +Nonblocking Multileader Locality, 20 leaders: 3.834983e-04 +Size 8 +PMPI: 1.720482e-03 +Pairwise: 1.263827e-02 +NonBlocking: 9.592376e-02 +Pairwise Hierarchical: 9.533451e-03 +Pairwise Node Aware: 8.901492e-04 +Nonblocking Hierarchical: 8.156724e-03 +Nonblocking Node Aware: 7.963056e-04 +Pairwise Multileader, 4 leaders: 2.987149e-03 +Pairwise Locality Aware, 4 leaders: 9.361978e-04 +Pairwise Multileader Locality, 4 leaders: 3.258034e-03 +Nonblocking Multileader, 4 leaders: 1.972916e-03 +Nonblocking Locality Aware, 4 leaders: 5.462651e-04 +Nonblocking Multileader Locality, 4 leaders: 2.669150e-03 +Pairwise Multileader, 10 leaders: 1.771314e-03 +Pairwise Locality Aware, 10 leaders: 1.460461e-03 +Pairwise Multileader Locality, 10 leaders: 1.353042e-03 +Nonblocking Multileader, 10 leaders: 1.314972e-02 +Nonblocking Locality Aware, 10 leaders: 4.991031e-02 +Nonblocking Multileader Locality, 10 leaders: 9.558421e-04 +Pairwise Multileader, 20 leaders: 2.742494e-03 +Pairwise Locality Aware, 20 leaders: 3.101839e-03 +Pairwise Multileader Locality, 20 leaders: 7.575544e-04 +Nonblocking Multileader, 20 leaders: 1.299853e-02 +Nonblocking Locality Aware, 20 leaders: 9.229251e-02 +Nonblocking Multileader Locality, 20 leaders: 6.406771e-04 +Size 16 +PMPI: 2.846359e-03 +Pairwise: 1.208225e-02 +NonBlocking: 1.176420e-01 +Pairwise Hierarchical: 2.335311e-02 +Pairwise Node Aware: 1.811199e-03 +Nonblocking Hierarchical: 1.654472e-02 +Nonblocking Node Aware: 1.196417e-03 +Pairwise Multileader, 4 leaders: 4.861932e-03 +Pairwise Locality Aware, 4 leaders: 1.454529e-03 +Pairwise Multileader Locality, 4 leaders: 5.954459e-03 +Nonblocking Multileader, 4 leaders: 3.809139e-03 +Nonblocking Locality Aware, 4 leaders: 1.323023e-03 +Nonblocking Multileader Locality, 4 leaders: 4.966127e-03 +Pairwise Multileader, 10 leaders: 3.442355e-03 +Pairwise Locality Aware, 10 leaders: 1.870502e-03 +Pairwise Multileader Locality, 10 leaders: 2.161774e-03 +Nonblocking Multileader, 10 leaders: 2.125759e-03 +Nonblocking Locality Aware, 10 leaders: 8.754209e-02 +Nonblocking Multileader Locality, 10 leaders: 1.810189e-03 +Pairwise Multileader, 20 leaders: 3.350448e-03 +Pairwise Locality Aware, 20 leaders: 2.558074e-03 +Pairwise Multileader Locality, 20 leaders: 1.411808e-03 +Nonblocking Multileader, 20 leaders: 2.675647e-02 +Nonblocking Locality Aware, 20 leaders: 9.236039e-02 +Nonblocking Multileader Locality, 20 leaders: 1.096979e-03 +Size 32 +PMPI: 4.516153e-03 +Pairwise: 1.345576e-02 +NonBlocking: 1.184366e-01 +Pairwise Hierarchical: 4.230387e-02 +Pairwise Node Aware: 2.548682e-03 +Nonblocking Hierarchical: 3.363275e-02 +Nonblocking Node Aware: 2.443417e-03 +Pairwise Multileader, 4 leaders: 1.417224e-02 +Pairwise Locality Aware, 4 leaders: 2.068860e-03 +Pairwise Multileader Locality, 4 leaders: 1.115622e-02 +Nonblocking Multileader, 4 leaders: 8.583413e-03 +Nonblocking Locality Aware, 4 leaders: 2.227289e-03 +Nonblocking Multileader Locality, 4 leaders: 8.761234e-03 +Pairwise Multileader, 10 leaders: 5.838895e-03 +Pairwise Locality Aware, 10 leaders: 3.125341e-03 +Pairwise Multileader Locality, 10 leaders: 6.229586e-03 +Nonblocking Multileader, 10 leaders: 1.675236e-02 +Nonblocking Locality Aware, 10 leaders: 1.164454e-01 +Nonblocking Multileader Locality, 10 leaders: 3.882623e-03 +Pairwise Multileader, 20 leaders: 4.189397e-03 +Pairwise Locality Aware, 20 leaders: 4.845425e-03 +Pairwise Multileader Locality, 20 leaders: 2.573718e-03 +Nonblocking Multileader, 20 leaders: 1.808374e-02 +Nonblocking Locality Aware, 20 leaders: 1.007409e-01 +Nonblocking Multileader Locality, 20 leaders: 1.893118e-03 +Size 64 +PMPI: 9.130303e-03 +Pairwise: 1.412137e-02 +NonBlocking: 1.409043e-01 +Pairwise Hierarchical: 8.180248e-02 +Pairwise Node Aware: 4.258802e-03 +Nonblocking Hierarchical: 6.399348e-02 +Nonblocking Node Aware: 3.544707e-03 +Pairwise Multileader, 4 leaders: 2.775328e-02 +Pairwise Locality Aware, 4 leaders: 4.638232e-03 +Pairwise Multileader Locality, 4 leaders: 2.290795e-02 +Nonblocking Multileader, 4 leaders: 1.638994e-02 +Nonblocking Locality Aware, 4 leaders: 4.770788e-03 +Nonblocking Multileader Locality, 4 leaders: 1.776708e-02 +Pairwise Multileader, 10 leaders: 1.373607e-02 +Pairwise Locality Aware, 10 leaders: 5.696899e-03 +Pairwise Multileader Locality, 10 leaders: 1.102084e-02 +Nonblocking Multileader, 10 leaders: 8.086314e-03 +Nonblocking Locality Aware, 10 leaders: 7.022337e-02 +Nonblocking Multileader Locality, 10 leaders: 8.815089e-03 +Pairwise Multileader, 20 leaders: 1.097840e-02 +Pairwise Locality Aware, 20 leaders: 8.953714e-03 +Pairwise Multileader Locality, 20 leaders: 6.549855e-03 +Nonblocking Multileader, 20 leaders: 2.060046e-02 +Nonblocking Locality Aware, 20 leaders: 1.371408e-01 +Nonblocking Multileader Locality, 20 leaders: 4.155948e-03 +Size 128 +PMPI: 1.640004e-02 +Pairwise: 1.764926e-02 +NonBlocking: 1.388498e-01 +Pairwise Hierarchical: 1.586475e-01 +Pairwise Node Aware: 1.022733e-02 +Nonblocking Hierarchical: 1.240141e-01 +Nonblocking Node Aware: 7.628952e-03 +Pairwise Multileader, 4 leaders: 5.165550e-02 +Pairwise Locality Aware, 4 leaders: 7.682410e-03 +Pairwise Multileader Locality, 4 leaders: 5.524139e-02 +Nonblocking Multileader, 4 leaders: 3.711858e-02 +Nonblocking Locality Aware, 4 leaders: 8.861306e-03 +Nonblocking Multileader Locality, 4 leaders: 4.205706e-02 +Pairwise Multileader, 10 leaders: 2.545453e-02 +Pairwise Locality Aware, 10 leaders: 9.040259e-03 +Pairwise Multileader Locality, 10 leaders: 2.467584e-02 +Nonblocking Multileader, 10 leaders: 1.730027e-02 +Nonblocking Locality Aware, 10 leaders: 1.470511e-02 +Nonblocking Multileader Locality, 10 leaders: 1.957519e-02 +Pairwise Multileader, 20 leaders: 1.412944e-02 +Pairwise Locality Aware, 20 leaders: 9.531524e-03 +Pairwise Multileader Locality, 20 leaders: 1.176668e-02 +Nonblocking Multileader, 20 leaders: 9.761643e-03 +Nonblocking Locality Aware, 20 leaders: 1.203249e-01 +Nonblocking Multileader Locality, 20 leaders: 9.748842e-03 +Size 256 +PMPI: 3.338327e-02 +Pairwise: 2.508443e-02 +NonBlocking: 1.462978e-01 +Pairwise Hierarchical: 3.073555e-01 +Pairwise Node Aware: 1.581782e-02 +Nonblocking Hierarchical: 2.471609e-01 +Nonblocking Node Aware: 1.378703e-02 +Pairwise Multileader, 4 leaders: 9.622253e-02 +Pairwise Locality Aware, 4 leaders: 1.516738e-02 +Pairwise Multileader Locality, 4 leaders: 1.045135e-01 +Nonblocking Multileader, 4 leaders: 7.010880e-02 +Nonblocking Locality Aware, 4 leaders: 1.520386e-02 +Nonblocking Multileader Locality, 4 leaders: 7.955588e-02 +Pairwise Multileader, 10 leaders: 6.184751e-02 +Pairwise Locality Aware, 10 leaders: 1.665518e-02 +Pairwise Multileader Locality, 10 leaders: 5.125067e-02 +Nonblocking Multileader, 10 leaders: 3.682636e-02 +Nonblocking Locality Aware, 10 leaders: 2.661753e-02 +Nonblocking Multileader Locality, 10 leaders: 4.206558e-02 +Pairwise Multileader, 20 leaders: 3.313171e-02 +Pairwise Locality Aware, 20 leaders: 1.990310e-02 +Pairwise Multileader Locality, 20 leaders: 2.588996e-02 +Nonblocking Multileader, 20 leaders: 5.411791e-02 +Nonblocking Locality Aware, 20 leaders: 7.486528e-02 +Nonblocking Multileader Locality, 20 leaders: 2.084529e-02 +Size 512 +PMPI: 4.379466e-02 +Pairwise: 3.462139e-02 +NonBlocking: 2.247827e-01 +Pairwise Hierarchical: 6.020665e-01 +Pairwise Node Aware: 3.277118e-02 +Nonblocking Hierarchical: 5.313575e-01 +Nonblocking Node Aware: 3.057192e-02 +Pairwise Multileader, 4 leaders: 1.840391e-01 +Pairwise Locality Aware, 4 leaders: 3.320577e-02 +Pairwise Multileader Locality, 4 leaders: 2.029076e-01 +Nonblocking Multileader, 4 leaders: 1.314693e-01 +Nonblocking Locality Aware, 4 leaders: 3.053571e-02 +Nonblocking Multileader Locality, 4 leaders: 1.563394e-01 +Pairwise Multileader, 10 leaders: 1.119233e-01 +Pairwise Locality Aware, 10 leaders: 3.806544e-02 +Pairwise Multileader Locality, 10 leaders: 1.046247e-01 +Nonblocking Multileader, 10 leaders: 7.256326e-02 +Nonblocking Locality Aware, 10 leaders: 3.153071e-02 +Nonblocking Multileader Locality, 10 leaders: 8.638805e-02 +Pairwise Multileader, 20 leaders: 5.705854e-02 +Pairwise Locality Aware, 20 leaders: 3.400908e-02 +Pairwise Multileader Locality, 20 leaders: 5.410885e-02 +Nonblocking Multileader, 20 leaders: 3.479698e-02 +Nonblocking Locality Aware, 20 leaders: 4.440748e-02 +Nonblocking Multileader Locality, 20 leaders: 4.357589e-02 +Test 3 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 4.177195e-04 +Pairwise: 5.199548e-03 +NonBlocking: 1.100360e-01 +Pairwise Hierarchical: 1.592366e-03 +Pairwise Node Aware: 4.886602e-04 +Nonblocking Hierarchical: 9.669632e-04 +Nonblocking Node Aware: 2.490696e-04 +Pairwise Multileader, 4 leaders: 7.300141e-04 +Pairwise Locality Aware, 4 leaders: 5.723628e-04 +Pairwise Multileader Locality, 4 leaders: 4.731126e-04 +Nonblocking Multileader, 4 leaders: 3.042079e-04 +Nonblocking Locality Aware, 4 leaders: 2.480673e-04 +Nonblocking Multileader Locality, 4 leaders: 3.638312e-04 +Pairwise Multileader, 10 leaders: 1.452423e-03 +Pairwise Locality Aware, 10 leaders: 1.197064e-03 +Pairwise Multileader Locality, 10 leaders: 2.889861e-04 +Nonblocking Multileader, 10 leaders: 5.073503e-04 +Nonblocking Locality Aware, 10 leaders: 6.193260e-02 +Nonblocking Multileader Locality, 10 leaders: 1.851638e-04 +Pairwise Multileader, 20 leaders: 2.123075e-03 +Pairwise Locality Aware, 20 leaders: 2.422968e-03 +Pairwise Multileader Locality, 20 leaders: 2.859249e-04 +Nonblocking Multileader, 20 leaders: 4.725907e-04 +Nonblocking Locality Aware, 20 leaders: 4.947900e-02 +Nonblocking Multileader Locality, 20 leaders: 1.489342e-04 +Size 2 +PMPI: 4.858686e-04 +Pairwise: 5.825031e-03 +NonBlocking: 7.519179e-02 +Pairwise Hierarchical: 2.718246e-03 +Pairwise Node Aware: 5.424754e-04 +Nonblocking Hierarchical: 2.149297e-03 +Nonblocking Node Aware: 3.267189e-04 +Pairwise Multileader, 4 leaders: 1.105247e-03 +Pairwise Locality Aware, 4 leaders: 8.103267e-04 +Pairwise Multileader Locality, 4 leaders: 8.643405e-04 +Nonblocking Multileader, 4 leaders: 5.451563e-04 +Nonblocking Locality Aware, 4 leaders: 4.253554e-04 +Nonblocking Multileader Locality, 4 leaders: 6.393091e-04 +Pairwise Multileader, 10 leaders: 1.063266e-03 +Pairwise Locality Aware, 10 leaders: 1.200477e-03 +Pairwise Multileader Locality, 10 leaders: 3.861480e-04 +Nonblocking Multileader, 10 leaders: 4.707489e-04 +Nonblocking Locality Aware, 10 leaders: 7.423120e-02 +Nonblocking Multileader Locality, 10 leaders: 2.792720e-04 +Pairwise Multileader, 20 leaders: 2.088824e-03 +Pairwise Locality Aware, 20 leaders: 2.463743e-03 +Pairwise Multileader Locality, 20 leaders: 3.333911e-04 +Nonblocking Multileader, 20 leaders: 7.522634e-04 +Nonblocking Locality Aware, 20 leaders: 7.428993e-02 +Nonblocking Multileader Locality, 20 leaders: 2.276940e-04 +Size 4 +PMPI: 8.629913e-04 +Pairwise: 1.424771e-02 +NonBlocking: 1.182462e-01 +Pairwise Hierarchical: 3.974227e-03 +Pairwise Node Aware: 5.762763e-04 +Nonblocking Hierarchical: 3.410554e-03 +Nonblocking Node Aware: 5.096489e-04 +Pairwise Multileader, 4 leaders: 1.463766e-03 +Pairwise Locality Aware, 4 leaders: 7.147027e-04 +Pairwise Multileader Locality, 4 leaders: 1.305748e-03 +Nonblocking Multileader, 4 leaders: 8.919045e-04 +Nonblocking Locality Aware, 4 leaders: 4.748210e-04 +Nonblocking Multileader Locality, 4 leaders: 1.058784e-03 +Pairwise Multileader, 10 leaders: 1.246617e-03 +Pairwise Locality Aware, 10 leaders: 1.583577e-03 +Pairwise Multileader Locality, 10 leaders: 5.766123e-04 +Nonblocking Multileader, 10 leaders: 1.534544e-03 +Nonblocking Locality Aware, 10 leaders: 8.664577e-02 +Nonblocking Multileader Locality, 10 leaders: 4.750576e-04 +Pairwise Multileader, 20 leaders: 2.747437e-03 +Pairwise Locality Aware, 20 leaders: 3.011540e-03 +Pairwise Multileader Locality, 20 leaders: 4.403552e-04 +Nonblocking Multileader, 20 leaders: 6.527307e-03 +Nonblocking Locality Aware, 20 leaders: 1.150737e-01 +Nonblocking Multileader Locality, 20 leaders: 3.478768e-04 +Size 8 +PMPI: 1.383073e-03 +Pairwise: 1.209230e-02 +NonBlocking: 1.176391e-01 +Pairwise Hierarchical: 9.703655e-03 +Pairwise Node Aware: 1.097592e-03 +Nonblocking Hierarchical: 8.185509e-03 +Nonblocking Node Aware: 8.704872e-04 +Pairwise Multileader, 4 leaders: 2.847978e-03 +Pairwise Locality Aware, 4 leaders: 1.009959e-03 +Pairwise Multileader Locality, 4 leaders: 3.320456e-03 +Nonblocking Multileader, 4 leaders: 1.988465e-03 +Nonblocking Locality Aware, 4 leaders: 7.998742e-04 +Nonblocking Multileader Locality, 4 leaders: 2.559378e-03 +Pairwise Multileader, 10 leaders: 3.145339e-03 +Pairwise Locality Aware, 10 leaders: 1.381897e-03 +Pairwise Multileader Locality, 10 leaders: 1.207761e-03 +Nonblocking Multileader, 10 leaders: 1.123892e-03 +Nonblocking Locality Aware, 10 leaders: 9.912500e-02 +Nonblocking Multileader Locality, 10 leaders: 9.959667e-04 +Pairwise Multileader, 20 leaders: 2.826962e-03 +Pairwise Locality Aware, 20 leaders: 2.720941e-03 +Pairwise Multileader Locality, 20 leaders: 7.736871e-04 +Nonblocking Multileader, 20 leaders: 2.105274e-02 +Nonblocking Locality Aware, 20 leaders: 1.150358e-01 +Nonblocking Multileader Locality, 20 leaders: 5.912624e-04 +Size 16 +PMPI: 2.731414e-03 +Pairwise: 1.378399e-02 +NonBlocking: 1.172609e-01 +Pairwise Hierarchical: 2.048913e-02 +Pairwise Node Aware: 1.206200e-03 +Nonblocking Hierarchical: 1.658036e-02 +Nonblocking Node Aware: 1.104238e-03 +Pairwise Multileader, 4 leaders: 5.139672e-03 +Pairwise Locality Aware, 4 leaders: 1.130417e-03 +Pairwise Multileader Locality, 4 leaders: 6.011454e-03 +Nonblocking Multileader, 4 leaders: 3.903029e-03 +Nonblocking Locality Aware, 4 leaders: 7.464127e-04 +Nonblocking Multileader Locality, 4 leaders: 4.743969e-03 +Pairwise Multileader, 10 leaders: 3.263687e-03 +Pairwise Locality Aware, 10 leaders: 1.891159e-03 +Pairwise Multileader Locality, 10 leaders: 2.184148e-03 +Nonblocking Multileader, 10 leaders: 1.589269e-03 +Nonblocking Locality Aware, 10 leaders: 9.249037e-02 +Nonblocking Multileader Locality, 10 leaders: 1.905348e-03 +Pairwise Multileader, 20 leaders: 3.358955e-03 +Pairwise Locality Aware, 20 leaders: 3.148463e-03 +Pairwise Multileader Locality, 20 leaders: 1.433196e-03 +Nonblocking Multileader, 20 leaders: 7.462538e-02 +Nonblocking Locality Aware, 20 leaders: 9.949609e-02 +Nonblocking Multileader Locality, 20 leaders: 1.142453e-03 +Size 32 +PMPI: 5.021963e-03 +Pairwise: 2.851689e-02 +NonBlocking: 1.182844e-01 +Pairwise Hierarchical: 4.353691e-02 +Pairwise Node Aware: 2.658576e-03 +Nonblocking Hierarchical: 3.378177e-02 +Nonblocking Node Aware: 2.531294e-03 +Pairwise Multileader, 4 leaders: 1.518369e-02 +Pairwise Locality Aware, 4 leaders: 3.066144e-03 +Pairwise Multileader Locality, 4 leaders: 1.237876e-02 +Nonblocking Multileader, 4 leaders: 8.688037e-03 +Nonblocking Locality Aware, 4 leaders: 2.563791e-03 +Nonblocking Multileader Locality, 4 leaders: 8.898357e-03 +Pairwise Multileader, 10 leaders: 6.895398e-03 +Pairwise Locality Aware, 10 leaders: 5.503507e-03 +Pairwise Multileader Locality, 10 leaders: 5.330689e-03 +Nonblocking Multileader, 10 leaders: 6.055794e-03 +Nonblocking Locality Aware, 10 leaders: 1.159301e-01 +Nonblocking Multileader Locality, 10 leaders: 3.911642e-03 +Pairwise Multileader, 20 leaders: 5.852261e-03 +Pairwise Locality Aware, 20 leaders: 3.613342e-03 +Pairwise Multileader Locality, 20 leaders: 2.711374e-03 +Nonblocking Multileader, 20 leaders: 1.961653e-03 +Nonblocking Locality Aware, 20 leaders: 9.349319e-02 +Nonblocking Multileader Locality, 20 leaders: 2.241040e-03 +Size 64 +PMPI: 9.184661e-03 +Pairwise: 1.449058e-02 +NonBlocking: 1.197476e-01 +Pairwise Hierarchical: 8.250996e-02 +Pairwise Node Aware: 4.324422e-03 +Nonblocking Hierarchical: 6.447028e-02 +Nonblocking Node Aware: 3.479407e-03 +Pairwise Multileader, 4 leaders: 2.607202e-02 +Pairwise Locality Aware, 4 leaders: 3.800045e-03 +Pairwise Multileader Locality, 4 leaders: 2.238377e-02 +Nonblocking Multileader, 4 leaders: 1.654057e-02 +Nonblocking Locality Aware, 4 leaders: 4.580665e-03 +Nonblocking Multileader Locality, 4 leaders: 1.750441e-02 +Pairwise Multileader, 10 leaders: 1.181180e-02 +Pairwise Locality Aware, 10 leaders: 4.439063e-03 +Pairwise Multileader Locality, 10 leaders: 1.121437e-02 +Nonblocking Multileader, 10 leaders: 7.963704e-03 +Nonblocking Locality Aware, 10 leaders: 7.687413e-02 +Nonblocking Multileader Locality, 10 leaders: 8.593714e-03 +Pairwise Multileader, 20 leaders: 7.840646e-03 +Pairwise Locality Aware, 20 leaders: 8.533049e-03 +Pairwise Multileader Locality, 20 leaders: 5.500784e-03 +Nonblocking Multileader, 20 leaders: 7.197084e-03 +Nonblocking Locality Aware, 20 leaders: 1.397069e-01 +Nonblocking Multileader Locality, 20 leaders: 4.263296e-03 +Size 128 +PMPI: 1.614850e-02 +Pairwise: 1.635382e-02 +NonBlocking: 1.340253e-01 +Pairwise Hierarchical: 1.586715e-01 +Pairwise Node Aware: 9.629359e-03 +Nonblocking Hierarchical: 1.240625e-01 +Nonblocking Node Aware: 7.829033e-03 +Pairwise Multileader, 4 leaders: 5.135338e-02 +Pairwise Locality Aware, 4 leaders: 7.685689e-03 +Pairwise Multileader Locality, 4 leaders: 5.476468e-02 +Nonblocking Multileader, 4 leaders: 3.643061e-02 +Nonblocking Locality Aware, 4 leaders: 9.075251e-03 +Nonblocking Multileader Locality, 4 leaders: 4.147502e-02 +Pairwise Multileader, 10 leaders: 2.310876e-02 +Pairwise Locality Aware, 10 leaders: 9.426657e-03 +Pairwise Multileader Locality, 10 leaders: 3.036257e-02 +Nonblocking Multileader, 10 leaders: 1.760695e-02 +Nonblocking Locality Aware, 10 leaders: 1.801661e-02 +Nonblocking Multileader Locality, 10 leaders: 1.965770e-02 +Pairwise Multileader, 20 leaders: 1.481874e-02 +Pairwise Locality Aware, 20 leaders: 7.913439e-03 +Pairwise Multileader Locality, 20 leaders: 1.160327e-02 +Nonblocking Multileader, 20 leaders: 1.893859e-02 +Nonblocking Locality Aware, 20 leaders: 1.187802e-01 +Nonblocking Multileader Locality, 20 leaders: 9.523601e-03 +Size 256 +PMPI: 3.201250e-02 +Pairwise: 2.644237e-02 +NonBlocking: 1.461284e-01 +Pairwise Hierarchical: 3.017678e-01 +Pairwise Node Aware: 1.389047e-02 +Nonblocking Hierarchical: 2.442938e-01 +Nonblocking Node Aware: 1.371604e-02 +Pairwise Multileader, 4 leaders: 9.465384e-02 +Pairwise Locality Aware, 4 leaders: 1.454488e-02 +Pairwise Multileader Locality, 4 leaders: 1.040869e-01 +Nonblocking Multileader, 4 leaders: 6.817821e-02 +Nonblocking Locality Aware, 4 leaders: 1.329863e-02 +Nonblocking Multileader Locality, 4 leaders: 7.985973e-02 +Pairwise Multileader, 10 leaders: 5.471322e-02 +Pairwise Locality Aware, 10 leaders: 1.509602e-02 +Pairwise Multileader Locality, 10 leaders: 5.113584e-02 +Nonblocking Multileader, 10 leaders: 3.700539e-02 +Nonblocking Locality Aware, 10 leaders: 2.316310e-02 +Nonblocking Multileader Locality, 10 leaders: 4.196643e-02 +Pairwise Multileader, 20 leaders: 3.172179e-02 +Pairwise Locality Aware, 20 leaders: 1.889130e-02 +Pairwise Multileader Locality, 20 leaders: 2.925860e-02 +Nonblocking Multileader, 20 leaders: 1.882508e-02 +Nonblocking Locality Aware, 20 leaders: 8.116269e-02 +Nonblocking Multileader Locality, 20 leaders: 2.059346e-02 +Size 512 +PMPI: 5.841433e-02 +Pairwise: 3.687600e-02 +NonBlocking: 1.774727e-01 +Pairwise Hierarchical: 6.047225e-01 +Pairwise Node Aware: 2.838240e-02 +Nonblocking Hierarchical: 5.361196e-01 +Nonblocking Node Aware: 2.833062e-02 +Pairwise Multileader, 4 leaders: 1.807650e-01 +Pairwise Locality Aware, 4 leaders: 3.230774e-02 +Pairwise Multileader Locality, 4 leaders: 2.029944e-01 +Nonblocking Multileader, 4 leaders: 1.310674e-01 +Nonblocking Locality Aware, 4 leaders: 3.094785e-02 +Nonblocking Multileader Locality, 4 leaders: 1.552704e-01 +Pairwise Multileader, 10 leaders: 1.031391e-01 +Pairwise Locality Aware, 10 leaders: 3.206278e-02 +Pairwise Multileader Locality, 10 leaders: 1.058076e-01 +Nonblocking Multileader, 10 leaders: 7.424508e-02 +Nonblocking Locality Aware, 10 leaders: 2.957752e-02 +Nonblocking Multileader Locality, 10 leaders: 8.623403e-02 +Pairwise Multileader, 20 leaders: 5.506940e-02 +Pairwise Locality Aware, 20 leaders: 3.355879e-02 +Pairwise Multileader Locality, 20 leaders: 5.349588e-02 +Nonblocking Multileader, 20 leaders: 3.414221e-02 +Nonblocking Locality Aware, 20 leaders: 3.949515e-02 +Nonblocking Multileader Locality, 20 leaders: 4.340511e-02 diff --git a/runscripts/dane_results/alltoall_N16.2484417.err b/runscripts/dane_results/alltoall_N16.2484417.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/dane_results/alltoall_N16.2484417.out b/runscripts/dane_results/alltoall_N16.2484417.out new file mode 100644 index 000000000..71ef72527 --- /dev/null +++ b/runscripts/dane_results/alltoall_N16.2484417.out @@ -0,0 +1,783 @@ +Test 1 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 3.951049e-04 +Pairwise: 9.095476e-03 +NonBlocking: 8.182989e-02 +Pairwise Hierarchical: 1.295555e-03 +Pairwise Node Aware: 6.614268e-04 +Nonblocking Hierarchical: 9.824945e-04 +Nonblocking Node Aware: 2.434814e-04 +Pairwise Multileader, 4 leaders: 6.881726e-04 +Pairwise Locality Aware, 4 leaders: 6.288571e-04 +Pairwise Multileader Locality, 4 leaders: 5.386971e-04 +Nonblocking Multileader, 4 leaders: 3.019856e-04 +Nonblocking Locality Aware, 4 leaders: 2.263584e-04 +Nonblocking Multileader Locality, 4 leaders: 3.661818e-04 +Pairwise Multileader, 10 leaders: 2.726664e-03 +Pairwise Locality Aware, 10 leaders: 3.002324e-03 +Pairwise Multileader Locality, 10 leaders: 3.986664e-04 +Nonblocking Multileader, 10 leaders: 6.963677e-04 +Nonblocking Locality Aware, 10 leaders: 2.366491e-02 +Nonblocking Multileader Locality, 10 leaders: 1.863901e-04 +Pairwise Multileader, 20 leaders: 2.970044e-03 +Pairwise Locality Aware, 20 leaders: 2.903453e-03 +Pairwise Multileader Locality, 20 leaders: 3.874753e-04 +Nonblocking Multileader, 20 leaders: 1.048084e-02 +Nonblocking Locality Aware, 20 leaders: 9.194405e-02 +Nonblocking Multileader Locality, 20 leaders: 1.360560e-04 +Size 2 +PMPI: 4.586353e-04 +Pairwise: 6.223047e-03 +NonBlocking: 4.259344e-02 +Pairwise Hierarchical: 2.736758e-03 +Pairwise Node Aware: 5.658492e-04 +Nonblocking Hierarchical: 2.155429e-03 +Nonblocking Node Aware: 3.206031e-04 +Pairwise Multileader, 4 leaders: 9.607021e-04 +Pairwise Locality Aware, 4 leaders: 8.101216e-04 +Pairwise Multileader Locality, 4 leaders: 8.278184e-04 +Nonblocking Multileader, 4 leaders: 5.536916e-04 +Nonblocking Locality Aware, 4 leaders: 4.314728e-04 +Nonblocking Multileader Locality, 4 leaders: 6.342798e-04 +Pairwise Multileader, 10 leaders: 1.201799e-03 +Pairwise Locality Aware, 10 leaders: 1.508152e-03 +Pairwise Multileader Locality, 10 leaders: 3.569228e-04 +Nonblocking Multileader, 10 leaders: 1.709379e-03 +Nonblocking Locality Aware, 10 leaders: 7.425704e-02 +Nonblocking Multileader Locality, 10 leaders: 2.600751e-04 +Pairwise Multileader, 20 leaders: 2.190349e-03 +Pairwise Locality Aware, 20 leaders: 2.193894e-03 +Pairwise Multileader Locality, 20 leaders: 3.451284e-04 +Nonblocking Multileader, 20 leaders: 1.030378e-02 +Nonblocking Locality Aware, 20 leaders: 7.232953e-02 +Nonblocking Multileader Locality, 20 leaders: 1.798014e-04 +Size 4 +PMPI: 7.788875e-04 +Pairwise: 9.822717e-03 +NonBlocking: 1.171398e-01 +Pairwise Hierarchical: 4.017139e-03 +Pairwise Node Aware: 6.241035e-04 +Nonblocking Hierarchical: 3.400718e-03 +Nonblocking Node Aware: 5.245189e-04 +Pairwise Multileader, 4 leaders: 1.412292e-03 +Pairwise Locality Aware, 4 leaders: 6.435435e-04 +Pairwise Multileader Locality, 4 leaders: 1.289161e-03 +Nonblocking Multileader, 4 leaders: 9.108889e-04 +Nonblocking Locality Aware, 4 leaders: 3.864291e-04 +Nonblocking Multileader Locality, 4 leaders: 1.038032e-03 +Pairwise Multileader, 10 leaders: 1.317525e-03 +Pairwise Locality Aware, 10 leaders: 1.569034e-03 +Pairwise Multileader Locality, 10 leaders: 5.763485e-04 +Nonblocking Multileader, 10 leaders: 3.108323e-03 +Nonblocking Locality Aware, 10 leaders: 7.435087e-02 +Nonblocking Multileader Locality, 10 leaders: 4.733013e-04 +Pairwise Multileader, 20 leaders: 1.842118e-03 +Pairwise Locality Aware, 20 leaders: 1.961481e-03 +Pairwise Multileader Locality, 20 leaders: 4.340486e-04 +Nonblocking Multileader, 20 leaders: 1.275543e-02 +Nonblocking Locality Aware, 20 leaders: 7.425556e-02 +Nonblocking Multileader Locality, 20 leaders: 2.822154e-04 +Size 8 +PMPI: 1.305092e-03 +Pairwise: 8.527333e-03 +NonBlocking: 1.175179e-01 +Pairwise Hierarchical: 9.666265e-03 +Pairwise Node Aware: 9.001856e-04 +Nonblocking Hierarchical: 8.163929e-03 +Nonblocking Node Aware: 8.704215e-04 +Pairwise Multileader, 4 leaders: 2.719600e-03 +Pairwise Locality Aware, 4 leaders: 7.224571e-04 +Pairwise Multileader Locality, 4 leaders: 3.315559e-03 +Nonblocking Multileader, 4 leaders: 2.057315e-03 +Nonblocking Locality Aware, 4 leaders: 6.006306e-04 +Nonblocking Multileader Locality, 4 leaders: 2.572758e-03 +Pairwise Multileader, 10 leaders: 2.321874e-03 +Pairwise Locality Aware, 10 leaders: 1.261734e-03 +Pairwise Multileader Locality, 10 leaders: 1.197442e-03 +Nonblocking Multileader, 10 leaders: 1.153445e-03 +Nonblocking Locality Aware, 10 leaders: 7.449711e-02 +Nonblocking Multileader Locality, 10 leaders: 9.788363e-04 +Pairwise Multileader, 20 leaders: 3.233499e-03 +Pairwise Locality Aware, 20 leaders: 2.604508e-03 +Pairwise Multileader Locality, 20 leaders: 7.985782e-04 +Nonblocking Multileader, 20 leaders: 1.901957e-02 +Nonblocking Locality Aware, 20 leaders: 1.116651e-01 +Nonblocking Multileader Locality, 20 leaders: 6.741051e-04 +Size 16 +PMPI: 2.656313e-03 +Pairwise: 1.239874e-02 +NonBlocking: 1.172044e-01 +Pairwise Hierarchical: 2.041276e-02 +Pairwise Node Aware: 1.187916e-03 +Nonblocking Hierarchical: 1.649066e-02 +Nonblocking Node Aware: 1.032410e-03 +Pairwise Multileader, 4 leaders: 4.757036e-03 +Pairwise Locality Aware, 4 leaders: 1.232588e-03 +Pairwise Multileader Locality, 4 leaders: 5.813412e-03 +Nonblocking Multileader, 4 leaders: 3.674594e-03 +Nonblocking Locality Aware, 4 leaders: 9.976517e-04 +Nonblocking Multileader Locality, 4 leaders: 4.677804e-03 +Pairwise Multileader, 10 leaders: 3.293521e-03 +Pairwise Locality Aware, 10 leaders: 2.152274e-03 +Pairwise Multileader Locality, 10 leaders: 2.222122e-03 +Nonblocking Multileader, 10 leaders: 3.201425e-03 +Nonblocking Locality Aware, 10 leaders: 8.729966e-02 +Nonblocking Multileader Locality, 10 leaders: 1.725423e-03 +Pairwise Multileader, 20 leaders: 3.445499e-03 +Pairwise Locality Aware, 20 leaders: 3.480772e-03 +Pairwise Multileader Locality, 20 leaders: 1.397844e-03 +Nonblocking Multileader, 20 leaders: 3.703098e-02 +Nonblocking Locality Aware, 20 leaders: 1.150818e-01 +Nonblocking Multileader Locality, 20 leaders: 1.021381e-03 +Size 32 +PMPI: 4.104214e-03 +Pairwise: 1.433682e-02 +NonBlocking: 1.179032e-01 +Pairwise Hierarchical: 4.233449e-02 +Pairwise Node Aware: 2.248240e-03 +Nonblocking Hierarchical: 3.351242e-02 +Nonblocking Node Aware: 2.056195e-03 +Pairwise Multileader, 4 leaders: 1.391551e-02 +Pairwise Locality Aware, 4 leaders: 1.775898e-03 +Pairwise Multileader Locality, 4 leaders: 1.102334e-02 +Nonblocking Multileader, 4 leaders: 8.738717e-03 +Nonblocking Locality Aware, 4 leaders: 1.455732e-03 +Nonblocking Multileader Locality, 4 leaders: 8.728629e-03 +Pairwise Multileader, 10 leaders: 6.243661e-03 +Pairwise Locality Aware, 10 leaders: 2.327329e-03 +Pairwise Multileader Locality, 10 leaders: 4.715381e-03 +Nonblocking Multileader, 10 leaders: 3.848552e-03 +Nonblocking Locality Aware, 10 leaders: 8.774136e-02 +Nonblocking Multileader Locality, 10 leaders: 3.899478e-03 +Pairwise Multileader, 20 leaders: 5.965572e-03 +Pairwise Locality Aware, 20 leaders: 3.081376e-03 +Pairwise Multileader Locality, 20 leaders: 2.726184e-03 +Nonblocking Multileader, 20 leaders: 3.129587e-02 +Nonblocking Locality Aware, 20 leaders: 1.161587e-01 +Nonblocking Multileader Locality, 20 leaders: 2.257448e-03 +Size 64 +PMPI: 8.033282e-03 +Pairwise: 1.595333e-02 +NonBlocking: 1.188665e-01 +Pairwise Hierarchical: 8.197623e-02 +Pairwise Node Aware: 3.916964e-03 +Nonblocking Hierarchical: 6.476970e-02 +Nonblocking Node Aware: 3.141523e-03 +Pairwise Multileader, 4 leaders: 2.625294e-02 +Pairwise Locality Aware, 4 leaders: 4.410295e-03 +Pairwise Multileader Locality, 4 leaders: 2.322919e-02 +Nonblocking Multileader, 4 leaders: 1.708181e-02 +Nonblocking Locality Aware, 4 leaders: 5.101248e-03 +Nonblocking Multileader Locality, 4 leaders: 1.789756e-02 +Pairwise Multileader, 10 leaders: 1.199349e-02 +Pairwise Locality Aware, 10 leaders: 4.665805e-03 +Pairwise Multileader Locality, 10 leaders: 1.029562e-02 +Nonblocking Multileader, 10 leaders: 7.948061e-03 +Nonblocking Locality Aware, 10 leaders: 7.655259e-02 +Nonblocking Multileader Locality, 10 leaders: 8.709503e-03 +Pairwise Multileader, 20 leaders: 9.764206e-03 +Pairwise Locality Aware, 20 leaders: 5.804421e-03 +Pairwise Multileader Locality, 20 leaders: 5.544197e-03 +Nonblocking Multileader, 20 leaders: 5.854357e-03 +Nonblocking Locality Aware, 20 leaders: 1.165235e-01 +Nonblocking Multileader Locality, 20 leaders: 4.358551e-03 +Size 128 +PMPI: 1.409557e-02 +Pairwise: 2.213366e-02 +NonBlocking: 1.140479e-01 +Pairwise Hierarchical: 1.570107e-01 +Pairwise Node Aware: 9.202944e-03 +Nonblocking Hierarchical: 1.234139e-01 +Nonblocking Node Aware: 7.197472e-03 +Pairwise Multileader, 4 leaders: 5.198888e-02 +Pairwise Locality Aware, 4 leaders: 6.987082e-03 +Pairwise Multileader Locality, 4 leaders: 5.511961e-02 +Nonblocking Multileader, 4 leaders: 3.714114e-02 +Nonblocking Locality Aware, 4 leaders: 7.739816e-03 +Nonblocking Multileader Locality, 4 leaders: 4.208668e-02 +Pairwise Multileader, 10 leaders: 2.283428e-02 +Pairwise Locality Aware, 10 leaders: 8.328849e-03 +Pairwise Multileader Locality, 10 leaders: 2.387542e-02 +Nonblocking Multileader, 10 leaders: 1.744543e-02 +Nonblocking Locality Aware, 10 leaders: 9.863438e-03 +Nonblocking Multileader Locality, 10 leaders: 1.968092e-02 +Pairwise Multileader, 20 leaders: 1.392461e-02 +Pairwise Locality Aware, 20 leaders: 9.659618e-03 +Pairwise Multileader Locality, 20 leaders: 1.217481e-02 +Nonblocking Multileader, 20 leaders: 1.061585e-02 +Nonblocking Locality Aware, 20 leaders: 8.382537e-02 +Nonblocking Multileader Locality, 20 leaders: 9.630367e-03 +Size 256 +PMPI: 2.781417e-02 +Pairwise: 2.407244e-02 +NonBlocking: 1.193220e-01 +Pairwise Hierarchical: 3.032682e-01 +Pairwise Node Aware: 1.405296e-02 +Nonblocking Hierarchical: 2.435504e-01 +Nonblocking Node Aware: 1.346725e-02 +Pairwise Multileader, 4 leaders: 9.407514e-02 +Pairwise Locality Aware, 4 leaders: 1.359424e-02 +Pairwise Multileader Locality, 4 leaders: 1.033094e-01 +Nonblocking Multileader, 4 leaders: 6.876006e-02 +Nonblocking Locality Aware, 4 leaders: 1.043125e-02 +Nonblocking Multileader Locality, 4 leaders: 7.976840e-02 +Pairwise Multileader, 10 leaders: 5.341832e-02 +Pairwise Locality Aware, 10 leaders: 1.433401e-02 +Pairwise Multileader Locality, 10 leaders: 5.150739e-02 +Nonblocking Multileader, 10 leaders: 3.733702e-02 +Nonblocking Locality Aware, 10 leaders: 1.909658e-02 +Nonblocking Multileader Locality, 10 leaders: 4.242185e-02 +Pairwise Multileader, 20 leaders: 2.954890e-02 +Pairwise Locality Aware, 20 leaders: 1.549805e-02 +Pairwise Multileader Locality, 20 leaders: 2.605435e-02 +Nonblocking Multileader, 20 leaders: 1.702528e-02 +Nonblocking Locality Aware, 20 leaders: 4.937324e-02 +Nonblocking Multileader Locality, 20 leaders: 2.058464e-02 +Size 512 +PMPI: 3.777492e-02 +Pairwise: 3.768837e-02 +NonBlocking: 1.551146e-01 +Pairwise Hierarchical: 6.045980e-01 +Pairwise Node Aware: 3.145506e-02 +Nonblocking Hierarchical: 5.361089e-01 +Nonblocking Node Aware: 2.934673e-02 +Pairwise Multileader, 4 leaders: 1.802948e-01 +Pairwise Locality Aware, 4 leaders: 3.177396e-02 +Pairwise Multileader Locality, 4 leaders: 2.032706e-01 +Nonblocking Multileader, 4 leaders: 1.319907e-01 +Nonblocking Locality Aware, 4 leaders: 2.280456e-02 +Nonblocking Multileader Locality, 4 leaders: 1.567226e-01 +Pairwise Multileader, 10 leaders: 1.033143e-01 +Pairwise Locality Aware, 10 leaders: 3.287865e-02 +Pairwise Multileader Locality, 10 leaders: 1.052541e-01 +Nonblocking Multileader, 10 leaders: 7.297264e-02 +Nonblocking Locality Aware, 10 leaders: 2.624621e-02 +Nonblocking Multileader Locality, 10 leaders: 8.673108e-02 +Pairwise Multileader, 20 leaders: 5.728719e-02 +Pairwise Locality Aware, 20 leaders: 3.367393e-02 +Pairwise Multileader Locality, 20 leaders: 5.331614e-02 +Nonblocking Multileader, 20 leaders: 3.318000e-02 +Nonblocking Locality Aware, 20 leaders: 3.830180e-02 +Nonblocking Multileader Locality, 20 leaders: 4.333425e-02 +Test 2 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 4.466031e-04 +Pairwise: 5.923330e-03 +NonBlocking: 6.888096e-02 +Pairwise Hierarchical: 1.092794e-03 +Pairwise Node Aware: 4.066509e-04 +Nonblocking Hierarchical: 9.490241e-04 +Nonblocking Node Aware: 2.443223e-04 +Pairwise Multileader, 4 leaders: 6.366004e-04 +Pairwise Locality Aware, 4 leaders: 5.829602e-04 +Pairwise Multileader Locality, 4 leaders: 4.641380e-04 +Nonblocking Multileader, 4 leaders: 3.009659e-04 +Nonblocking Locality Aware, 4 leaders: 2.618865e-04 +Nonblocking Multileader Locality, 4 leaders: 3.547558e-04 +Pairwise Multileader, 10 leaders: 1.041648e-03 +Pairwise Locality Aware, 10 leaders: 1.168286e-03 +Pairwise Multileader Locality, 10 leaders: 2.989563e-04 +Nonblocking Multileader, 10 leaders: 4.468355e-04 +Nonblocking Locality Aware, 10 leaders: 1.755713e-02 +Nonblocking Multileader Locality, 10 leaders: 1.802882e-04 +Pairwise Multileader, 20 leaders: 1.733928e-03 +Pairwise Locality Aware, 20 leaders: 2.168019e-03 +Pairwise Multileader Locality, 20 leaders: 2.690534e-04 +Nonblocking Multileader, 20 leaders: 3.303316e-03 +Nonblocking Locality Aware, 20 leaders: 8.666856e-02 +Nonblocking Multileader Locality, 20 leaders: 1.405825e-04 +Size 2 +PMPI: 4.673293e-04 +Pairwise: 5.012283e-03 +NonBlocking: 4.269306e-02 +Pairwise Hierarchical: 2.571566e-03 +Pairwise Node Aware: 4.458577e-04 +Nonblocking Hierarchical: 2.182860e-03 +Nonblocking Node Aware: 3.491361e-04 +Pairwise Multileader, 4 leaders: 9.747037e-04 +Pairwise Locality Aware, 4 leaders: 6.842090e-04 +Pairwise Multileader Locality, 4 leaders: 8.303007e-04 +Nonblocking Multileader, 4 leaders: 5.519832e-04 +Nonblocking Locality Aware, 4 leaders: 3.521240e-04 +Nonblocking Multileader Locality, 4 leaders: 6.333597e-04 +Pairwise Multileader, 10 leaders: 1.251125e-03 +Pairwise Locality Aware, 10 leaders: 1.392978e-03 +Pairwise Multileader Locality, 10 leaders: 3.369222e-04 +Nonblocking Multileader, 10 leaders: 9.892159e-04 +Nonblocking Locality Aware, 10 leaders: 8.664392e-02 +Nonblocking Multileader Locality, 10 leaders: 2.563576e-04 +Pairwise Multileader, 20 leaders: 1.926417e-03 +Pairwise Locality Aware, 20 leaders: 2.716425e-03 +Pairwise Multileader Locality, 20 leaders: 2.932854e-04 +Nonblocking Multileader, 20 leaders: 8.051278e-03 +Nonblocking Locality Aware, 20 leaders: 6.208077e-02 +Nonblocking Multileader Locality, 20 leaders: 1.780635e-04 +Size 4 +PMPI: 7.965613e-04 +Pairwise: 1.065355e-02 +NonBlocking: 1.633314e-01 +Pairwise Hierarchical: 4.103042e-03 +Pairwise Node Aware: 6.144065e-04 +Nonblocking Hierarchical: 3.516447e-03 +Nonblocking Node Aware: 4.706322e-04 +Pairwise Multileader, 4 leaders: 1.500050e-03 +Pairwise Locality Aware, 4 leaders: 6.453033e-04 +Pairwise Multileader Locality, 4 leaders: 1.411099e-03 +Nonblocking Multileader, 4 leaders: 9.085494e-04 +Nonblocking Locality Aware, 4 leaders: 3.232814e-04 +Nonblocking Multileader Locality, 4 leaders: 1.157201e-03 +Pairwise Multileader, 10 leaders: 1.411628e-03 +Pairwise Locality Aware, 10 leaders: 1.563828e-03 +Pairwise Multileader Locality, 10 leaders: 5.712308e-04 +Nonblocking Multileader, 10 leaders: 1.620095e-03 +Nonblocking Locality Aware, 10 leaders: 7.440046e-02 +Nonblocking Multileader Locality, 10 leaders: 4.953237e-04 +Pairwise Multileader, 20 leaders: 1.998654e-03 +Pairwise Locality Aware, 20 leaders: 2.802080e-03 +Pairwise Multileader Locality, 20 leaders: 4.557754e-04 +Nonblocking Multileader, 20 leaders: 3.742312e-02 +Nonblocking Locality Aware, 20 leaders: 1.113018e-01 +Nonblocking Multileader Locality, 20 leaders: 3.575743e-04 +Size 8 +PMPI: 1.274262e-03 +Pairwise: 1.193183e-02 +NonBlocking: 1.179608e-01 +Pairwise Hierarchical: 9.785565e-03 +Pairwise Node Aware: 8.581460e-04 +Nonblocking Hierarchical: 8.150782e-03 +Nonblocking Node Aware: 7.538026e-04 +Pairwise Multileader, 4 leaders: 3.379327e-03 +Pairwise Locality Aware, 4 leaders: 7.105863e-04 +Pairwise Multileader Locality, 4 leaders: 3.460207e-03 +Nonblocking Multileader, 4 leaders: 2.294660e-03 +Nonblocking Locality Aware, 4 leaders: 5.036949e-04 +Nonblocking Multileader Locality, 4 leaders: 2.731080e-03 +Pairwise Multileader, 10 leaders: 1.800010e-03 +Pairwise Locality Aware, 10 leaders: 1.352432e-03 +Pairwise Multileader Locality, 10 leaders: 1.202443e-03 +Nonblocking Multileader, 10 leaders: 1.261784e-03 +Nonblocking Locality Aware, 10 leaders: 7.438087e-02 +Nonblocking Multileader Locality, 10 leaders: 9.814482e-04 +Pairwise Multileader, 20 leaders: 2.762482e-03 +Pairwise Locality Aware, 20 leaders: 2.102144e-03 +Pairwise Multileader Locality, 20 leaders: 6.954222e-04 +Nonblocking Multileader, 20 leaders: 2.771489e-02 +Nonblocking Locality Aware, 20 leaders: 9.204371e-02 +Nonblocking Multileader Locality, 20 leaders: 5.176575e-04 +Size 16 +PMPI: 2.757466e-03 +Pairwise: 1.000883e-02 +NonBlocking: 1.178396e-01 +Pairwise Hierarchical: 2.057996e-02 +Pairwise Node Aware: 1.253036e-03 +Nonblocking Hierarchical: 1.657789e-02 +Nonblocking Node Aware: 1.172125e-03 +Pairwise Multileader, 4 leaders: 4.941626e-03 +Pairwise Locality Aware, 4 leaders: 1.002348e-03 +Pairwise Multileader Locality, 4 leaders: 6.095266e-03 +Nonblocking Multileader, 4 leaders: 3.914194e-03 +Nonblocking Locality Aware, 4 leaders: 7.283098e-04 +Nonblocking Multileader Locality, 4 leaders: 4.779947e-03 +Pairwise Multileader, 10 leaders: 2.947640e-03 +Pairwise Locality Aware, 10 leaders: 1.629055e-03 +Pairwise Multileader Locality, 10 leaders: 2.209164e-03 +Nonblocking Multileader, 10 leaders: 1.631943e-03 +Nonblocking Locality Aware, 10 leaders: 7.521692e-02 +Nonblocking Multileader Locality, 10 leaders: 1.932853e-03 +Pairwise Multileader, 20 leaders: 3.121723e-03 +Pairwise Locality Aware, 20 leaders: 2.309889e-03 +Pairwise Multileader Locality, 20 leaders: 1.450133e-03 +Nonblocking Multileader, 20 leaders: 4.228023e-02 +Nonblocking Locality Aware, 20 leaders: 9.922919e-02 +Nonblocking Multileader Locality, 20 leaders: 9.255449e-04 +Size 32 +PMPI: 4.105559e-03 +Pairwise: 9.870451e-03 +NonBlocking: 1.403264e-01 +Pairwise Hierarchical: 4.217345e-02 +Pairwise Node Aware: 2.290347e-03 +Nonblocking Hierarchical: 3.380198e-02 +Nonblocking Node Aware: 2.228625e-03 +Pairwise Multileader, 4 leaders: 1.356437e-02 +Pairwise Locality Aware, 4 leaders: 2.037016e-03 +Pairwise Multileader Locality, 4 leaders: 1.100815e-02 +Nonblocking Multileader, 4 leaders: 8.662116e-03 +Nonblocking Locality Aware, 4 leaders: 1.820641e-03 +Nonblocking Multileader Locality, 4 leaders: 8.802822e-03 +Pairwise Multileader, 10 leaders: 5.639395e-03 +Pairwise Locality Aware, 10 leaders: 3.055196e-03 +Pairwise Multileader Locality, 10 leaders: 4.738892e-03 +Nonblocking Multileader, 10 leaders: 3.402673e-03 +Nonblocking Locality Aware, 10 leaders: 1.159076e-01 +Nonblocking Multileader Locality, 10 leaders: 3.848948e-03 +Pairwise Multileader, 20 leaders: 4.287707e-03 +Pairwise Locality Aware, 20 leaders: 3.540007e-03 +Pairwise Multileader Locality, 20 leaders: 2.660027e-03 +Nonblocking Multileader, 20 leaders: 4.386316e-02 +Nonblocking Locality Aware, 20 leaders: 1.165603e-01 +Nonblocking Multileader Locality, 20 leaders: 2.036354e-03 +Size 64 +PMPI: 8.294713e-03 +Pairwise: 1.505410e-02 +NonBlocking: 1.416257e-01 +Pairwise Hierarchical: 8.184013e-02 +Pairwise Node Aware: 4.010452e-03 +Nonblocking Hierarchical: 6.457789e-02 +Nonblocking Node Aware: 3.234079e-03 +Pairwise Multileader, 4 leaders: 2.596765e-02 +Pairwise Locality Aware, 4 leaders: 3.720595e-03 +Pairwise Multileader Locality, 4 leaders: 2.214372e-02 +Nonblocking Multileader, 4 leaders: 1.651275e-02 +Nonblocking Locality Aware, 4 leaders: 4.405511e-03 +Nonblocking Multileader Locality, 4 leaders: 1.763381e-02 +Pairwise Multileader, 10 leaders: 1.166585e-02 +Pairwise Locality Aware, 10 leaders: 4.211941e-03 +Pairwise Multileader Locality, 10 leaders: 1.014136e-02 +Nonblocking Multileader, 10 leaders: 7.808566e-03 +Nonblocking Locality Aware, 10 leaders: 6.470576e-02 +Nonblocking Multileader Locality, 10 leaders: 8.636737e-03 +Pairwise Multileader, 20 leaders: 7.947822e-03 +Pairwise Locality Aware, 20 leaders: 4.685134e-03 +Pairwise Multileader Locality, 20 leaders: 5.415907e-03 +Nonblocking Multileader, 20 leaders: 4.612588e-03 +Nonblocking Locality Aware, 20 leaders: 1.178484e-01 +Nonblocking Multileader Locality, 20 leaders: 4.305274e-03 +Size 128 +PMPI: 1.345296e-02 +Pairwise: 1.326448e-02 +NonBlocking: 1.378178e-01 +Pairwise Hierarchical: 1.578630e-01 +Pairwise Node Aware: 8.782189e-03 +Nonblocking Hierarchical: 1.250113e-01 +Nonblocking Node Aware: 6.951004e-03 +Pairwise Multileader, 4 leaders: 5.166231e-02 +Pairwise Locality Aware, 4 leaders: 7.558093e-03 +Pairwise Multileader Locality, 4 leaders: 5.470897e-02 +Nonblocking Multileader, 4 leaders: 3.690694e-02 +Nonblocking Locality Aware, 4 leaders: 1.028406e-02 +Nonblocking Multileader Locality, 4 leaders: 4.221187e-02 +Pairwise Multileader, 10 leaders: 2.261543e-02 +Pairwise Locality Aware, 10 leaders: 8.631065e-03 +Pairwise Multileader Locality, 10 leaders: 2.391034e-02 +Nonblocking Multileader, 10 leaders: 1.730456e-02 +Nonblocking Locality Aware, 10 leaders: 1.039762e-02 +Nonblocking Multileader Locality, 10 leaders: 1.951685e-02 +Pairwise Multileader, 20 leaders: 1.457000e-02 +Pairwise Locality Aware, 20 leaders: 8.421821e-03 +Pairwise Multileader Locality, 20 leaders: 1.173429e-02 +Nonblocking Multileader, 20 leaders: 9.437933e-03 +Nonblocking Locality Aware, 20 leaders: 7.364106e-02 +Nonblocking Multileader Locality, 20 leaders: 9.561000e-03 +Size 256 +PMPI: 2.762654e-02 +Pairwise: 2.459198e-02 +NonBlocking: 1.212397e-01 +Pairwise Hierarchical: 3.071966e-01 +Pairwise Node Aware: 1.367392e-02 +Nonblocking Hierarchical: 2.452160e-01 +Nonblocking Node Aware: 1.350095e-02 +Pairwise Multileader, 4 leaders: 9.463377e-02 +Pairwise Locality Aware, 4 leaders: 1.437593e-02 +Pairwise Multileader Locality, 4 leaders: 1.047465e-01 +Nonblocking Multileader, 4 leaders: 6.900361e-02 +Nonblocking Locality Aware, 4 leaders: 1.267562e-02 +Nonblocking Multileader Locality, 4 leaders: 7.968354e-02 +Pairwise Multileader, 10 leaders: 5.427224e-02 +Pairwise Locality Aware, 10 leaders: 1.719388e-02 +Pairwise Multileader Locality, 10 leaders: 5.112861e-02 +Nonblocking Multileader, 10 leaders: 3.702864e-02 +Nonblocking Locality Aware, 10 leaders: 2.699075e-02 +Nonblocking Multileader Locality, 10 leaders: 4.196661e-02 +Pairwise Multileader, 20 leaders: 3.144581e-02 +Pairwise Locality Aware, 20 leaders: 1.805647e-02 +Pairwise Multileader Locality, 20 leaders: 2.593099e-02 +Nonblocking Multileader, 20 leaders: 1.861973e-02 +Nonblocking Locality Aware, 20 leaders: 6.827988e-02 +Nonblocking Multileader Locality, 20 leaders: 2.079290e-02 +Size 512 +PMPI: 5.108115e-02 +Pairwise: 3.932628e-02 +NonBlocking: 1.804671e-01 +Pairwise Hierarchical: 5.982152e-01 +Pairwise Node Aware: 3.169344e-02 +Nonblocking Hierarchical: 5.420444e-01 +Nonblocking Node Aware: 2.970882e-02 +Pairwise Multileader, 4 leaders: 1.763421e-01 +Pairwise Locality Aware, 4 leaders: 3.371272e-02 +Pairwise Multileader Locality, 4 leaders: 1.999064e-01 +Nonblocking Multileader, 4 leaders: 1.305504e-01 +Nonblocking Locality Aware, 4 leaders: 2.888802e-02 +Nonblocking Multileader Locality, 4 leaders: 1.562373e-01 +Pairwise Multileader, 10 leaders: 1.027622e-01 +Pairwise Locality Aware, 10 leaders: 3.728287e-02 +Pairwise Multileader Locality, 10 leaders: 1.051706e-01 +Nonblocking Multileader, 10 leaders: 7.324588e-02 +Nonblocking Locality Aware, 10 leaders: 3.073526e-02 +Nonblocking Multileader Locality, 10 leaders: 8.675102e-02 +Pairwise Multileader, 20 leaders: 5.745122e-02 +Pairwise Locality Aware, 20 leaders: 3.422719e-02 +Pairwise Multileader Locality, 20 leaders: 5.363702e-02 +Nonblocking Multileader, 20 leaders: 3.440509e-02 +Nonblocking Locality Aware, 20 leaders: 4.972070e-02 +Nonblocking Multileader Locality, 20 leaders: 4.351223e-02 +Test 3 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 4.056194e-04 +Pairwise: 5.620454e-03 +NonBlocking: 8.160220e-02 +Pairwise Hierarchical: 2.426619e-03 +Pairwise Node Aware: 3.935712e-04 +Nonblocking Hierarchical: 9.469995e-04 +Nonblocking Node Aware: 2.444437e-04 +Pairwise Multileader, 4 leaders: 8.000696e-04 +Pairwise Locality Aware, 4 leaders: 5.782332e-04 +Pairwise Multileader Locality, 4 leaders: 4.832044e-04 +Nonblocking Multileader, 4 leaders: 3.049983e-04 +Nonblocking Locality Aware, 4 leaders: 2.488800e-04 +Nonblocking Multileader Locality, 4 leaders: 3.540779e-04 +Pairwise Multileader, 10 leaders: 9.298051e-04 +Pairwise Locality Aware, 10 leaders: 1.466546e-03 +Pairwise Multileader Locality, 10 leaders: 2.661429e-04 +Nonblocking Multileader, 10 leaders: 7.598156e-04 +Nonblocking Locality Aware, 10 leaders: 2.420559e-02 +Nonblocking Multileader Locality, 10 leaders: 1.740324e-04 +Pairwise Multileader, 20 leaders: 2.290719e-03 +Pairwise Locality Aware, 20 leaders: 2.010099e-03 +Pairwise Multileader Locality, 20 leaders: 2.312329e-04 +Nonblocking Multileader, 20 leaders: 4.564236e-03 +Nonblocking Locality Aware, 20 leaders: 7.537643e-02 +Nonblocking Multileader Locality, 20 leaders: 1.487332e-04 +Size 2 +PMPI: 4.492727e-04 +Pairwise: 6.237778e-03 +NonBlocking: 7.012331e-02 +Pairwise Hierarchical: 2.505455e-03 +Pairwise Node Aware: 4.448270e-04 +Nonblocking Hierarchical: 2.150244e-03 +Nonblocking Node Aware: 3.190208e-04 +Pairwise Multileader, 4 leaders: 9.339997e-04 +Pairwise Locality Aware, 4 leaders: 7.185598e-04 +Pairwise Multileader Locality, 4 leaders: 8.266691e-04 +Nonblocking Multileader, 4 leaders: 5.544250e-04 +Nonblocking Locality Aware, 4 leaders: 4.009050e-04 +Nonblocking Multileader Locality, 4 leaders: 6.382016e-04 +Pairwise Multileader, 10 leaders: 9.763208e-04 +Pairwise Locality Aware, 10 leaders: 1.380183e-03 +Pairwise Multileader Locality, 10 leaders: 3.736775e-04 +Nonblocking Multileader, 10 leaders: 3.157212e-04 +Nonblocking Locality Aware, 10 leaders: 3.729574e-02 +Nonblocking Multileader Locality, 10 leaders: 2.575503e-04 +Pairwise Multileader, 20 leaders: 2.573619e-03 +Pairwise Locality Aware, 20 leaders: 1.830637e-03 +Pairwise Multileader Locality, 20 leaders: 2.918481e-04 +Nonblocking Multileader, 20 leaders: 2.978462e-03 +Nonblocking Locality Aware, 20 leaders: 4.915209e-02 +Nonblocking Multileader Locality, 20 leaders: 2.054243e-04 +Size 4 +PMPI: 7.935999e-04 +Pairwise: 9.845321e-03 +NonBlocking: 1.236220e-01 +Pairwise Hierarchical: 4.098530e-03 +Pairwise Node Aware: 6.536774e-04 +Nonblocking Hierarchical: 3.463738e-03 +Nonblocking Node Aware: 5.249497e-04 +Pairwise Multileader, 4 leaders: 1.459137e-03 +Pairwise Locality Aware, 4 leaders: 6.249236e-04 +Pairwise Multileader Locality, 4 leaders: 1.304609e-03 +Nonblocking Multileader, 4 leaders: 9.204816e-04 +Nonblocking Locality Aware, 4 leaders: 4.078424e-04 +Nonblocking Multileader Locality, 4 leaders: 1.077838e-03 +Pairwise Multileader, 10 leaders: 1.471342e-03 +Pairwise Locality Aware, 10 leaders: 1.484384e-03 +Pairwise Multileader Locality, 10 leaders: 5.639770e-04 +Nonblocking Multileader, 10 leaders: 5.300194e-03 +Nonblocking Locality Aware, 10 leaders: 7.429704e-02 +Nonblocking Multileader Locality, 10 leaders: 4.433458e-04 +Pairwise Multileader, 20 leaders: 2.601150e-03 +Pairwise Locality Aware, 20 leaders: 2.667395e-03 +Pairwise Multileader Locality, 20 leaders: 4.458030e-04 +Nonblocking Multileader, 20 leaders: 9.036785e-03 +Nonblocking Locality Aware, 20 leaders: 1.148907e-01 +Nonblocking Multileader Locality, 20 leaders: 3.609798e-04 +Size 8 +PMPI: 1.223040e-03 +Pairwise: 1.037076e-02 +NonBlocking: 1.180412e-01 +Pairwise Hierarchical: 9.747594e-03 +Pairwise Node Aware: 9.376359e-04 +Nonblocking Hierarchical: 8.148180e-03 +Nonblocking Node Aware: 9.315388e-04 +Pairwise Multileader, 4 leaders: 2.807399e-03 +Pairwise Locality Aware, 4 leaders: 8.593878e-04 +Pairwise Multileader Locality, 4 leaders: 3.306262e-03 +Nonblocking Multileader, 4 leaders: 2.062807e-03 +Nonblocking Locality Aware, 4 leaders: 7.325220e-04 +Nonblocking Multileader Locality, 4 leaders: 2.637292e-03 +Pairwise Multileader, 10 leaders: 1.674812e-03 +Pairwise Locality Aware, 10 leaders: 1.585787e-03 +Pairwise Multileader Locality, 10 leaders: 1.216776e-03 +Nonblocking Multileader, 10 leaders: 2.545573e-03 +Nonblocking Locality Aware, 10 leaders: 6.211607e-02 +Nonblocking Multileader Locality, 10 leaders: 7.850484e-04 +Pairwise Multileader, 20 leaders: 2.895088e-03 +Pairwise Locality Aware, 20 leaders: 2.378786e-03 +Pairwise Multileader Locality, 20 leaders: 7.264164e-04 +Nonblocking Multileader, 20 leaders: 3.749767e-02 +Nonblocking Locality Aware, 20 leaders: 7.446293e-02 +Nonblocking Multileader Locality, 20 leaders: 5.886097e-04 +Size 16 +PMPI: 2.722176e-03 +Pairwise: 8.486429e-03 +NonBlocking: 2.085683e-01 +Pairwise Hierarchical: 2.031100e-02 +Pairwise Node Aware: 1.246274e-03 +Nonblocking Hierarchical: 1.656281e-02 +Nonblocking Node Aware: 1.161196e-03 +Pairwise Multileader, 4 leaders: 4.728464e-03 +Pairwise Locality Aware, 4 leaders: 1.061274e-03 +Pairwise Multileader Locality, 4 leaders: 5.900487e-03 +Nonblocking Multileader, 4 leaders: 3.747925e-03 +Nonblocking Locality Aware, 4 leaders: 7.206111e-04 +Nonblocking Multileader Locality, 4 leaders: 4.696564e-03 +Pairwise Multileader, 10 leaders: 2.808903e-03 +Pairwise Locality Aware, 10 leaders: 1.411947e-03 +Pairwise Multileader Locality, 10 leaders: 2.230006e-03 +Nonblocking Multileader, 10 leaders: 1.866768e-03 +Nonblocking Locality Aware, 10 leaders: 8.709662e-02 +Nonblocking Multileader Locality, 10 leaders: 1.943980e-03 +Pairwise Multileader, 20 leaders: 3.172317e-03 +Pairwise Locality Aware, 20 leaders: 2.742957e-03 +Pairwise Multileader Locality, 20 leaders: 1.447614e-03 +Nonblocking Multileader, 20 leaders: 3.386702e-02 +Nonblocking Locality Aware, 20 leaders: 8.691813e-02 +Nonblocking Multileader Locality, 20 leaders: 1.174326e-03 +Size 32 +PMPI: 4.332600e-03 +Pairwise: 9.629696e-03 +NonBlocking: 1.397581e-01 +Pairwise Hierarchical: 4.271613e-02 +Pairwise Node Aware: 2.409488e-03 +Nonblocking Hierarchical: 3.374043e-02 +Nonblocking Node Aware: 2.323854e-03 +Pairwise Multileader, 4 leaders: 1.353206e-02 +Pairwise Locality Aware, 4 leaders: 1.874698e-03 +Pairwise Multileader Locality, 4 leaders: 1.113067e-02 +Nonblocking Multileader, 4 leaders: 8.647183e-03 +Nonblocking Locality Aware, 4 leaders: 1.449837e-03 +Nonblocking Multileader Locality, 4 leaders: 8.827992e-03 +Pairwise Multileader, 10 leaders: 5.232639e-03 +Pairwise Locality Aware, 10 leaders: 2.477749e-03 +Pairwise Multileader Locality, 10 leaders: 4.829182e-03 +Nonblocking Multileader, 10 leaders: 3.392422e-03 +Nonblocking Locality Aware, 10 leaders: 8.797499e-02 +Nonblocking Multileader Locality, 10 leaders: 3.869471e-03 +Pairwise Multileader, 20 leaders: 3.902940e-03 +Pairwise Locality Aware, 20 leaders: 2.675184e-03 +Pairwise Multileader Locality, 20 leaders: 2.628264e-03 +Nonblocking Multileader, 20 leaders: 2.611468e-02 +Nonblocking Locality Aware, 20 leaders: 1.155667e-01 +Nonblocking Multileader Locality, 20 leaders: 2.257770e-03 +Size 64 +PMPI: 8.088600e-03 +Pairwise: 9.669455e-03 +NonBlocking: 1.406851e-01 +Pairwise Hierarchical: 8.227013e-02 +Pairwise Node Aware: 4.128537e-03 +Nonblocking Hierarchical: 6.439827e-02 +Nonblocking Node Aware: 3.203968e-03 +Pairwise Multileader, 4 leaders: 2.591870e-02 +Pairwise Locality Aware, 4 leaders: 3.700636e-03 +Pairwise Multileader Locality, 4 leaders: 2.233706e-02 +Nonblocking Multileader, 4 leaders: 1.643241e-02 +Nonblocking Locality Aware, 4 leaders: 3.959992e-03 +Nonblocking Multileader Locality, 4 leaders: 1.764935e-02 +Pairwise Multileader, 10 leaders: 1.190045e-02 +Pairwise Locality Aware, 10 leaders: 4.272802e-03 +Pairwise Multileader Locality, 10 leaders: 1.006677e-02 +Nonblocking Multileader, 10 leaders: 7.992551e-03 +Nonblocking Locality Aware, 10 leaders: 7.713388e-02 +Nonblocking Multileader Locality, 10 leaders: 8.565480e-03 +Pairwise Multileader, 20 leaders: 7.256888e-03 +Pairwise Locality Aware, 20 leaders: 5.425456e-03 +Pairwise Multileader Locality, 20 leaders: 5.539714e-03 +Nonblocking Multileader, 20 leaders: 4.009747e-03 +Nonblocking Locality Aware, 20 leaders: 1.162965e-01 +Nonblocking Multileader Locality, 20 leaders: 4.389476e-03 +Size 128 +PMPI: 1.373813e-02 +Pairwise: 1.410625e-02 +NonBlocking: 1.386339e-01 +Pairwise Hierarchical: 1.570768e-01 +Pairwise Node Aware: 8.783328e-03 +Nonblocking Hierarchical: 1.246298e-01 +Nonblocking Node Aware: 7.122048e-03 +Pairwise Multileader, 4 leaders: 5.186588e-02 +Pairwise Locality Aware, 4 leaders: 7.561386e-03 +Pairwise Multileader Locality, 4 leaders: 5.483121e-02 +Nonblocking Multileader, 4 leaders: 3.696069e-02 +Nonblocking Locality Aware, 4 leaders: 9.158687e-03 +Nonblocking Multileader Locality, 4 leaders: 4.162841e-02 +Pairwise Multileader, 10 leaders: 2.233809e-02 +Pairwise Locality Aware, 10 leaders: 8.700488e-03 +Pairwise Multileader Locality, 10 leaders: 2.397820e-02 +Nonblocking Multileader, 10 leaders: 1.724228e-02 +Nonblocking Locality Aware, 10 leaders: 1.059502e-02 +Nonblocking Multileader Locality, 10 leaders: 1.974844e-02 +Pairwise Multileader, 20 leaders: 1.367326e-02 +Pairwise Locality Aware, 20 leaders: 7.690287e-03 +Pairwise Multileader Locality, 20 leaders: 1.176550e-02 +Nonblocking Multileader, 20 leaders: 9.377652e-03 +Nonblocking Locality Aware, 20 leaders: 8.421389e-02 +Nonblocking Multileader Locality, 20 leaders: 9.607994e-03 +Size 256 +PMPI: 2.802163e-02 +Pairwise: 2.364133e-02 +NonBlocking: 1.437098e-01 +Pairwise Hierarchical: 3.050258e-01 +Pairwise Node Aware: 1.316456e-02 +Nonblocking Hierarchical: 2.472468e-01 +Nonblocking Node Aware: 1.358503e-02 +Pairwise Multileader, 4 leaders: 9.536624e-02 +Pairwise Locality Aware, 4 leaders: 1.350322e-02 +Pairwise Multileader Locality, 4 leaders: 1.053605e-01 +Nonblocking Multileader, 4 leaders: 6.885676e-02 +Nonblocking Locality Aware, 4 leaders: 1.012003e-02 +Nonblocking Multileader Locality, 4 leaders: 8.027354e-02 +Pairwise Multileader, 10 leaders: 5.695907e-02 +Pairwise Locality Aware, 10 leaders: 1.554696e-02 +Pairwise Multileader Locality, 10 leaders: 5.116433e-02 +Nonblocking Multileader, 10 leaders: 3.721726e-02 +Nonblocking Locality Aware, 10 leaders: 2.259544e-02 +Nonblocking Multileader Locality, 10 leaders: 4.217403e-02 +Pairwise Multileader, 20 leaders: 3.159568e-02 +Pairwise Locality Aware, 20 leaders: 1.714226e-02 +Pairwise Multileader Locality, 20 leaders: 2.604765e-02 +Nonblocking Multileader, 20 leaders: 1.854408e-02 +Nonblocking Locality Aware, 20 leaders: 8.152541e-02 +Nonblocking Multileader Locality, 20 leaders: 2.066877e-02 +Size 512 +PMPI: 6.078197e-02 +Pairwise: 3.804570e-02 +NonBlocking: 1.576209e-01 +Pairwise Hierarchical: 6.268032e-01 +Pairwise Node Aware: 2.959684e-02 +Nonblocking Hierarchical: 5.415808e-01 +Nonblocking Node Aware: 2.732935e-02 +Pairwise Multileader, 4 leaders: 1.791945e-01 +Pairwise Locality Aware, 4 leaders: 3.295622e-02 +Pairwise Multileader Locality, 4 leaders: 1.992444e-01 +Nonblocking Multileader, 4 leaders: 1.309408e-01 +Nonblocking Locality Aware, 4 leaders: 2.853539e-02 +Nonblocking Multileader Locality, 4 leaders: 1.558574e-01 +Pairwise Multileader, 10 leaders: 1.042019e-01 +Pairwise Locality Aware, 10 leaders: 3.230283e-02 +Pairwise Multileader Locality, 10 leaders: 1.045483e-01 +Nonblocking Multileader, 10 leaders: 7.314384e-02 +Nonblocking Locality Aware, 10 leaders: 2.703545e-02 +Nonblocking Multileader Locality, 10 leaders: 8.693618e-02 +Pairwise Multileader, 20 leaders: 5.820574e-02 +Pairwise Locality Aware, 20 leaders: 3.517655e-02 +Pairwise Multileader Locality, 20 leaders: 5.383770e-02 +Nonblocking Multileader, 20 leaders: 3.379631e-02 +Nonblocking Locality Aware, 20 leaders: 3.945241e-02 +Nonblocking Multileader Locality, 20 leaders: 4.352796e-02 diff --git a/runscripts/dane_results/alltoall_N2.2373822.err b/runscripts/dane_results/alltoall_N2.2373822.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/dane_results/alltoall_N2.2373822.out b/runscripts/dane_results/alltoall_N2.2373822.out new file mode 100644 index 000000000..a14ce3cb5 --- /dev/null +++ b/runscripts/dane_results/alltoall_N2.2373822.out @@ -0,0 +1,1173 @@ +Test 1 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 4.276349e-05 +Pairwise: 3.804963e-04 +NonBlocking: 1.771278e-04 +Pairwise Hierarchical: 1.787788e-04 +Pairwise Node Aware: 1.731286e-04 +Nonblocking Hierarchical: 1.827358e-04 +Nonblocking Node Aware: 1.726738e-04 +Pairwise Multileader, 4 leaders: 6.400250e-05 +Pairwise Locality Aware, 4 leaders: 7.739849e-05 +Pairwise Multileader Locality, 4 leaders: 5.884273e-05 +Nonblocking Multileader, 4 leaders: 4.691155e-05 +Nonblocking Locality Aware, 4 leaders: 4.647622e-05 +Nonblocking Multileader Locality, 4 leaders: 5.454090e-05 +Pairwise Multileader, 10 leaders: 8.423686e-05 +Pairwise Locality Aware, 10 leaders: 9.158761e-05 +Pairwise Multileader Locality, 10 leaders: 3.869771e-05 +Nonblocking Multileader, 10 leaders: 3.588255e-05 +Nonblocking Locality Aware, 10 leaders: 3.709910e-05 +Nonblocking Multileader Locality, 10 leaders: 3.518294e-05 +Pairwise Multileader, 20 leaders: 1.612980e-04 +Pairwise Locality Aware, 20 leaders: 1.604323e-04 +Pairwise Multileader Locality, 20 leaders: 5.530878e-05 +Nonblocking Multileader, 20 leaders: 5.460870e-05 +Nonblocking Locality Aware, 20 leaders: 5.178353e-05 +Nonblocking Multileader Locality, 20 leaders: 4.106240e-05 +Size 2 +PMPI: 4.525024e-05 +Pairwise: 3.720246e-04 +NonBlocking: 1.777919e-04 +Pairwise Hierarchical: 2.154721e-04 +Pairwise Node Aware: 1.750282e-04 +Nonblocking Hierarchical: 2.174970e-04 +Nonblocking Node Aware: 1.725047e-04 +Pairwise Multileader, 4 leaders: 7.670095e-05 +Pairwise Locality Aware, 4 leaders: 7.097202e-05 +Pairwise Multileader Locality, 4 leaders: 7.874429e-05 +Nonblocking Multileader, 4 leaders: 6.247562e-05 +Nonblocking Locality Aware, 4 leaders: 4.667021e-05 +Nonblocking Multileader Locality, 4 leaders: 7.464616e-05 +Pairwise Multileader, 10 leaders: 9.085244e-05 +Pairwise Locality Aware, 10 leaders: 9.423087e-05 +Pairwise Multileader Locality, 10 leaders: 4.956413e-05 +Nonblocking Multileader, 10 leaders: 4.106905e-05 +Nonblocking Locality Aware, 10 leaders: 4.218519e-05 +Nonblocking Multileader Locality, 10 leaders: 4.342822e-05 +Pairwise Multileader, 20 leaders: 1.630426e-04 +Pairwise Locality Aware, 20 leaders: 1.577652e-04 +Pairwise Multileader Locality, 20 leaders: 6.350078e-05 +Nonblocking Multileader, 20 leaders: 5.893382e-05 +Nonblocking Locality Aware, 20 leaders: 5.197226e-05 +Nonblocking Multileader Locality, 20 leaders: 4.626331e-05 +Size 4 +PMPI: 5.484136e-05 +Pairwise: 7.680734e-04 +NonBlocking: 1.099978e-03 +Pairwise Hierarchical: 2.852817e-04 +Pairwise Node Aware: 2.254772e-04 +Nonblocking Hierarchical: 2.859958e-04 +Nonblocking Node Aware: 1.753895e-04 +Pairwise Multileader, 4 leaders: 1.139590e-04 +Pairwise Locality Aware, 4 leaders: 7.988815e-05 +Pairwise Multileader Locality, 4 leaders: 1.111357e-04 +Nonblocking Multileader, 4 leaders: 8.927199e-05 +Nonblocking Locality Aware, 4 leaders: 4.910677e-05 +Nonblocking Multileader Locality, 4 leaders: 1.052969e-04 +Pairwise Multileader, 10 leaders: 9.915848e-05 +Pairwise Locality Aware, 10 leaders: 9.661583e-05 +Pairwise Multileader Locality, 10 leaders: 6.503150e-05 +Nonblocking Multileader, 10 leaders: 4.989059e-05 +Nonblocking Locality Aware, 10 leaders: 4.888176e-05 +Nonblocking Multileader Locality, 10 leaders: 5.781580e-05 +Pairwise Multileader, 20 leaders: 1.675604e-04 +Pairwise Locality Aware, 20 leaders: 1.628446e-04 +Pairwise Multileader Locality, 20 leaders: 7.650700e-05 +Nonblocking Multileader, 20 leaders: 6.410682e-05 +Nonblocking Locality Aware, 20 leaders: 5.889263e-05 +Nonblocking Multileader Locality, 20 leaders: 5.733998e-05 +Size 8 +PMPI: 7.643733e-05 +Pairwise: 7.744308e-04 +NonBlocking: 1.681068e-03 +Pairwise Hierarchical: 8.010506e-04 +Pairwise Node Aware: 2.544808e-04 +Nonblocking Hierarchical: 7.535050e-04 +Nonblocking Node Aware: 1.746973e-04 +Pairwise Multileader, 4 leaders: 2.207715e-04 +Pairwise Locality Aware, 4 leaders: 8.646980e-05 +Pairwise Multileader Locality, 4 leaders: 3.006085e-04 +Nonblocking Multileader, 4 leaders: 1.802621e-04 +Nonblocking Locality Aware, 4 leaders: 5.506709e-05 +Nonblocking Multileader Locality, 4 leaders: 2.568797e-04 +Pairwise Multileader, 10 leaders: 1.206486e-04 +Pairwise Locality Aware, 10 leaders: 9.564758e-05 +Pairwise Multileader Locality, 10 leaders: 1.012042e-04 +Nonblocking Multileader, 10 leaders: 7.062911e-05 +Nonblocking Locality Aware, 10 leaders: 5.439631e-05 +Nonblocking Multileader Locality, 10 leaders: 8.605099e-05 +Pairwise Multileader, 20 leaders: 1.791274e-04 +Pairwise Locality Aware, 20 leaders: 1.684123e-04 +Pairwise Multileader Locality, 20 leaders: 1.019636e-04 +Nonblocking Multileader, 20 leaders: 7.643154e-05 +Nonblocking Locality Aware, 20 leaders: 7.147014e-05 +Nonblocking Multileader Locality, 20 leaders: 7.756910e-05 +Size 16 +PMPI: 1.307773e-04 +Pairwise: 7.679676e-04 +NonBlocking: 1.185098e-03 +Pairwise Hierarchical: 1.508212e-03 +Pairwise Node Aware: 3.233793e-04 +Nonblocking Hierarchical: 1.459372e-03 +Nonblocking Node Aware: 1.724886e-04 +Pairwise Multileader, 4 leaders: 4.500991e-04 +Pairwise Locality Aware, 4 leaders: 1.175080e-04 +Pairwise Multileader Locality, 4 leaders: 5.495852e-04 +Nonblocking Multileader, 4 leaders: 3.572319e-04 +Nonblocking Locality Aware, 4 leaders: 9.337894e-05 +Nonblocking Multileader Locality, 4 leaders: 4.810580e-04 +Pairwise Multileader, 10 leaders: 2.278123e-04 +Pairwise Locality Aware, 10 leaders: 1.177544e-04 +Pairwise Multileader Locality, 10 leaders: 1.902097e-04 +Nonblocking Multileader, 10 leaders: 1.451204e-04 +Nonblocking Locality Aware, 10 leaders: 9.134156e-05 +Nonblocking Multileader Locality, 10 leaders: 1.630891e-04 +Pairwise Multileader, 20 leaders: 2.080655e-04 +Pairwise Locality Aware, 20 leaders: 1.752735e-04 +Pairwise Multileader Locality, 20 leaders: 1.490144e-04 +Nonblocking Multileader, 20 leaders: 1.055672e-04 +Nonblocking Locality Aware, 20 leaders: 1.006916e-04 +Nonblocking Multileader Locality, 20 leaders: 1.107149e-04 +Size 32 +PMPI: 2.440813e-04 +Pairwise: 7.825655e-04 +NonBlocking: 5.043882e-03 +Pairwise Hierarchical: 2.982320e-03 +Pairwise Node Aware: 4.232561e-04 +Nonblocking Hierarchical: 2.870363e-03 +Nonblocking Node Aware: 2.121147e-04 +Pairwise Multileader, 4 leaders: 9.737544e-04 +Pairwise Locality Aware, 4 leaders: 1.818662e-04 +Pairwise Multileader Locality, 4 leaders: 1.006368e-03 +Nonblocking Multileader, 4 leaders: 8.531574e-04 +Nonblocking Locality Aware, 4 leaders: 1.320830e-04 +Nonblocking Multileader Locality, 4 leaders: 9.270593e-04 +Pairwise Multileader, 10 leaders: 4.237135e-04 +Pairwise Locality Aware, 10 leaders: 1.648441e-04 +Pairwise Multileader Locality, 10 leaders: 4.219458e-04 +Nonblocking Multileader, 10 leaders: 3.038230e-04 +Nonblocking Locality Aware, 10 leaders: 1.534668e-04 +Nonblocking Multileader Locality, 10 leaders: 3.897982e-04 +Pairwise Multileader, 20 leaders: 3.121691e-04 +Pairwise Locality Aware, 20 leaders: 2.080408e-04 +Pairwise Multileader Locality, 20 leaders: 2.476494e-04 +Nonblocking Multileader, 20 leaders: 1.664851e-04 +Nonblocking Locality Aware, 20 leaders: 1.837931e-04 +Nonblocking Multileader Locality, 20 leaders: 1.958421e-04 +Size 64 +PMPI: 4.518336e-04 +Pairwise: 7.994225e-04 +NonBlocking: 3.728190e-02 +Pairwise Hierarchical: 6.815730e-03 +Pairwise Node Aware: 4.576201e-04 +Nonblocking Hierarchical: 6.514656e-03 +Nonblocking Node Aware: 2.867176e-04 +Pairwise Multileader, 4 leaders: 2.335237e-03 +Pairwise Locality Aware, 4 leaders: 3.105077e-04 +Pairwise Multileader Locality, 4 leaders: 2.340314e-03 +Nonblocking Multileader, 4 leaders: 1.913363e-03 +Nonblocking Locality Aware, 4 leaders: 2.373497e-04 +Nonblocking Multileader Locality, 4 leaders: 2.155638e-03 +Pairwise Multileader, 10 leaders: 9.126144e-04 +Pairwise Locality Aware, 10 leaders: 2.890645e-04 +Pairwise Multileader Locality, 10 leaders: 9.679811e-04 +Nonblocking Multileader, 10 leaders: 7.030415e-04 +Nonblocking Locality Aware, 10 leaders: 2.345774e-04 +Nonblocking Multileader Locality, 10 leaders: 8.723431e-04 +Pairwise Multileader, 20 leaders: 5.865545e-04 +Pairwise Locality Aware, 20 leaders: 3.217168e-04 +Pairwise Multileader Locality, 20 leaders: 5.011440e-04 +Nonblocking Multileader, 20 leaders: 3.524271e-04 +Nonblocking Locality Aware, 20 leaders: 3.148117e-04 +Nonblocking Multileader Locality, 20 leaders: 4.406405e-04 +Size 128 +PMPI: 7.907437e-04 +Pairwise: 9.180271e-04 +NonBlocking: 1.215337e-02 +Pairwise Hierarchical: 1.347752e-02 +Pairwise Node Aware: 7.608166e-04 +Nonblocking Hierarchical: 1.293922e-02 +Nonblocking Node Aware: 4.958427e-04 +Pairwise Multileader, 4 leaders: 4.375546e-03 +Pairwise Locality Aware, 4 leaders: 5.662524e-04 +Pairwise Multileader Locality, 4 leaders: 4.560509e-03 +Nonblocking Multileader, 4 leaders: 3.695193e-03 +Nonblocking Locality Aware, 4 leaders: 4.590393e-04 +Nonblocking Multileader Locality, 4 leaders: 4.213373e-03 +Pairwise Multileader, 10 leaders: 1.785995e-03 +Pairwise Locality Aware, 10 leaders: 5.548217e-04 +Pairwise Multileader Locality, 10 leaders: 1.848488e-03 +Nonblocking Multileader, 10 leaders: 1.440675e-03 +Nonblocking Locality Aware, 10 leaders: 4.885241e-04 +Nonblocking Multileader Locality, 10 leaders: 1.722277e-03 +Pairwise Multileader, 20 leaders: 1.043241e-03 +Pairwise Locality Aware, 20 leaders: 5.571958e-04 +Pairwise Multileader Locality, 20 leaders: 1.084621e-03 +Nonblocking Multileader, 20 leaders: 7.291687e-04 +Nonblocking Locality Aware, 20 leaders: 4.707961e-04 +Nonblocking Multileader Locality, 20 leaders: 9.941945e-04 +Size 256 +PMPI: 1.508566e-03 +Pairwise: 1.259965e-03 +NonBlocking: 3.779595e-02 +Pairwise Hierarchical: 2.856783e-02 +Pairwise Node Aware: 1.231135e-03 +Nonblocking Hierarchical: 2.835080e-02 +Nonblocking Node Aware: 8.945426e-04 +Pairwise Multileader, 4 leaders: 7.750728e-03 +Pairwise Locality Aware, 4 leaders: 1.045405e-03 +Pairwise Multileader Locality, 4 leaders: 8.340628e-03 +Nonblocking Multileader, 4 leaders: 6.608941e-03 +Nonblocking Locality Aware, 4 leaders: 8.920820e-04 +Nonblocking Multileader Locality, 4 leaders: 7.638827e-03 +Pairwise Multileader, 10 leaders: 4.335449e-03 +Pairwise Locality Aware, 10 leaders: 1.041713e-03 +Pairwise Multileader Locality, 10 leaders: 3.680945e-03 +Nonblocking Multileader, 10 leaders: 3.184170e-03 +Nonblocking Locality Aware, 10 leaders: 9.044426e-04 +Nonblocking Multileader Locality, 10 leaders: 3.527486e-03 +Pairwise Multileader, 20 leaders: 2.192158e-03 +Pairwise Locality Aware, 20 leaders: 1.024986e-03 +Pairwise Multileader Locality, 20 leaders: 2.221766e-03 +Nonblocking Multileader, 20 leaders: 1.575399e-03 +Nonblocking Locality Aware, 20 leaders: 8.753789e-04 +Nonblocking Multileader Locality, 20 leaders: 1.984794e-03 +Size 512 +PMPI: 2.037748e-03 +Pairwise: 2.036793e-03 +NonBlocking: 3.913240e-03 +Pairwise Hierarchical: 5.735577e-02 +Pairwise Node Aware: 2.185600e-03 +Nonblocking Hierarchical: 5.878112e-02 +Nonblocking Node Aware: 1.739857e-03 +Pairwise Multileader, 4 leaders: 1.702688e-02 +Pairwise Locality Aware, 4 leaders: 2.022183e-03 +Pairwise Multileader Locality, 4 leaders: 1.673075e-02 +Nonblocking Multileader, 4 leaders: 1.389047e-02 +Nonblocking Locality Aware, 4 leaders: 1.660096e-03 +Nonblocking Multileader Locality, 4 leaders: 1.544870e-02 +Pairwise Multileader, 10 leaders: 9.105158e-03 +Pairwise Locality Aware, 10 leaders: 2.008375e-03 +Pairwise Multileader Locality, 10 leaders: 8.080256e-03 +Nonblocking Multileader, 10 leaders: 6.805010e-03 +Nonblocking Locality Aware, 10 leaders: 1.719251e-03 +Nonblocking Multileader Locality, 10 leaders: 7.604666e-03 +Pairwise Multileader, 20 leaders: 4.406236e-03 +Pairwise Locality Aware, 20 leaders: 2.156721e-03 +Pairwise Multileader Locality, 20 leaders: 4.174549e-03 +Nonblocking Multileader, 20 leaders: 3.098402e-03 +Nonblocking Locality Aware, 20 leaders: 2.156028e-03 +Nonblocking Multileader Locality, 20 leaders: 3.722081e-03 +Size 1024 +PMPI: 4.146456e-03 +Pairwise: 3.945222e-03 +NonBlocking: 5.533278e-03 +Pairwise Hierarchical: 1.067379e-01 +Pairwise Node Aware: 4.403534e-03 +Nonblocking Hierarchical: 1.101147e-01 +Nonblocking Node Aware: 3.558316e-03 +Pairwise Multileader, 4 leaders: 3.872780e-02 +Pairwise Locality Aware, 4 leaders: 4.151841e-03 +Pairwise Multileader Locality, 4 leaders: 3.931550e-02 +Nonblocking Multileader, 4 leaders: 3.244212e-02 +Nonblocking Locality Aware, 4 leaders: 3.546098e-03 +Nonblocking Multileader Locality, 4 leaders: 3.726974e-02 +Pairwise Multileader, 10 leaders: 2.009482e-02 +Pairwise Locality Aware, 10 leaders: 4.006854e-03 +Pairwise Multileader Locality, 10 leaders: 1.905339e-02 +Nonblocking Multileader, 10 leaders: 1.534308e-02 +Nonblocking Locality Aware, 10 leaders: 3.564363e-03 +Nonblocking Multileader Locality, 10 leaders: 1.777963e-02 +Pairwise Multileader, 20 leaders: 1.142777e-02 +Pairwise Locality Aware, 20 leaders: 4.247808e-03 +Pairwise Multileader Locality, 20 leaders: 8.979520e-03 +Nonblocking Multileader, 20 leaders: 7.737806e-03 +Nonblocking Locality Aware, 20 leaders: 3.371158e-03 +Nonblocking Multileader Locality, 20 leaders: 8.210762e-03 +Size 2048 +PMPI: 8.018127e-03 +Pairwise: 8.287621e-03 +NonBlocking: 1.161191e-02 +Pairwise Hierarchical: 2.102681e-01 +Pairwise Node Aware: 9.666443e-03 +Nonblocking Hierarchical: 2.142556e-01 +Nonblocking Node Aware: 8.236685e-03 +Pairwise Multileader, 4 leaders: 7.442760e-02 +Pairwise Locality Aware, 4 leaders: 7.923967e-03 +Pairwise Multileader Locality, 4 leaders: 7.857282e-02 +Nonblocking Multileader, 4 leaders: 6.256120e-02 +Nonblocking Locality Aware, 4 leaders: 7.360620e-03 +Nonblocking Multileader Locality, 4 leaders: 7.346496e-02 +Pairwise Multileader, 10 leaders: 4.047580e-02 +Pairwise Locality Aware, 10 leaders: 8.165665e-03 +Pairwise Multileader Locality, 10 leaders: 4.082532e-02 +Nonblocking Multileader, 10 leaders: 3.239103e-02 +Nonblocking Locality Aware, 10 leaders: 6.878389e-03 +Nonblocking Multileader Locality, 10 leaders: 3.863433e-02 +Pairwise Multileader, 20 leaders: 2.390650e-02 +Pairwise Locality Aware, 20 leaders: 9.104579e-03 +Pairwise Multileader Locality, 20 leaders: 2.069807e-02 +Nonblocking Multileader, 20 leaders: 1.671440e-02 +Nonblocking Locality Aware, 20 leaders: 6.908327e-03 +Nonblocking Multileader Locality, 20 leaders: 1.897347e-02 +Size 4096 +PMPI: 1.622161e-02 +Pairwise: 1.463829e-02 +NonBlocking: 1.948034e-02 +Pairwise Hierarchical: 4.406074e-01 +Pairwise Node Aware: 2.098103e-02 +Nonblocking Hierarchical: 4.424690e-01 +Nonblocking Node Aware: 1.828022e-02 +Pairwise Multileader, 4 leaders: 1.358785e-01 +Pairwise Locality Aware, 4 leaders: 1.854422e-02 +Pairwise Multileader Locality, 4 leaders: 1.512690e-01 +Nonblocking Multileader, 4 leaders: 1.208092e-01 +Nonblocking Locality Aware, 4 leaders: 1.644484e-02 +Nonblocking Multileader Locality, 4 leaders: 1.395908e-01 +Pairwise Multileader, 10 leaders: 8.192596e-02 +Pairwise Locality Aware, 10 leaders: 1.908328e-02 +Pairwise Multileader Locality, 10 leaders: 8.546867e-02 +Nonblocking Multileader, 10 leaders: 6.596704e-02 +Nonblocking Locality Aware, 10 leaders: 1.572424e-02 +Nonblocking Multileader Locality, 10 leaders: 8.234881e-02 +Pairwise Multileader, 20 leaders: 4.652342e-02 +Pairwise Locality Aware, 20 leaders: 1.943976e-02 +Pairwise Multileader Locality, 20 leaders: 4.374146e-02 +Nonblocking Multileader, 20 leaders: 3.354452e-02 +Nonblocking Locality Aware, 20 leaders: 1.572550e-02 +Nonblocking Multileader Locality, 20 leaders: 4.077548e-02 +Size 8192 +PMPI: 2.925450e-02 +Pairwise: 2.868695e-02 +NonBlocking: 3.047572e-02 +Pairwise Hierarchical: 9.234946e-01 +Pairwise Node Aware: 3.957082e-02 +Nonblocking Hierarchical: 9.269619e-01 +Nonblocking Node Aware: 3.667192e-02 +Pairwise Multileader, 4 leaders: 2.617182e-01 +Pairwise Locality Aware, 4 leaders: 3.865418e-02 +Pairwise Multileader Locality, 4 leaders: 2.915946e-01 +Nonblocking Multileader, 4 leaders: 2.287511e-01 +Nonblocking Locality Aware, 4 leaders: 3.516459e-02 +Nonblocking Multileader Locality, 4 leaders: 2.706028e-01 +Pairwise Multileader, 10 leaders: 1.552317e-01 +Pairwise Locality Aware, 10 leaders: 3.955198e-02 +Pairwise Multileader Locality, 10 leaders: 1.654255e-01 +Nonblocking Multileader, 10 leaders: 1.296292e-01 +Nonblocking Locality Aware, 10 leaders: 3.481738e-02 +Nonblocking Multileader Locality, 10 leaders: 1.579266e-01 +Pairwise Multileader, 20 leaders: 9.456175e-02 +Pairwise Locality Aware, 20 leaders: 4.036840e-02 +Pairwise Multileader Locality, 20 leaders: 9.968922e-02 +Nonblocking Multileader, 20 leaders: 7.543045e-02 +Nonblocking Locality Aware, 20 leaders: 3.446077e-02 +Nonblocking Multileader Locality, 20 leaders: 9.080911e-02 +Size 16384 +PMPI: 5.424783e-02 +Pairwise: 5.424902e-02 +NonBlocking: 5.353987e-02 +Pairwise Hierarchical: 1.904804e+00 +Pairwise Node Aware: 7.917527e-02 +Nonblocking Hierarchical: 1.944009e+00 +Nonblocking Node Aware: 7.314226e-02 +Pairwise Multileader, 4 leaders: 5.439906e-01 +Pairwise Locality Aware, 4 leaders: 7.835511e-02 +Pairwise Multileader Locality, 4 leaders: 6.268772e-01 +Nonblocking Multileader, 4 leaders: 5.030936e-01 +Nonblocking Locality Aware, 4 leaders: 7.145958e-02 +Nonblocking Multileader Locality, 4 leaders: 5.755191e-01 +Pairwise Multileader, 10 leaders: 3.262181e-01 +Pairwise Locality Aware, 10 leaders: 8.066027e-02 +Pairwise Multileader Locality, 10 leaders: 3.710022e-01 +Nonblocking Multileader, 10 leaders: 2.576226e-01 +Nonblocking Locality Aware, 10 leaders: 7.236344e-02 +Nonblocking Multileader Locality, 10 leaders: 3.126534e-01 +Pairwise Multileader, 20 leaders: 1.735489e-01 +Pairwise Locality Aware, 20 leaders: 8.181963e-02 +Pairwise Multileader Locality, 20 leaders: 1.919578e-01 +Nonblocking Multileader, 20 leaders: 1.479712e-01 +Nonblocking Locality Aware, 20 leaders: 7.477232e-02 +Nonblocking Multileader Locality, 20 leaders: 1.752366e-01 +Test 2 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 4.255111e-05 +Pairwise: 4.234065e-04 +NonBlocking: 1.911344e-04 +Pairwise Hierarchical: 1.808474e-04 +Pairwise Node Aware: 1.739545e-04 +Nonblocking Hierarchical: 1.844626e-04 +Nonblocking Node Aware: 1.682724e-04 +Pairwise Multileader, 4 leaders: 6.377748e-05 +Pairwise Locality Aware, 4 leaders: 7.261984e-05 +Pairwise Multileader Locality, 4 leaders: 5.914736e-05 +Nonblocking Multileader, 4 leaders: 4.745406e-05 +Nonblocking Locality Aware, 4 leaders: 4.673290e-05 +Nonblocking Multileader Locality, 4 leaders: 5.513354e-05 +Pairwise Multileader, 10 leaders: 8.435598e-05 +Pairwise Locality Aware, 10 leaders: 9.171713e-05 +Pairwise Multileader Locality, 10 leaders: 3.959500e-05 +Nonblocking Multileader, 10 leaders: 3.602806e-05 +Nonblocking Locality Aware, 10 leaders: 3.823860e-05 +Nonblocking Multileader Locality, 10 leaders: 3.570811e-05 +Pairwise Multileader, 20 leaders: 1.620547e-04 +Pairwise Locality Aware, 20 leaders: 1.625197e-04 +Pairwise Multileader Locality, 20 leaders: 5.531955e-05 +Nonblocking Multileader, 20 leaders: 5.471164e-05 +Nonblocking Locality Aware, 20 leaders: 5.290098e-05 +Nonblocking Multileader Locality, 20 leaders: 4.134016e-05 +Size 2 +PMPI: 4.441198e-05 +Pairwise: 3.757704e-04 +NonBlocking: 1.716168e-04 +Pairwise Hierarchical: 2.929555e-04 +Pairwise Node Aware: 1.772972e-04 +Nonblocking Hierarchical: 2.900124e-04 +Nonblocking Node Aware: 1.683370e-04 +Pairwise Multileader, 4 leaders: 7.775903e-05 +Pairwise Locality Aware, 4 leaders: 7.538747e-05 +Pairwise Multileader Locality, 4 leaders: 7.953024e-05 +Nonblocking Multileader, 4 leaders: 6.297378e-05 +Nonblocking Locality Aware, 4 leaders: 4.697469e-05 +Nonblocking Multileader Locality, 4 leaders: 7.538303e-05 +Pairwise Multileader, 10 leaders: 9.052675e-05 +Pairwise Locality Aware, 10 leaders: 9.405362e-05 +Pairwise Multileader Locality, 10 leaders: 5.072834e-05 +Nonblocking Multileader, 10 leaders: 4.152809e-05 +Nonblocking Locality Aware, 10 leaders: 3.959889e-05 +Nonblocking Multileader Locality, 10 leaders: 4.428283e-05 +Pairwise Multileader, 20 leaders: 1.643712e-04 +Pairwise Locality Aware, 20 leaders: 1.581463e-04 +Pairwise Multileader Locality, 20 leaders: 6.338198e-05 +Nonblocking Multileader, 20 leaders: 5.930749e-05 +Nonblocking Locality Aware, 20 leaders: 5.329107e-05 +Nonblocking Multileader Locality, 20 leaders: 4.634383e-05 +Size 4 +PMPI: 5.470896e-05 +Pairwise: 7.638409e-04 +NonBlocking: 1.080898e-03 +Pairwise Hierarchical: 4.320302e-04 +Pairwise Node Aware: 2.399370e-04 +Nonblocking Hierarchical: 4.358597e-04 +Nonblocking Node Aware: 1.737883e-04 +Pairwise Multileader, 4 leaders: 1.145047e-04 +Pairwise Locality Aware, 4 leaders: 8.195359e-05 +Pairwise Multileader Locality, 4 leaders: 1.117310e-04 +Nonblocking Multileader, 4 leaders: 9.028631e-05 +Nonblocking Locality Aware, 4 leaders: 4.922424e-05 +Nonblocking Multileader Locality, 4 leaders: 1.052105e-04 +Pairwise Multileader, 10 leaders: 1.004771e-04 +Pairwise Locality Aware, 10 leaders: 9.562576e-05 +Pairwise Multileader Locality, 10 leaders: 6.638800e-05 +Nonblocking Multileader, 10 leaders: 5.066249e-05 +Nonblocking Locality Aware, 10 leaders: 4.536030e-05 +Nonblocking Multileader Locality, 10 leaders: 5.829216e-05 +Pairwise Multileader, 20 leaders: 1.687841e-04 +Pairwise Locality Aware, 20 leaders: 1.645051e-04 +Pairwise Multileader Locality, 20 leaders: 7.842249e-05 +Nonblocking Multileader, 20 leaders: 6.461848e-05 +Nonblocking Locality Aware, 20 leaders: 5.992921e-05 +Nonblocking Multileader Locality, 20 leaders: 5.808418e-05 +Size 8 +PMPI: 7.316961e-05 +Pairwise: 7.800603e-04 +NonBlocking: 1.206558e-03 +Pairwise Hierarchical: 8.062975e-04 +Pairwise Node Aware: 2.452300e-04 +Nonblocking Hierarchical: 7.602136e-04 +Nonblocking Node Aware: 1.733395e-04 +Pairwise Multileader, 4 leaders: 2.223280e-04 +Pairwise Locality Aware, 4 leaders: 8.895697e-05 +Pairwise Multileader Locality, 4 leaders: 3.038804e-04 +Nonblocking Multileader, 4 leaders: 1.857730e-04 +Nonblocking Locality Aware, 4 leaders: 6.809908e-05 +Nonblocking Multileader Locality, 4 leaders: 2.560687e-04 +Pairwise Multileader, 10 leaders: 1.226759e-04 +Pairwise Locality Aware, 10 leaders: 9.674734e-05 +Pairwise Multileader Locality, 10 leaders: 1.008003e-04 +Nonblocking Multileader, 10 leaders: 7.141565e-05 +Nonblocking Locality Aware, 10 leaders: 5.345153e-05 +Nonblocking Multileader Locality, 10 leaders: 8.547502e-05 +Pairwise Multileader, 20 leaders: 1.800979e-04 +Pairwise Locality Aware, 20 leaders: 1.676302e-04 +Pairwise Multileader Locality, 20 leaders: 1.027182e-04 +Nonblocking Multileader, 20 leaders: 7.705192e-05 +Nonblocking Locality Aware, 20 leaders: 7.065298e-05 +Nonblocking Multileader Locality, 20 leaders: 7.814166e-05 +Size 16 +PMPI: 1.300794e-04 +Pairwise: 7.812849e-04 +NonBlocking: 1.374338e-03 +Pairwise Hierarchical: 1.459622e-03 +Pairwise Node Aware: 3.203580e-04 +Nonblocking Hierarchical: 1.456013e-03 +Nonblocking Node Aware: 1.759660e-04 +Pairwise Multileader, 4 leaders: 4.431549e-04 +Pairwise Locality Aware, 4 leaders: 1.168051e-04 +Pairwise Multileader Locality, 4 leaders: 5.374180e-04 +Nonblocking Multileader, 4 leaders: 3.579947e-04 +Nonblocking Locality Aware, 4 leaders: 7.994640e-05 +Nonblocking Multileader Locality, 4 leaders: 4.691730e-04 +Pairwise Multileader, 10 leaders: 2.271420e-04 +Pairwise Locality Aware, 10 leaders: 1.156489e-04 +Pairwise Multileader Locality, 10 leaders: 1.930498e-04 +Nonblocking Multileader, 10 leaders: 1.435451e-04 +Nonblocking Locality Aware, 10 leaders: 8.592250e-05 +Nonblocking Multileader Locality, 10 leaders: 1.665628e-04 +Pairwise Multileader, 20 leaders: 2.073328e-04 +Pairwise Locality Aware, 20 leaders: 1.739235e-04 +Pairwise Multileader Locality, 20 leaders: 1.503638e-04 +Nonblocking Multileader, 20 leaders: 1.034667e-04 +Nonblocking Locality Aware, 20 leaders: 9.271360e-05 +Nonblocking Multileader Locality, 20 leaders: 1.115109e-04 +Size 32 +PMPI: 2.328450e-04 +Pairwise: 7.926904e-04 +NonBlocking: 2.919676e-03 +Pairwise Hierarchical: 3.053194e-03 +Pairwise Node Aware: 3.880791e-04 +Nonblocking Hierarchical: 2.909772e-03 +Nonblocking Node Aware: 2.025346e-04 +Pairwise Multileader, 4 leaders: 1.014269e-03 +Pairwise Locality Aware, 4 leaders: 1.842157e-04 +Pairwise Multileader Locality, 4 leaders: 1.033953e-03 +Nonblocking Multileader, 4 leaders: 8.661054e-04 +Nonblocking Locality Aware, 4 leaders: 1.470749e-04 +Nonblocking Multileader Locality, 4 leaders: 9.497854e-04 +Pairwise Multileader, 10 leaders: 4.299607e-04 +Pairwise Locality Aware, 10 leaders: 1.696772e-04 +Pairwise Multileader Locality, 10 leaders: 4.354284e-04 +Nonblocking Multileader, 10 leaders: 3.085284e-04 +Nonblocking Locality Aware, 10 leaders: 1.475307e-04 +Nonblocking Multileader Locality, 10 leaders: 3.966379e-04 +Pairwise Multileader, 20 leaders: 3.084721e-04 +Pairwise Locality Aware, 20 leaders: 2.041774e-04 +Pairwise Multileader Locality, 20 leaders: 2.522490e-04 +Nonblocking Multileader, 20 leaders: 1.695719e-04 +Nonblocking Locality Aware, 20 leaders: 1.556063e-04 +Nonblocking Multileader Locality, 20 leaders: 2.019446e-04 +Size 64 +PMPI: 4.228530e-04 +Pairwise: 8.055664e-04 +NonBlocking: 5.101863e-02 +Pairwise Hierarchical: 6.933246e-03 +Pairwise Node Aware: 4.761551e-04 +Nonblocking Hierarchical: 6.627770e-03 +Nonblocking Node Aware: 3.014471e-04 +Pairwise Multileader, 4 leaders: 2.273784e-03 +Pairwise Locality Aware, 4 leaders: 3.115649e-04 +Pairwise Multileader Locality, 4 leaders: 2.328181e-03 +Nonblocking Multileader, 4 leaders: 1.912207e-03 +Nonblocking Locality Aware, 4 leaders: 2.390942e-04 +Nonblocking Multileader Locality, 4 leaders: 2.145424e-03 +Pairwise Multileader, 10 leaders: 8.968378e-04 +Pairwise Locality Aware, 10 leaders: 2.924344e-04 +Pairwise Multileader Locality, 10 leaders: 9.272910e-04 +Nonblocking Multileader, 10 leaders: 6.919353e-04 +Nonblocking Locality Aware, 10 leaders: 2.534704e-04 +Nonblocking Multileader Locality, 10 leaders: 8.280063e-04 +Pairwise Multileader, 20 leaders: 5.621545e-04 +Pairwise Locality Aware, 20 leaders: 3.231325e-04 +Pairwise Multileader Locality, 20 leaders: 5.413051e-04 +Nonblocking Multileader, 20 leaders: 3.425704e-04 +Nonblocking Locality Aware, 20 leaders: 2.772412e-04 +Nonblocking Multileader Locality, 20 leaders: 4.575999e-04 +Size 128 +PMPI: 8.030301e-04 +Pairwise: 8.974581e-04 +NonBlocking: 1.300850e-02 +Pairwise Hierarchical: 1.414901e-02 +Pairwise Node Aware: 7.456847e-04 +Nonblocking Hierarchical: 1.350797e-02 +Nonblocking Node Aware: 5.018478e-04 +Pairwise Multileader, 4 leaders: 4.462630e-03 +Pairwise Locality Aware, 4 leaders: 5.727159e-04 +Pairwise Multileader Locality, 4 leaders: 4.656898e-03 +Nonblocking Multileader, 4 leaders: 3.728655e-03 +Nonblocking Locality Aware, 4 leaders: 4.717669e-04 +Nonblocking Multileader Locality, 4 leaders: 4.300708e-03 +Pairwise Multileader, 10 leaders: 1.805425e-03 +Pairwise Locality Aware, 10 leaders: 5.959337e-04 +Pairwise Multileader Locality, 10 leaders: 1.855575e-03 +Nonblocking Multileader, 10 leaders: 1.448535e-03 +Nonblocking Locality Aware, 10 leaders: 5.267209e-04 +Nonblocking Multileader Locality, 10 leaders: 1.705327e-03 +Pairwise Multileader, 20 leaders: 1.078913e-03 +Pairwise Locality Aware, 20 leaders: 5.624681e-04 +Pairwise Multileader Locality, 20 leaders: 1.079717e-03 +Nonblocking Multileader, 20 leaders: 7.235507e-04 +Nonblocking Locality Aware, 20 leaders: 5.256266e-04 +Nonblocking Multileader Locality, 20 leaders: 9.903382e-04 +Size 256 +PMPI: 1.487206e-03 +Pairwise: 1.300976e-03 +NonBlocking: 3.774357e-02 +Pairwise Hierarchical: 2.949788e-02 +Pairwise Node Aware: 1.233061e-03 +Nonblocking Hierarchical: 2.896790e-02 +Nonblocking Node Aware: 8.942960e-04 +Pairwise Multileader, 4 leaders: 7.848192e-03 +Pairwise Locality Aware, 4 leaders: 1.012174e-03 +Pairwise Multileader Locality, 4 leaders: 8.471324e-03 +Nonblocking Multileader, 4 leaders: 6.609230e-03 +Nonblocking Locality Aware, 4 leaders: 8.501723e-04 +Nonblocking Multileader Locality, 4 leaders: 7.757860e-03 +Pairwise Multileader, 10 leaders: 4.447008e-03 +Pairwise Locality Aware, 10 leaders: 1.115610e-03 +Pairwise Multileader Locality, 10 leaders: 3.758941e-03 +Nonblocking Multileader, 10 leaders: 3.235638e-03 +Nonblocking Locality Aware, 10 leaders: 1.056593e-03 +Nonblocking Multileader Locality, 10 leaders: 3.575443e-03 +Pairwise Multileader, 20 leaders: 2.243774e-03 +Pairwise Locality Aware, 20 leaders: 1.150534e-03 +Pairwise Multileader Locality, 20 leaders: 2.231136e-03 +Nonblocking Multileader, 20 leaders: 1.604883e-03 +Nonblocking Locality Aware, 20 leaders: 1.071816e-03 +Nonblocking Multileader Locality, 20 leaders: 1.969908e-03 +Size 512 +PMPI: 2.264534e-03 +Pairwise: 2.216952e-03 +NonBlocking: 1.528832e-02 +Pairwise Hierarchical: 5.813032e-02 +Pairwise Node Aware: 2.186754e-03 +Nonblocking Hierarchical: 5.991067e-02 +Nonblocking Node Aware: 1.734670e-03 +Pairwise Multileader, 4 leaders: 1.738860e-02 +Pairwise Locality Aware, 4 leaders: 2.047361e-03 +Pairwise Multileader Locality, 4 leaders: 1.681100e-02 +Nonblocking Multileader, 4 leaders: 1.391154e-02 +Nonblocking Locality Aware, 4 leaders: 1.728037e-03 +Nonblocking Multileader Locality, 4 leaders: 1.560315e-02 +Pairwise Multileader, 10 leaders: 9.252328e-03 +Pairwise Locality Aware, 10 leaders: 1.978303e-03 +Pairwise Multileader Locality, 10 leaders: 8.108784e-03 +Nonblocking Multileader, 10 leaders: 6.963815e-03 +Nonblocking Locality Aware, 10 leaders: 1.742665e-03 +Nonblocking Multileader Locality, 10 leaders: 7.654109e-03 +Pairwise Multileader, 20 leaders: 4.350441e-03 +Pairwise Locality Aware, 20 leaders: 1.973299e-03 +Pairwise Multileader Locality, 20 leaders: 4.372817e-03 +Nonblocking Multileader, 20 leaders: 3.061244e-03 +Nonblocking Locality Aware, 20 leaders: 1.700247e-03 +Nonblocking Multileader Locality, 20 leaders: 3.880790e-03 +Size 1024 +PMPI: 3.789376e-03 +Pairwise: 3.804029e-03 +NonBlocking: 6.560312e-03 +Pairwise Hierarchical: 1.058461e-01 +Pairwise Node Aware: 4.397617e-03 +Nonblocking Hierarchical: 1.096287e-01 +Nonblocking Node Aware: 3.553075e-03 +Pairwise Multileader, 4 leaders: 3.936367e-02 +Pairwise Locality Aware, 4 leaders: 4.052297e-03 +Pairwise Multileader Locality, 4 leaders: 3.977173e-02 +Nonblocking Multileader, 4 leaders: 3.244798e-02 +Nonblocking Locality Aware, 4 leaders: 3.524699e-03 +Nonblocking Multileader Locality, 4 leaders: 3.700596e-02 +Pairwise Multileader, 10 leaders: 2.016380e-02 +Pairwise Locality Aware, 10 leaders: 3.919060e-03 +Pairwise Multileader Locality, 10 leaders: 1.907383e-02 +Nonblocking Multileader, 10 leaders: 1.545449e-02 +Nonblocking Locality Aware, 10 leaders: 3.339616e-03 +Nonblocking Multileader Locality, 10 leaders: 1.770663e-02 +Pairwise Multileader, 20 leaders: 1.140017e-02 +Pairwise Locality Aware, 20 leaders: 4.021979e-03 +Pairwise Multileader Locality, 20 leaders: 9.155321e-03 +Nonblocking Multileader, 20 leaders: 7.733105e-03 +Nonblocking Locality Aware, 20 leaders: 3.425317e-03 +Nonblocking Multileader Locality, 20 leaders: 8.285092e-03 +Size 2048 +PMPI: 7.685208e-03 +Pairwise: 7.702628e-03 +NonBlocking: 9.313329e-03 +Pairwise Hierarchical: 2.113335e-01 +Pairwise Node Aware: 9.689595e-03 +Nonblocking Hierarchical: 2.146973e-01 +Nonblocking Node Aware: 8.311781e-03 +Pairwise Multileader, 4 leaders: 7.387711e-02 +Pairwise Locality Aware, 4 leaders: 8.103065e-03 +Pairwise Multileader Locality, 4 leaders: 7.915618e-02 +Nonblocking Multileader, 4 leaders: 6.301880e-02 +Nonblocking Locality Aware, 4 leaders: 7.388705e-03 +Nonblocking Multileader Locality, 4 leaders: 7.338040e-02 +Pairwise Multileader, 10 leaders: 4.050558e-02 +Pairwise Locality Aware, 10 leaders: 8.117175e-03 +Pairwise Multileader Locality, 10 leaders: 4.075472e-02 +Nonblocking Multileader, 10 leaders: 3.233335e-02 +Nonblocking Locality Aware, 10 leaders: 6.959906e-03 +Nonblocking Multileader Locality, 10 leaders: 3.851834e-02 +Pairwise Multileader, 20 leaders: 2.479261e-02 +Pairwise Locality Aware, 20 leaders: 8.727084e-03 +Pairwise Multileader Locality, 20 leaders: 2.078805e-02 +Nonblocking Multileader, 20 leaders: 1.685542e-02 +Nonblocking Locality Aware, 20 leaders: 6.832996e-03 +Nonblocking Multileader Locality, 20 leaders: 1.902408e-02 +Size 4096 +PMPI: 1.752042e-02 +Pairwise: 1.495629e-02 +NonBlocking: 1.960407e-02 +Pairwise Hierarchical: 4.407202e-01 +Pairwise Node Aware: 2.109128e-02 +Nonblocking Hierarchical: 4.411558e-01 +Nonblocking Node Aware: 1.821729e-02 +Pairwise Multileader, 4 leaders: 1.372872e-01 +Pairwise Locality Aware, 4 leaders: 1.877788e-02 +Pairwise Multileader Locality, 4 leaders: 1.520663e-01 +Nonblocking Multileader, 4 leaders: 1.213949e-01 +Nonblocking Locality Aware, 4 leaders: 1.658897e-02 +Nonblocking Multileader Locality, 4 leaders: 1.397489e-01 +Pairwise Multileader, 10 leaders: 8.192348e-02 +Pairwise Locality Aware, 10 leaders: 1.895413e-02 +Pairwise Multileader Locality, 10 leaders: 8.610491e-02 +Nonblocking Multileader, 10 leaders: 6.609441e-02 +Nonblocking Locality Aware, 10 leaders: 1.535226e-02 +Nonblocking Multileader Locality, 10 leaders: 8.300462e-02 +Pairwise Multileader, 20 leaders: 4.729319e-02 +Pairwise Locality Aware, 20 leaders: 1.942917e-02 +Pairwise Multileader Locality, 20 leaders: 4.418105e-02 +Nonblocking Multileader, 20 leaders: 3.401980e-02 +Nonblocking Locality Aware, 20 leaders: 1.569018e-02 +Nonblocking Multileader Locality, 20 leaders: 4.072068e-02 +Size 8192 +PMPI: 2.877526e-02 +Pairwise: 3.021941e-02 +NonBlocking: 2.850281e-02 +Pairwise Hierarchical: 9.339098e-01 +Pairwise Node Aware: 3.975509e-02 +Nonblocking Hierarchical: 9.429677e-01 +Nonblocking Node Aware: 3.665777e-02 +Pairwise Multileader, 4 leaders: 2.622502e-01 +Pairwise Locality Aware, 4 leaders: 3.900364e-02 +Pairwise Multileader Locality, 4 leaders: 2.919406e-01 +Nonblocking Multileader, 4 leaders: 2.284499e-01 +Nonblocking Locality Aware, 4 leaders: 3.537097e-02 +Nonblocking Multileader Locality, 4 leaders: 2.692904e-01 +Pairwise Multileader, 10 leaders: 1.557446e-01 +Pairwise Locality Aware, 10 leaders: 3.969080e-02 +Pairwise Multileader Locality, 10 leaders: 1.659877e-01 +Nonblocking Multileader, 10 leaders: 1.296425e-01 +Nonblocking Locality Aware, 10 leaders: 3.531601e-02 +Nonblocking Multileader Locality, 10 leaders: 1.574660e-01 +Pairwise Multileader, 20 leaders: 9.492367e-02 +Pairwise Locality Aware, 20 leaders: 4.046664e-02 +Pairwise Multileader Locality, 20 leaders: 9.976270e-02 +Nonblocking Multileader, 20 leaders: 7.561201e-02 +Nonblocking Locality Aware, 20 leaders: 3.419757e-02 +Nonblocking Multileader Locality, 20 leaders: 9.051698e-02 +Size 16384 +PMPI: 5.464322e-02 +Pairwise: 5.430092e-02 +NonBlocking: 5.353704e-02 +Pairwise Hierarchical: 1.910568e+00 +Pairwise Node Aware: 7.987998e-02 +Nonblocking Hierarchical: 1.928501e+00 +Nonblocking Node Aware: 7.357054e-02 +Pairwise Multileader, 4 leaders: 5.449728e-01 +Pairwise Locality Aware, 4 leaders: 7.891235e-02 +Pairwise Multileader Locality, 4 leaders: 6.203524e-01 +Nonblocking Multileader, 4 leaders: 4.990047e-01 +Nonblocking Locality Aware, 4 leaders: 7.158271e-02 +Nonblocking Multileader Locality, 4 leaders: 5.782865e-01 +Pairwise Multileader, 10 leaders: 3.241943e-01 +Pairwise Locality Aware, 10 leaders: 8.041701e-02 +Pairwise Multileader Locality, 10 leaders: 3.695862e-01 +Nonblocking Multileader, 10 leaders: 2.585933e-01 +Nonblocking Locality Aware, 10 leaders: 7.262065e-02 +Nonblocking Multileader Locality, 10 leaders: 3.129381e-01 +Pairwise Multileader, 20 leaders: 1.740910e-01 +Pairwise Locality Aware, 20 leaders: 8.178097e-02 +Pairwise Multileader Locality, 20 leaders: 1.922119e-01 +Nonblocking Multileader, 20 leaders: 1.479732e-01 +Nonblocking Locality Aware, 20 leaders: 7.458692e-02 +Nonblocking Multileader Locality, 20 leaders: 1.758090e-01 +Test 3 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 4.271415e-05 +Pairwise: 3.888763e-04 +NonBlocking: 1.840483e-04 +Pairwise Hierarchical: 1.824120e-04 +Pairwise Node Aware: 1.749983e-04 +Nonblocking Hierarchical: 1.869690e-04 +Nonblocking Node Aware: 1.726731e-04 +Pairwise Multileader, 4 leaders: 6.465291e-05 +Pairwise Locality Aware, 4 leaders: 7.460645e-05 +Pairwise Multileader Locality, 4 leaders: 5.897754e-05 +Nonblocking Multileader, 4 leaders: 4.766020e-05 +Nonblocking Locality Aware, 4 leaders: 4.854677e-05 +Nonblocking Multileader Locality, 4 leaders: 5.504315e-05 +Pairwise Multileader, 10 leaders: 8.725538e-05 +Pairwise Locality Aware, 10 leaders: 9.195267e-05 +Pairwise Multileader Locality, 10 leaders: 3.936767e-05 +Nonblocking Multileader, 10 leaders: 3.672049e-05 +Nonblocking Locality Aware, 10 leaders: 4.211281e-05 +Nonblocking Multileader Locality, 10 leaders: 3.606460e-05 +Pairwise Multileader, 20 leaders: 1.620191e-04 +Pairwise Locality Aware, 20 leaders: 1.616722e-04 +Pairwise Multileader Locality, 20 leaders: 5.532008e-05 +Nonblocking Multileader, 20 leaders: 5.558901e-05 +Nonblocking Locality Aware, 20 leaders: 4.911871e-05 +Nonblocking Multileader Locality, 20 leaders: 4.169885e-05 +Size 2 +PMPI: 4.482203e-05 +Pairwise: 3.811335e-04 +NonBlocking: 1.776434e-04 +Pairwise Hierarchical: 3.096523e-04 +Pairwise Node Aware: 1.772840e-04 +Nonblocking Hierarchical: 2.954394e-04 +Nonblocking Node Aware: 1.739282e-04 +Pairwise Multileader, 4 leaders: 7.924504e-05 +Pairwise Locality Aware, 4 leaders: 7.358101e-05 +Pairwise Multileader Locality, 4 leaders: 8.003704e-05 +Nonblocking Multileader, 4 leaders: 6.279560e-05 +Nonblocking Locality Aware, 4 leaders: 4.712176e-05 +Nonblocking Multileader Locality, 4 leaders: 7.530611e-05 +Pairwise Multileader, 10 leaders: 1.785988e-04 +Pairwise Locality Aware, 10 leaders: 9.481921e-05 +Pairwise Multileader Locality, 10 leaders: 5.056067e-05 +Nonblocking Multileader, 10 leaders: 4.190633e-05 +Nonblocking Locality Aware, 10 leaders: 4.208529e-05 +Nonblocking Multileader Locality, 10 leaders: 4.412300e-05 +Pairwise Multileader, 20 leaders: 1.632665e-04 +Pairwise Locality Aware, 20 leaders: 1.593334e-04 +Pairwise Multileader Locality, 20 leaders: 6.397261e-05 +Nonblocking Multileader, 20 leaders: 5.951390e-05 +Nonblocking Locality Aware, 20 leaders: 5.472171e-05 +Nonblocking Multileader Locality, 20 leaders: 4.649619e-05 +Size 4 +PMPI: 5.375966e-05 +Pairwise: 7.791951e-04 +NonBlocking: 8.903343e-04 +Pairwise Hierarchical: 4.330530e-04 +Pairwise Node Aware: 2.346815e-04 +Nonblocking Hierarchical: 4.400307e-04 +Nonblocking Node Aware: 1.753431e-04 +Pairwise Multileader, 4 leaders: 1.155813e-04 +Pairwise Locality Aware, 4 leaders: 8.201044e-05 +Pairwise Multileader Locality, 4 leaders: 1.135702e-04 +Nonblocking Multileader, 4 leaders: 8.895081e-05 +Nonblocking Locality Aware, 4 leaders: 5.002451e-05 +Nonblocking Multileader Locality, 4 leaders: 1.090213e-04 +Pairwise Multileader, 10 leaders: 1.007571e-04 +Pairwise Locality Aware, 10 leaders: 9.669059e-05 +Pairwise Multileader Locality, 10 leaders: 6.598431e-05 +Nonblocking Multileader, 10 leaders: 5.199886e-05 +Nonblocking Locality Aware, 10 leaders: 4.990227e-05 +Nonblocking Multileader Locality, 10 leaders: 5.787587e-05 +Pairwise Multileader, 20 leaders: 1.679286e-04 +Pairwise Locality Aware, 20 leaders: 1.637606e-04 +Pairwise Multileader Locality, 20 leaders: 7.665324e-05 +Nonblocking Multileader, 20 leaders: 6.428935e-05 +Nonblocking Locality Aware, 20 leaders: 8.321750e-05 +Nonblocking Multileader Locality, 20 leaders: 5.751624e-05 +Size 8 +PMPI: 7.423834e-05 +Pairwise: 7.758020e-04 +NonBlocking: 1.183215e-03 +Pairwise Hierarchical: 8.313926e-04 +Pairwise Node Aware: 2.559281e-04 +Nonblocking Hierarchical: 7.836657e-04 +Nonblocking Node Aware: 1.755232e-04 +Pairwise Multileader, 4 leaders: 2.203794e-04 +Pairwise Locality Aware, 4 leaders: 8.786082e-05 +Pairwise Multileader Locality, 4 leaders: 2.836280e-04 +Nonblocking Multileader, 4 leaders: 1.796759e-04 +Nonblocking Locality Aware, 4 leaders: 5.575410e-05 +Nonblocking Multileader Locality, 4 leaders: 2.419518e-04 +Pairwise Multileader, 10 leaders: 1.217628e-04 +Pairwise Locality Aware, 10 leaders: 9.763557e-05 +Pairwise Multileader Locality, 10 leaders: 1.036164e-04 +Nonblocking Multileader, 10 leaders: 6.980213e-05 +Nonblocking Locality Aware, 10 leaders: 5.818596e-05 +Nonblocking Multileader Locality, 10 leaders: 8.808895e-05 +Pairwise Multileader, 20 leaders: 1.783110e-04 +Pairwise Locality Aware, 20 leaders: 1.698581e-04 +Pairwise Multileader Locality, 20 leaders: 1.092226e-04 +Nonblocking Multileader, 20 leaders: 7.711299e-05 +Nonblocking Locality Aware, 20 leaders: 6.918873e-05 +Nonblocking Multileader Locality, 20 leaders: 8.431219e-05 +Size 16 +PMPI: 1.316618e-04 +Pairwise: 7.628627e-04 +NonBlocking: 1.455986e-03 +Pairwise Hierarchical: 1.536989e-03 +Pairwise Node Aware: 3.132443e-04 +Nonblocking Hierarchical: 1.504785e-03 +Nonblocking Node Aware: 1.778469e-04 +Pairwise Multileader, 4 leaders: 4.153849e-04 +Pairwise Locality Aware, 4 leaders: 1.152535e-04 +Pairwise Multileader Locality, 4 leaders: 5.007685e-04 +Nonblocking Multileader, 4 leaders: 3.467696e-04 +Nonblocking Locality Aware, 4 leaders: 9.177291e-05 +Nonblocking Multileader Locality, 4 leaders: 4.473201e-04 +Pairwise Multileader, 10 leaders: 2.330024e-04 +Pairwise Locality Aware, 10 leaders: 1.166347e-04 +Pairwise Multileader Locality, 10 leaders: 1.956561e-04 +Nonblocking Multileader, 10 leaders: 1.453622e-04 +Nonblocking Locality Aware, 10 leaders: 9.300099e-05 +Nonblocking Multileader Locality, 10 leaders: 1.694650e-04 +Pairwise Multileader, 20 leaders: 2.100338e-04 +Pairwise Locality Aware, 20 leaders: 1.739104e-04 +Pairwise Multileader Locality, 20 leaders: 1.590533e-04 +Nonblocking Multileader, 20 leaders: 1.059742e-04 +Nonblocking Locality Aware, 20 leaders: 1.217085e-04 +Nonblocking Multileader Locality, 20 leaders: 1.234898e-04 +Size 32 +PMPI: 2.389571e-04 +Pairwise: 7.835799e-04 +NonBlocking: 7.714076e-03 +Pairwise Hierarchical: 3.180821e-03 +Pairwise Node Aware: 4.053690e-04 +Nonblocking Hierarchical: 3.053709e-03 +Nonblocking Node Aware: 2.142571e-04 +Pairwise Multileader, 4 leaders: 1.041049e-03 +Pairwise Locality Aware, 4 leaders: 1.899597e-04 +Pairwise Multileader Locality, 4 leaders: 1.033956e-03 +Nonblocking Multileader, 4 leaders: 8.682046e-04 +Nonblocking Locality Aware, 4 leaders: 1.702520e-04 +Nonblocking Multileader Locality, 4 leaders: 9.589566e-04 +Pairwise Multileader, 10 leaders: 4.317192e-04 +Pairwise Locality Aware, 10 leaders: 1.668790e-04 +Pairwise Multileader Locality, 10 leaders: 4.324186e-04 +Nonblocking Multileader, 10 leaders: 3.146054e-04 +Nonblocking Locality Aware, 10 leaders: 1.745527e-04 +Nonblocking Multileader Locality, 10 leaders: 3.997450e-04 +Pairwise Multileader, 20 leaders: 3.129561e-04 +Pairwise Locality Aware, 20 leaders: 2.255525e-04 +Pairwise Multileader Locality, 20 leaders: 2.519941e-04 +Nonblocking Multileader, 20 leaders: 1.924468e-04 +Nonblocking Locality Aware, 20 leaders: 2.250840e-04 +Nonblocking Multileader Locality, 20 leaders: 1.947585e-04 +Size 64 +PMPI: 4.196225e-04 +Pairwise: 8.268893e-04 +NonBlocking: 1.271499e-02 +Pairwise Hierarchical: 6.949928e-03 +Pairwise Node Aware: 4.723637e-04 +Nonblocking Hierarchical: 6.588557e-03 +Nonblocking Node Aware: 2.923793e-04 +Pairwise Multileader, 4 leaders: 2.409341e-03 +Pairwise Locality Aware, 4 leaders: 3.217723e-04 +Pairwise Multileader Locality, 4 leaders: 2.377182e-03 +Nonblocking Multileader, 4 leaders: 1.955673e-03 +Nonblocking Locality Aware, 4 leaders: 2.721728e-04 +Nonblocking Multileader Locality, 4 leaders: 2.115025e-03 +Pairwise Multileader, 10 leaders: 9.131127e-04 +Pairwise Locality Aware, 10 leaders: 2.904223e-04 +Pairwise Multileader Locality, 10 leaders: 9.855993e-04 +Nonblocking Multileader, 10 leaders: 7.196785e-04 +Nonblocking Locality Aware, 10 leaders: 2.387610e-04 +Nonblocking Multileader Locality, 10 leaders: 8.930600e-04 +Pairwise Multileader, 20 leaders: 5.645960e-04 +Pairwise Locality Aware, 20 leaders: 3.145669e-04 +Pairwise Multileader Locality, 20 leaders: 5.379406e-04 +Nonblocking Multileader, 20 leaders: 3.341625e-04 +Nonblocking Locality Aware, 20 leaders: 2.295456e-04 +Nonblocking Multileader Locality, 20 leaders: 4.768406e-04 +Size 128 +PMPI: 8.171471e-04 +Pairwise: 9.130493e-04 +NonBlocking: 3.743351e-02 +Pairwise Hierarchical: 1.378144e-02 +Pairwise Node Aware: 7.378876e-04 +Nonblocking Hierarchical: 1.335415e-02 +Nonblocking Node Aware: 5.061008e-04 +Pairwise Multileader, 4 leaders: 4.460086e-03 +Pairwise Locality Aware, 4 leaders: 5.343196e-04 +Pairwise Multileader Locality, 4 leaders: 4.628741e-03 +Nonblocking Multileader, 4 leaders: 3.748392e-03 +Nonblocking Locality Aware, 4 leaders: 4.309822e-04 +Nonblocking Multileader Locality, 4 leaders: 4.104609e-03 +Pairwise Multileader, 10 leaders: 1.796918e-03 +Pairwise Locality Aware, 10 leaders: 5.542331e-04 +Pairwise Multileader Locality, 10 leaders: 1.866091e-03 +Nonblocking Multileader, 10 leaders: 1.443645e-03 +Nonblocking Locality Aware, 10 leaders: 4.822583e-04 +Nonblocking Multileader Locality, 10 leaders: 1.703077e-03 +Pairwise Multileader, 20 leaders: 1.048374e-03 +Pairwise Locality Aware, 20 leaders: 5.534862e-04 +Pairwise Multileader Locality, 20 leaders: 1.079835e-03 +Nonblocking Multileader, 20 leaders: 7.069519e-04 +Nonblocking Locality Aware, 20 leaders: 4.421477e-04 +Nonblocking Multileader Locality, 20 leaders: 9.940309e-04 +Size 256 +PMPI: 1.474948e-03 +Pairwise: 1.254017e-03 +NonBlocking: 6.222586e-02 +Pairwise Hierarchical: 2.925636e-02 +Pairwise Node Aware: 1.235779e-03 +Nonblocking Hierarchical: 2.906072e-02 +Nonblocking Node Aware: 8.964901e-04 +Pairwise Multileader, 4 leaders: 7.986583e-03 +Pairwise Locality Aware, 4 leaders: 1.042978e-03 +Pairwise Multileader Locality, 4 leaders: 8.496414e-03 +Nonblocking Multileader, 4 leaders: 6.700646e-03 +Nonblocking Locality Aware, 4 leaders: 8.422929e-04 +Nonblocking Multileader Locality, 4 leaders: 7.723262e-03 +Pairwise Multileader, 10 leaders: 4.465296e-03 +Pairwise Locality Aware, 10 leaders: 1.078634e-03 +Pairwise Multileader Locality, 10 leaders: 3.788177e-03 +Nonblocking Multileader, 10 leaders: 3.263413e-03 +Nonblocking Locality Aware, 10 leaders: 1.053959e-03 +Nonblocking Multileader Locality, 10 leaders: 3.625294e-03 +Pairwise Multileader, 20 leaders: 2.201813e-03 +Pairwise Locality Aware, 20 leaders: 1.124138e-03 +Pairwise Multileader Locality, 20 leaders: 2.228626e-03 +Nonblocking Multileader, 20 leaders: 1.606380e-03 +Nonblocking Locality Aware, 20 leaders: 9.346787e-04 +Nonblocking Multileader Locality, 20 leaders: 1.953970e-03 +Size 512 +PMPI: 1.943460e-03 +Pairwise: 2.029497e-03 +NonBlocking: 2.191578e-03 +Pairwise Hierarchical: 5.809980e-02 +Pairwise Node Aware: 2.188545e-03 +Nonblocking Hierarchical: 5.992529e-02 +Nonblocking Node Aware: 1.744845e-03 +Pairwise Multileader, 4 leaders: 1.743139e-02 +Pairwise Locality Aware, 4 leaders: 2.048429e-03 +Pairwise Multileader Locality, 4 leaders: 1.696177e-02 +Nonblocking Multileader, 4 leaders: 1.408482e-02 +Nonblocking Locality Aware, 4 leaders: 1.786727e-03 +Nonblocking Multileader Locality, 4 leaders: 1.557694e-02 +Pairwise Multileader, 10 leaders: 9.390735e-03 +Pairwise Locality Aware, 10 leaders: 2.010267e-03 +Pairwise Multileader Locality, 10 leaders: 8.255992e-03 +Nonblocking Multileader, 10 leaders: 6.973125e-03 +Nonblocking Locality Aware, 10 leaders: 1.750786e-03 +Nonblocking Multileader Locality, 10 leaders: 7.881588e-03 +Pairwise Multileader, 20 leaders: 4.463843e-03 +Pairwise Locality Aware, 20 leaders: 2.074169e-03 +Pairwise Multileader Locality, 20 leaders: 4.249574e-03 +Nonblocking Multileader, 20 leaders: 3.150088e-03 +Nonblocking Locality Aware, 20 leaders: 1.927653e-03 +Nonblocking Multileader Locality, 20 leaders: 3.776249e-03 +Size 1024 +PMPI: 3.668662e-03 +Pairwise: 3.802929e-03 +NonBlocking: 4.564117e-03 +Pairwise Hierarchical: 1.067989e-01 +Pairwise Node Aware: 4.389351e-03 +Nonblocking Hierarchical: 1.102716e-01 +Nonblocking Node Aware: 3.560419e-03 +Pairwise Multileader, 4 leaders: 3.836695e-02 +Pairwise Locality Aware, 4 leaders: 4.163588e-03 +Pairwise Multileader Locality, 4 leaders: 4.057503e-02 +Nonblocking Multileader, 4 leaders: 3.224245e-02 +Nonblocking Locality Aware, 4 leaders: 3.511591e-03 +Nonblocking Multileader Locality, 4 leaders: 3.763316e-02 +Pairwise Multileader, 10 leaders: 2.025866e-02 +Pairwise Locality Aware, 10 leaders: 3.981647e-03 +Pairwise Multileader Locality, 10 leaders: 1.913275e-02 +Nonblocking Multileader, 10 leaders: 1.537773e-02 +Nonblocking Locality Aware, 10 leaders: 3.380735e-03 +Nonblocking Multileader Locality, 10 leaders: 1.815166e-02 +Pairwise Multileader, 20 leaders: 1.141410e-02 +Pairwise Locality Aware, 20 leaders: 4.187129e-03 +Pairwise Multileader Locality, 20 leaders: 9.046827e-03 +Nonblocking Multileader, 20 leaders: 7.795500e-03 +Nonblocking Locality Aware, 20 leaders: 3.332447e-03 +Nonblocking Multileader Locality, 20 leaders: 8.320235e-03 +Size 2048 +PMPI: 7.998169e-03 +Pairwise: 7.967624e-03 +NonBlocking: 9.807858e-03 +Pairwise Hierarchical: 2.142541e-01 +Pairwise Node Aware: 9.667322e-03 +Nonblocking Hierarchical: 2.155490e-01 +Nonblocking Node Aware: 8.349661e-03 +Pairwise Multileader, 4 leaders: 7.520076e-02 +Pairwise Locality Aware, 4 leaders: 8.002927e-03 +Pairwise Multileader Locality, 4 leaders: 7.996722e-02 +Nonblocking Multileader, 4 leaders: 6.376137e-02 +Nonblocking Locality Aware, 4 leaders: 7.371484e-03 +Nonblocking Multileader Locality, 4 leaders: 7.427376e-02 +Pairwise Multileader, 10 leaders: 4.053948e-02 +Pairwise Locality Aware, 10 leaders: 8.146614e-03 +Pairwise Multileader Locality, 10 leaders: 4.083770e-02 +Nonblocking Multileader, 10 leaders: 3.258868e-02 +Nonblocking Locality Aware, 10 leaders: 6.909862e-03 +Nonblocking Multileader Locality, 10 leaders: 3.885545e-02 +Pairwise Multileader, 20 leaders: 2.428059e-02 +Pairwise Locality Aware, 20 leaders: 9.176143e-03 +Pairwise Multileader Locality, 20 leaders: 2.078348e-02 +Nonblocking Multileader, 20 leaders: 1.677638e-02 +Nonblocking Locality Aware, 20 leaders: 6.804043e-03 +Nonblocking Multileader Locality, 20 leaders: 1.873685e-02 +Size 4096 +PMPI: 1.815443e-02 +Pairwise: 1.561796e-02 +NonBlocking: 2.000210e-02 +Pairwise Hierarchical: 4.416428e-01 +Pairwise Node Aware: 2.090606e-02 +Nonblocking Hierarchical: 4.467328e-01 +Nonblocking Node Aware: 1.770531e-02 +Pairwise Multileader, 4 leaders: 1.386185e-01 +Pairwise Locality Aware, 4 leaders: 1.860378e-02 +Pairwise Multileader Locality, 4 leaders: 1.525002e-01 +Nonblocking Multileader, 4 leaders: 1.231205e-01 +Nonblocking Locality Aware, 4 leaders: 1.624805e-02 +Nonblocking Multileader Locality, 4 leaders: 1.408746e-01 +Pairwise Multileader, 10 leaders: 8.136557e-02 +Pairwise Locality Aware, 10 leaders: 1.911957e-02 +Pairwise Multileader Locality, 10 leaders: 8.621838e-02 +Nonblocking Multileader, 10 leaders: 6.679014e-02 +Nonblocking Locality Aware, 10 leaders: 1.550696e-02 +Nonblocking Multileader Locality, 10 leaders: 8.338412e-02 +Pairwise Multileader, 20 leaders: 4.709092e-02 +Pairwise Locality Aware, 20 leaders: 1.931646e-02 +Pairwise Multileader Locality, 20 leaders: 4.421881e-02 +Nonblocking Multileader, 20 leaders: 3.353616e-02 +Nonblocking Locality Aware, 20 leaders: 1.625040e-02 +Nonblocking Multileader Locality, 20 leaders: 4.062333e-02 +Size 8192 +PMPI: 2.904078e-02 +Pairwise: 2.994288e-02 +NonBlocking: 3.085145e-02 +Pairwise Hierarchical: 9.426488e-01 +Pairwise Node Aware: 3.956366e-02 +Nonblocking Hierarchical: 9.349017e-01 +Nonblocking Node Aware: 3.648615e-02 +Pairwise Multileader, 4 leaders: 2.629360e-01 +Pairwise Locality Aware, 4 leaders: 3.878819e-02 +Pairwise Multileader Locality, 4 leaders: 2.942339e-01 +Nonblocking Multileader, 4 leaders: 2.323300e-01 +Nonblocking Locality Aware, 4 leaders: 3.492060e-02 +Nonblocking Multileader Locality, 4 leaders: 2.727742e-01 +Pairwise Multileader, 10 leaders: 1.555050e-01 +Pairwise Locality Aware, 10 leaders: 3.953403e-02 +Pairwise Multileader Locality, 10 leaders: 1.663358e-01 +Nonblocking Multileader, 10 leaders: 1.300659e-01 +Nonblocking Locality Aware, 10 leaders: 3.491998e-02 +Nonblocking Multileader Locality, 10 leaders: 1.590302e-01 +Pairwise Multileader, 20 leaders: 9.450945e-02 +Pairwise Locality Aware, 20 leaders: 4.035865e-02 +Pairwise Multileader Locality, 20 leaders: 1.008085e-01 +Nonblocking Multileader, 20 leaders: 7.568862e-02 +Nonblocking Locality Aware, 20 leaders: 3.472184e-02 +Nonblocking Multileader Locality, 20 leaders: 9.115551e-02 +Size 16384 +PMPI: 5.428765e-02 +Pairwise: 5.505079e-02 +NonBlocking: 5.353709e-02 +Pairwise Hierarchical: 1.936985e+00 +Pairwise Node Aware: 7.926660e-02 +Nonblocking Hierarchical: 1.941863e+00 +Nonblocking Node Aware: 7.376959e-02 +Pairwise Multileader, 4 leaders: 5.439072e-01 +Pairwise Locality Aware, 4 leaders: 7.885137e-02 +Pairwise Multileader Locality, 4 leaders: 6.302060e-01 +Nonblocking Multileader, 4 leaders: 5.064648e-01 +Nonblocking Locality Aware, 4 leaders: 7.220578e-02 +Nonblocking Multileader Locality, 4 leaders: 5.747963e-01 +Pairwise Multileader, 10 leaders: 3.272681e-01 +Pairwise Locality Aware, 10 leaders: 8.053983e-02 +Pairwise Multileader Locality, 10 leaders: 3.696251e-01 +Nonblocking Multileader, 10 leaders: 2.602230e-01 +Nonblocking Locality Aware, 10 leaders: 7.244876e-02 +Nonblocking Multileader Locality, 10 leaders: 3.144646e-01 +Pairwise Multileader, 20 leaders: 1.750125e-01 +Pairwise Locality Aware, 20 leaders: 8.180384e-02 +Pairwise Multileader Locality, 20 leaders: 1.936508e-01 +Nonblocking Multileader, 20 leaders: 1.491231e-01 +Nonblocking Locality Aware, 20 leaders: 7.491783e-02 +Nonblocking Multileader Locality, 20 leaders: 1.758763e-01 diff --git a/runscripts/dane_results/alltoall_N2.2484422.err b/runscripts/dane_results/alltoall_N2.2484422.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/dane_results/alltoall_N2.2484422.out b/runscripts/dane_results/alltoall_N2.2484422.out new file mode 100644 index 000000000..7da6cae4f --- /dev/null +++ b/runscripts/dane_results/alltoall_N2.2484422.out @@ -0,0 +1,783 @@ +Test 1 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 4.422042e-05 +Pairwise: 4.052250e-04 +NonBlocking: 2.065070e-04 +Pairwise Hierarchical: 1.819207e-04 +Pairwise Node Aware: 1.753897e-04 +Nonblocking Hierarchical: 1.862988e-04 +Nonblocking Node Aware: 1.762154e-04 +Pairwise Multileader, 4 leaders: 6.415850e-05 +Pairwise Locality Aware, 4 leaders: 7.560520e-05 +Pairwise Multileader Locality, 4 leaders: 5.874180e-05 +Nonblocking Multileader, 4 leaders: 4.799557e-05 +Nonblocking Locality Aware, 4 leaders: 5.058251e-05 +Nonblocking Multileader Locality, 4 leaders: 5.590403e-05 +Pairwise Multileader, 10 leaders: 8.540533e-05 +Pairwise Locality Aware, 10 leaders: 9.242102e-05 +Pairwise Multileader Locality, 10 leaders: 4.068845e-05 +Nonblocking Multileader, 10 leaders: 3.768838e-05 +Nonblocking Locality Aware, 10 leaders: 3.826870e-05 +Nonblocking Multileader Locality, 10 leaders: 3.599439e-05 +Pairwise Multileader, 20 leaders: 1.818706e-04 +Pairwise Locality Aware, 20 leaders: 1.945814e-04 +Pairwise Multileader Locality, 20 leaders: 5.596022e-05 +Nonblocking Multileader, 20 leaders: 5.496364e-05 +Nonblocking Locality Aware, 20 leaders: 4.960111e-05 +Nonblocking Multileader Locality, 20 leaders: 4.311713e-05 +Size 2 +PMPI: 4.852093e-05 +Pairwise: 3.769069e-04 +NonBlocking: 1.877024e-04 +Pairwise Hierarchical: 2.256933e-04 +Pairwise Node Aware: 1.805799e-04 +Nonblocking Hierarchical: 2.205401e-04 +Nonblocking Node Aware: 1.743415e-04 +Pairwise Multileader, 4 leaders: 8.485067e-05 +Pairwise Locality Aware, 4 leaders: 8.083464e-05 +Pairwise Multileader Locality, 4 leaders: 8.389554e-05 +Nonblocking Multileader, 4 leaders: 6.367974e-05 +Nonblocking Locality Aware, 4 leaders: 5.192187e-05 +Nonblocking Multileader Locality, 4 leaders: 7.847907e-05 +Pairwise Multileader, 10 leaders: 9.259252e-05 +Pairwise Locality Aware, 10 leaders: 9.688461e-05 +Pairwise Multileader Locality, 10 leaders: 5.075856e-05 +Nonblocking Multileader, 10 leaders: 4.366311e-05 +Nonblocking Locality Aware, 10 leaders: 4.735551e-05 +Nonblocking Multileader Locality, 10 leaders: 4.388079e-05 +Pairwise Multileader, 20 leaders: 1.748253e-04 +Pairwise Locality Aware, 20 leaders: 1.828578e-04 +Pairwise Multileader Locality, 20 leaders: 6.469524e-05 +Nonblocking Multileader, 20 leaders: 6.078306e-05 +Nonblocking Locality Aware, 20 leaders: 5.693723e-05 +Nonblocking Multileader Locality, 20 leaders: 4.890891e-05 +Size 4 +PMPI: 5.565864e-05 +Pairwise: 7.717076e-04 +NonBlocking: 4.120605e-03 +Pairwise Hierarchical: 4.283315e-04 +Pairwise Node Aware: 2.285598e-04 +Nonblocking Hierarchical: 4.313684e-04 +Nonblocking Node Aware: 1.760376e-04 +Pairwise Multileader, 4 leaders: 1.200999e-04 +Pairwise Locality Aware, 4 leaders: 8.068467e-05 +Pairwise Multileader Locality, 4 leaders: 1.102576e-04 +Nonblocking Multileader, 4 leaders: 9.166136e-05 +Nonblocking Locality Aware, 4 leaders: 5.229378e-05 +Nonblocking Multileader Locality, 4 leaders: 1.054137e-04 +Pairwise Multileader, 10 leaders: 1.046349e-04 +Pairwise Locality Aware, 10 leaders: 1.087998e-04 +Pairwise Multileader Locality, 10 leaders: 6.663816e-05 +Nonblocking Multileader, 10 leaders: 5.057474e-05 +Nonblocking Locality Aware, 10 leaders: 5.793873e-05 +Nonblocking Multileader Locality, 10 leaders: 6.324209e-05 +Pairwise Multileader, 20 leaders: 1.682095e-04 +Pairwise Locality Aware, 20 leaders: 1.692230e-04 +Pairwise Multileader Locality, 20 leaders: 7.807312e-05 +Nonblocking Multileader, 20 leaders: 6.400311e-05 +Nonblocking Locality Aware, 20 leaders: 5.938695e-05 +Nonblocking Multileader Locality, 20 leaders: 5.913142e-05 +Size 8 +PMPI: 8.173719e-05 +Pairwise: 7.842124e-04 +NonBlocking: 6.980190e-03 +Pairwise Hierarchical: 7.550765e-04 +Pairwise Node Aware: 2.562680e-04 +Nonblocking Hierarchical: 7.307380e-04 +Nonblocking Node Aware: 1.751728e-04 +Pairwise Multileader, 4 leaders: 2.286852e-04 +Pairwise Locality Aware, 4 leaders: 1.002883e-04 +Pairwise Multileader Locality, 4 leaders: 2.987549e-04 +Nonblocking Multileader, 4 leaders: 1.825425e-04 +Nonblocking Locality Aware, 4 leaders: 8.303168e-05 +Nonblocking Multileader Locality, 4 leaders: 2.577932e-04 +Pairwise Multileader, 10 leaders: 1.225891e-04 +Pairwise Locality Aware, 10 leaders: 1.130598e-04 +Pairwise Multileader Locality, 10 leaders: 1.061939e-04 +Nonblocking Multileader, 10 leaders: 7.159625e-05 +Nonblocking Locality Aware, 10 leaders: 6.048990e-05 +Nonblocking Multileader Locality, 10 leaders: 9.184729e-05 +Pairwise Multileader, 20 leaders: 1.859861e-04 +Pairwise Locality Aware, 20 leaders: 1.720250e-04 +Pairwise Multileader Locality, 20 leaders: 1.112546e-04 +Nonblocking Multileader, 20 leaders: 8.088586e-05 +Nonblocking Locality Aware, 20 leaders: 7.529514e-05 +Nonblocking Multileader Locality, 20 leaders: 8.306826e-05 +Size 16 +PMPI: 1.479999e-04 +Pairwise: 7.901591e-04 +NonBlocking: 8.648915e-03 +Pairwise Hierarchical: 1.549781e-03 +Pairwise Node Aware: 3.279460e-04 +Nonblocking Hierarchical: 1.501572e-03 +Nonblocking Node Aware: 1.778224e-04 +Pairwise Multileader, 4 leaders: 4.626753e-04 +Pairwise Locality Aware, 4 leaders: 1.256830e-04 +Pairwise Multileader Locality, 4 leaders: 5.138421e-04 +Nonblocking Multileader, 4 leaders: 3.643684e-04 +Nonblocking Locality Aware, 4 leaders: 9.336652e-05 +Nonblocking Multileader Locality, 4 leaders: 4.465284e-04 +Pairwise Multileader, 10 leaders: 2.364631e-04 +Pairwise Locality Aware, 10 leaders: 1.262387e-04 +Pairwise Multileader Locality, 10 leaders: 2.017237e-04 +Nonblocking Multileader, 10 leaders: 1.427023e-04 +Nonblocking Locality Aware, 10 leaders: 9.282826e-05 +Nonblocking Multileader Locality, 10 leaders: 1.769513e-04 +Pairwise Multileader, 20 leaders: 2.156255e-04 +Pairwise Locality Aware, 20 leaders: 1.765034e-04 +Pairwise Multileader Locality, 20 leaders: 1.616600e-04 +Nonblocking Multileader, 20 leaders: 1.115355e-04 +Nonblocking Locality Aware, 20 leaders: 1.065784e-04 +Nonblocking Multileader Locality, 20 leaders: 1.245640e-04 +Size 32 +PMPI: 2.695655e-04 +Pairwise: 7.807378e-04 +NonBlocking: 1.217702e-02 +Pairwise Hierarchical: 3.050025e-03 +Pairwise Node Aware: 4.292421e-04 +Nonblocking Hierarchical: 2.953244e-03 +Nonblocking Node Aware: 2.377487e-04 +Pairwise Multileader, 4 leaders: 9.974791e-04 +Pairwise Locality Aware, 4 leaders: 1.841523e-04 +Pairwise Multileader Locality, 4 leaders: 1.019171e-03 +Nonblocking Multileader, 4 leaders: 8.699513e-04 +Nonblocking Locality Aware, 4 leaders: 1.541009e-04 +Nonblocking Multileader Locality, 4 leaders: 9.467273e-04 +Pairwise Multileader, 10 leaders: 4.463381e-04 +Pairwise Locality Aware, 10 leaders: 1.854653e-04 +Pairwise Multileader Locality, 10 leaders: 4.408072e-04 +Nonblocking Multileader, 10 leaders: 3.076180e-04 +Nonblocking Locality Aware, 10 leaders: 1.880932e-04 +Nonblocking Multileader Locality, 10 leaders: 4.003090e-04 +Pairwise Multileader, 20 leaders: 3.087366e-04 +Pairwise Locality Aware, 20 leaders: 2.403177e-04 +Pairwise Multileader Locality, 20 leaders: 2.517077e-04 +Nonblocking Multileader, 20 leaders: 1.668081e-04 +Nonblocking Locality Aware, 20 leaders: 1.931769e-04 +Nonblocking Multileader Locality, 20 leaders: 1.931358e-04 +Size 64 +PMPI: 4.370418e-04 +Pairwise: 8.709871e-04 +NonBlocking: 3.078258e-02 +Pairwise Hierarchical: 6.956694e-03 +Pairwise Node Aware: 5.009979e-04 +Nonblocking Hierarchical: 6.671177e-03 +Nonblocking Node Aware: 2.903052e-04 +Pairwise Multileader, 4 leaders: 2.335847e-03 +Pairwise Locality Aware, 4 leaders: 3.250431e-04 +Pairwise Multileader Locality, 4 leaders: 2.365550e-03 +Nonblocking Multileader, 4 leaders: 1.941815e-03 +Nonblocking Locality Aware, 4 leaders: 2.542656e-04 +Nonblocking Multileader Locality, 4 leaders: 2.129288e-03 +Pairwise Multileader, 10 leaders: 9.184554e-04 +Pairwise Locality Aware, 10 leaders: 3.313303e-04 +Pairwise Multileader Locality, 10 leaders: 1.009139e-03 +Nonblocking Multileader, 10 leaders: 7.069871e-04 +Nonblocking Locality Aware, 10 leaders: 2.923009e-04 +Nonblocking Multileader Locality, 10 leaders: 9.141229e-04 +Pairwise Multileader, 20 leaders: 5.645870e-04 +Pairwise Locality Aware, 20 leaders: 3.173788e-04 +Pairwise Multileader Locality, 20 leaders: 5.246710e-04 +Nonblocking Multileader, 20 leaders: 3.400582e-04 +Nonblocking Locality Aware, 20 leaders: 2.367056e-04 +Nonblocking Multileader Locality, 20 leaders: 4.575183e-04 +Size 128 +PMPI: 9.130196e-04 +Pairwise: 9.454718e-04 +NonBlocking: 4.678796e-02 +Pairwise Hierarchical: 1.395875e-02 +Pairwise Node Aware: 8.151246e-04 +Nonblocking Hierarchical: 1.332442e-02 +Nonblocking Node Aware: 5.192201e-04 +Pairwise Multileader, 4 leaders: 4.416626e-03 +Pairwise Locality Aware, 4 leaders: 6.373338e-04 +Pairwise Multileader Locality, 4 leaders: 4.631077e-03 +Nonblocking Multileader, 4 leaders: 3.725975e-03 +Nonblocking Locality Aware, 4 leaders: 5.486060e-04 +Nonblocking Multileader Locality, 4 leaders: 4.161481e-03 +Pairwise Multileader, 10 leaders: 1.854088e-03 +Pairwise Locality Aware, 10 leaders: 6.447229e-04 +Pairwise Multileader Locality, 10 leaders: 1.895799e-03 +Nonblocking Multileader, 10 leaders: 1.468855e-03 +Nonblocking Locality Aware, 10 leaders: 5.669776e-04 +Nonblocking Multileader Locality, 10 leaders: 1.752301e-03 +Pairwise Multileader, 20 leaders: 1.124514e-03 +Pairwise Locality Aware, 20 leaders: 6.235563e-04 +Pairwise Multileader Locality, 20 leaders: 1.065893e-03 +Nonblocking Multileader, 20 leaders: 7.969613e-04 +Nonblocking Locality Aware, 20 leaders: 5.810121e-04 +Nonblocking Multileader Locality, 20 leaders: 9.853344e-04 +Size 256 +PMPI: 1.764119e-03 +Pairwise: 1.404564e-03 +NonBlocking: 7.453036e-02 +Pairwise Hierarchical: 2.834957e-02 +Pairwise Node Aware: 1.308075e-03 +Nonblocking Hierarchical: 2.848892e-02 +Nonblocking Node Aware: 1.051606e-03 +Pairwise Multileader, 4 leaders: 7.904430e-03 +Pairwise Locality Aware, 4 leaders: 1.093370e-03 +Pairwise Multileader Locality, 4 leaders: 9.054201e-03 +Nonblocking Multileader, 4 leaders: 6.626470e-03 +Nonblocking Locality Aware, 4 leaders: 9.251804e-04 +Nonblocking Multileader Locality, 4 leaders: 7.623098e-03 +Pairwise Multileader, 10 leaders: 4.500982e-03 +Pairwise Locality Aware, 10 leaders: 1.276111e-03 +Pairwise Multileader Locality, 10 leaders: 3.780664e-03 +Nonblocking Multileader, 10 leaders: 3.245595e-03 +Nonblocking Locality Aware, 10 leaders: 1.208774e-03 +Nonblocking Multileader Locality, 10 leaders: 3.603061e-03 +Pairwise Multileader, 20 leaders: 2.241159e-03 +Pairwise Locality Aware, 20 leaders: 1.163582e-03 +Pairwise Multileader Locality, 20 leaders: 2.266700e-03 +Nonblocking Multileader, 20 leaders: 1.584185e-03 +Nonblocking Locality Aware, 20 leaders: 1.027047e-03 +Nonblocking Multileader Locality, 20 leaders: 2.050522e-03 +Size 512 +PMPI: 2.078387e-03 +Pairwise: 2.140870e-03 +NonBlocking: 2.264463e-03 +Pairwise Hierarchical: 5.695609e-02 +Pairwise Node Aware: 2.277820e-03 +Nonblocking Hierarchical: 5.975258e-02 +Nonblocking Node Aware: 2.170462e-03 +Pairwise Multileader, 4 leaders: 1.731446e-02 +Pairwise Locality Aware, 4 leaders: 2.105618e-03 +Pairwise Multileader Locality, 4 leaders: 1.719275e-02 +Nonblocking Multileader, 4 leaders: 1.398562e-02 +Nonblocking Locality Aware, 4 leaders: 1.796610e-03 +Nonblocking Multileader Locality, 4 leaders: 1.556267e-02 +Pairwise Multileader, 10 leaders: 9.448231e-03 +Pairwise Locality Aware, 10 leaders: 2.120901e-03 +Pairwise Multileader Locality, 10 leaders: 8.363421e-03 +Nonblocking Multileader, 10 leaders: 7.042903e-03 +Nonblocking Locality Aware, 10 leaders: 2.147739e-03 +Nonblocking Multileader Locality, 10 leaders: 7.689008e-03 +Pairwise Multileader, 20 leaders: 4.536593e-03 +Pairwise Locality Aware, 20 leaders: 2.519723e-03 +Pairwise Multileader Locality, 20 leaders: 4.284376e-03 +Nonblocking Multileader, 20 leaders: 3.113265e-03 +Nonblocking Locality Aware, 20 leaders: 2.150461e-03 +Nonblocking Multileader Locality, 20 leaders: 3.901030e-03 +Test 2 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 4.736958e-05 +Pairwise: 4.386686e-04 +NonBlocking: 1.861052e-04 +Pairwise Hierarchical: 1.824014e-04 +Pairwise Node Aware: 1.849346e-04 +Nonblocking Hierarchical: 1.919302e-04 +Nonblocking Node Aware: 1.760182e-04 +Pairwise Multileader, 4 leaders: 6.507022e-05 +Pairwise Locality Aware, 4 leaders: 8.154148e-05 +Pairwise Multileader Locality, 4 leaders: 6.103731e-05 +Nonblocking Multileader, 4 leaders: 4.761933e-05 +Nonblocking Locality Aware, 4 leaders: 5.095317e-05 +Nonblocking Multileader Locality, 4 leaders: 5.653454e-05 +Pairwise Multileader, 10 leaders: 9.510807e-05 +Pairwise Locality Aware, 10 leaders: 1.027912e-04 +Pairwise Multileader Locality, 10 leaders: 4.015108e-05 +Nonblocking Multileader, 10 leaders: 3.670189e-05 +Nonblocking Locality Aware, 10 leaders: 4.424476e-05 +Nonblocking Multileader Locality, 10 leaders: 3.854220e-05 +Pairwise Multileader, 20 leaders: 1.834132e-04 +Pairwise Locality Aware, 20 leaders: 1.866218e-04 +Pairwise Multileader Locality, 20 leaders: 5.893780e-05 +Nonblocking Multileader, 20 leaders: 6.219385e-05 +Nonblocking Locality Aware, 20 leaders: 7.269171e-05 +Nonblocking Multileader Locality, 20 leaders: 4.508503e-05 +Size 2 +PMPI: 5.171232e-05 +Pairwise: 4.570502e-04 +NonBlocking: 1.991150e-04 +Pairwise Hierarchical: 2.938013e-04 +Pairwise Node Aware: 1.862571e-04 +Nonblocking Hierarchical: 2.919432e-04 +Nonblocking Node Aware: 1.798664e-04 +Pairwise Multileader, 4 leaders: 8.457949e-05 +Pairwise Locality Aware, 4 leaders: 8.549490e-05 +Pairwise Multileader Locality, 4 leaders: 8.424060e-05 +Nonblocking Multileader, 4 leaders: 6.861002e-05 +Nonblocking Locality Aware, 4 leaders: 5.766913e-05 +Nonblocking Multileader Locality, 4 leaders: 7.919811e-05 +Pairwise Multileader, 10 leaders: 1.046679e-04 +Pairwise Locality Aware, 10 leaders: 1.229109e-04 +Pairwise Multileader Locality, 10 leaders: 5.581383e-05 +Nonblocking Multileader, 10 leaders: 4.467257e-05 +Nonblocking Locality Aware, 10 leaders: 5.002551e-05 +Nonblocking Multileader Locality, 10 leaders: 4.814559e-05 +Pairwise Multileader, 20 leaders: 1.848855e-04 +Pairwise Locality Aware, 20 leaders: 1.617429e-04 +Pairwise Multileader Locality, 20 leaders: 6.357517e-05 +Nonblocking Multileader, 20 leaders: 6.402357e-05 +Nonblocking Locality Aware, 20 leaders: 5.693400e-05 +Nonblocking Multileader Locality, 20 leaders: 4.686759e-05 +Size 4 +PMPI: 5.936345e-05 +Pairwise: 8.887503e-04 +NonBlocking: 4.840297e-03 +Pairwise Hierarchical: 2.907984e-04 +Pairwise Node Aware: 2.394205e-04 +Nonblocking Hierarchical: 2.926833e-04 +Nonblocking Node Aware: 1.810418e-04 +Pairwise Multileader, 4 leaders: 1.165429e-04 +Pairwise Locality Aware, 4 leaders: 8.437534e-05 +Pairwise Multileader Locality, 4 leaders: 1.150911e-04 +Nonblocking Multileader, 4 leaders: 9.105866e-05 +Nonblocking Locality Aware, 4 leaders: 5.041489e-05 +Nonblocking Multileader Locality, 4 leaders: 1.061544e-04 +Pairwise Multileader, 10 leaders: 1.108629e-04 +Pairwise Locality Aware, 10 leaders: 1.127505e-04 +Pairwise Multileader Locality, 10 leaders: 6.892194e-05 +Nonblocking Multileader, 10 leaders: 5.122789e-05 +Nonblocking Locality Aware, 10 leaders: 5.121691e-05 +Nonblocking Multileader Locality, 10 leaders: 6.211277e-05 +Pairwise Multileader, 20 leaders: 1.787912e-04 +Pairwise Locality Aware, 20 leaders: 1.666058e-04 +Pairwise Multileader Locality, 20 leaders: 7.678613e-05 +Nonblocking Multileader, 20 leaders: 7.038968e-05 +Nonblocking Locality Aware, 20 leaders: 6.393414e-05 +Nonblocking Multileader Locality, 20 leaders: 5.759568e-05 +Size 8 +PMPI: 8.293680e-05 +Pairwise: 8.814698e-04 +NonBlocking: 5.246090e-03 +Pairwise Hierarchical: 8.002808e-04 +Pairwise Node Aware: 2.510250e-04 +Nonblocking Hierarchical: 7.549839e-04 +Nonblocking Node Aware: 1.809058e-04 +Pairwise Multileader, 4 leaders: 2.365864e-04 +Pairwise Locality Aware, 4 leaders: 8.855438e-05 +Pairwise Multileader Locality, 4 leaders: 3.080538e-04 +Nonblocking Multileader, 4 leaders: 1.824321e-04 +Nonblocking Locality Aware, 4 leaders: 6.333305e-05 +Nonblocking Multileader Locality, 4 leaders: 2.571459e-04 +Pairwise Multileader, 10 leaders: 1.266248e-04 +Pairwise Locality Aware, 10 leaders: 1.148483e-04 +Pairwise Multileader Locality, 10 leaders: 1.023118e-04 +Nonblocking Multileader, 10 leaders: 7.173276e-05 +Nonblocking Locality Aware, 10 leaders: 6.464305e-05 +Nonblocking Multileader Locality, 10 leaders: 8.682820e-05 +Pairwise Multileader, 20 leaders: 2.041797e-04 +Pairwise Locality Aware, 20 leaders: 1.807685e-04 +Pairwise Multileader Locality, 20 leaders: 1.087196e-04 +Nonblocking Multileader, 20 leaders: 8.273708e-05 +Nonblocking Locality Aware, 20 leaders: 1.491922e-04 +Nonblocking Multileader Locality, 20 leaders: 8.069865e-05 +Size 16 +PMPI: 1.502253e-04 +Pairwise: 8.287723e-04 +NonBlocking: 5.683411e-03 +Pairwise Hierarchical: 1.526216e-03 +Pairwise Node Aware: 3.111532e-04 +Nonblocking Hierarchical: 1.475366e-03 +Nonblocking Node Aware: 1.893092e-04 +Pairwise Multileader, 4 leaders: 4.487440e-04 +Pairwise Locality Aware, 4 leaders: 1.178709e-04 +Pairwise Multileader Locality, 4 leaders: 5.295168e-04 +Nonblocking Multileader, 4 leaders: 3.590474e-04 +Nonblocking Locality Aware, 4 leaders: 8.995956e-05 +Nonblocking Multileader Locality, 4 leaders: 4.578834e-04 +Pairwise Multileader, 10 leaders: 2.456139e-04 +Pairwise Locality Aware, 10 leaders: 1.257609e-04 +Pairwise Multileader Locality, 10 leaders: 1.954930e-04 +Nonblocking Multileader, 10 leaders: 1.533800e-04 +Nonblocking Locality Aware, 10 leaders: 1.079263e-04 +Nonblocking Multileader Locality, 10 leaders: 1.723574e-04 +Pairwise Multileader, 20 leaders: 2.095484e-04 +Pairwise Locality Aware, 20 leaders: 1.897166e-04 +Pairwise Multileader Locality, 20 leaders: 1.688598e-04 +Nonblocking Multileader, 20 leaders: 1.073358e-04 +Nonblocking Locality Aware, 20 leaders: 1.169956e-04 +Nonblocking Multileader Locality, 20 leaders: 1.359433e-04 +Size 32 +PMPI: 2.496425e-04 +Pairwise: 7.864648e-04 +NonBlocking: 1.523475e-02 +Pairwise Hierarchical: 3.092105e-03 +Pairwise Node Aware: 3.965217e-04 +Nonblocking Hierarchical: 2.977228e-03 +Nonblocking Node Aware: 2.138861e-04 +Pairwise Multileader, 4 leaders: 1.098844e-03 +Pairwise Locality Aware, 4 leaders: 2.193537e-04 +Pairwise Multileader Locality, 4 leaders: 1.050375e-03 +Nonblocking Multileader, 4 leaders: 8.638062e-04 +Nonblocking Locality Aware, 4 leaders: 1.911393e-04 +Nonblocking Multileader Locality, 4 leaders: 9.521585e-04 +Pairwise Multileader, 10 leaders: 4.314029e-04 +Pairwise Locality Aware, 10 leaders: 1.991389e-04 +Pairwise Multileader Locality, 10 leaders: 4.420530e-04 +Nonblocking Multileader, 10 leaders: 3.166953e-04 +Nonblocking Locality Aware, 10 leaders: 1.712461e-04 +Nonblocking Multileader Locality, 10 leaders: 4.042835e-04 +Pairwise Multileader, 20 leaders: 3.338966e-04 +Pairwise Locality Aware, 20 leaders: 2.110385e-04 +Pairwise Multileader Locality, 20 leaders: 2.669872e-04 +Nonblocking Multileader, 20 leaders: 1.834640e-04 +Nonblocking Locality Aware, 20 leaders: 1.706119e-04 +Nonblocking Multileader Locality, 20 leaders: 2.008549e-04 +Size 64 +PMPI: 5.070875e-04 +Pairwise: 8.991937e-04 +NonBlocking: 6.183474e-02 +Pairwise Hierarchical: 6.964169e-03 +Pairwise Node Aware: 5.223188e-04 +Nonblocking Hierarchical: 6.658077e-03 +Nonblocking Node Aware: 3.124621e-04 +Pairwise Multileader, 4 leaders: 2.393249e-03 +Pairwise Locality Aware, 4 leaders: 3.496928e-04 +Pairwise Multileader Locality, 4 leaders: 2.367426e-03 +Nonblocking Multileader, 4 leaders: 1.929242e-03 +Nonblocking Locality Aware, 4 leaders: 2.725834e-04 +Nonblocking Multileader Locality, 4 leaders: 2.101204e-03 +Pairwise Multileader, 10 leaders: 8.886200e-04 +Pairwise Locality Aware, 10 leaders: 3.371151e-04 +Pairwise Multileader Locality, 10 leaders: 9.447480e-04 +Nonblocking Multileader, 10 leaders: 6.940737e-04 +Nonblocking Locality Aware, 10 leaders: 2.868223e-04 +Nonblocking Multileader Locality, 10 leaders: 8.654012e-04 +Pairwise Multileader, 20 leaders: 5.999844e-04 +Pairwise Locality Aware, 20 leaders: 3.524807e-04 +Pairwise Multileader Locality, 20 leaders: 5.664769e-04 +Nonblocking Multileader, 20 leaders: 3.472862e-04 +Nonblocking Locality Aware, 20 leaders: 2.841314e-04 +Nonblocking Multileader Locality, 20 leaders: 4.743671e-04 +Size 128 +PMPI: 9.794460e-04 +Pairwise: 9.204844e-04 +NonBlocking: 3.203832e-02 +Pairwise Hierarchical: 1.363762e-02 +Pairwise Node Aware: 7.733290e-04 +Nonblocking Hierarchical: 1.341201e-02 +Nonblocking Node Aware: 5.074048e-04 +Pairwise Multileader, 4 leaders: 4.204081e-03 +Pairwise Locality Aware, 4 leaders: 6.685932e-04 +Pairwise Multileader Locality, 4 leaders: 4.589259e-03 +Nonblocking Multileader, 4 leaders: 3.637778e-03 +Nonblocking Locality Aware, 4 leaders: 6.120859e-04 +Nonblocking Multileader Locality, 4 leaders: 4.083280e-03 +Pairwise Multileader, 10 leaders: 1.800919e-03 +Pairwise Locality Aware, 10 leaders: 6.487038e-04 +Pairwise Multileader Locality, 10 leaders: 1.876690e-03 +Nonblocking Multileader, 10 leaders: 1.457317e-03 +Nonblocking Locality Aware, 10 leaders: 5.350153e-04 +Nonblocking Multileader Locality, 10 leaders: 1.739238e-03 +Pairwise Multileader, 20 leaders: 1.107168e-03 +Pairwise Locality Aware, 20 leaders: 5.935146e-04 +Pairwise Multileader Locality, 20 leaders: 1.108902e-03 +Nonblocking Multileader, 20 leaders: 7.294178e-04 +Nonblocking Locality Aware, 20 leaders: 4.533790e-04 +Nonblocking Multileader Locality, 20 leaders: 1.021411e-03 +Size 256 +PMPI: 1.540966e-03 +Pairwise: 1.301049e-03 +NonBlocking: 7.454845e-02 +Pairwise Hierarchical: 2.869635e-02 +Pairwise Node Aware: 1.264986e-03 +Nonblocking Hierarchical: 2.908024e-02 +Nonblocking Node Aware: 1.030340e-03 +Pairwise Multileader, 4 leaders: 8.023315e-03 +Pairwise Locality Aware, 4 leaders: 1.082308e-03 +Pairwise Multileader Locality, 4 leaders: 8.485005e-03 +Nonblocking Multileader, 4 leaders: 6.673557e-03 +Nonblocking Locality Aware, 4 leaders: 8.519453e-04 +Nonblocking Multileader Locality, 4 leaders: 7.917372e-03 +Pairwise Multileader, 10 leaders: 4.427865e-03 +Pairwise Locality Aware, 10 leaders: 1.105822e-03 +Pairwise Multileader Locality, 10 leaders: 3.769635e-03 +Nonblocking Multileader, 10 leaders: 3.284312e-03 +Nonblocking Locality Aware, 10 leaders: 1.058303e-03 +Nonblocking Multileader Locality, 10 leaders: 3.580954e-03 +Pairwise Multileader, 20 leaders: 2.375075e-03 +Pairwise Locality Aware, 20 leaders: 1.094945e-03 +Pairwise Multileader Locality, 20 leaders: 2.207135e-03 +Nonblocking Multileader, 20 leaders: 1.609767e-03 +Nonblocking Locality Aware, 20 leaders: 1.027313e-03 +Nonblocking Multileader Locality, 20 leaders: 1.975959e-03 +Size 512 +PMPI: 2.141440e-03 +Pairwise: 2.501657e-03 +NonBlocking: 2.231671e-03 +Pairwise Hierarchical: 5.708161e-02 +Pairwise Node Aware: 2.406714e-03 +Nonblocking Hierarchical: 5.978678e-02 +Nonblocking Node Aware: 1.769783e-03 +Pairwise Multileader, 4 leaders: 1.739828e-02 +Pairwise Locality Aware, 4 leaders: 2.321574e-03 +Pairwise Multileader Locality, 4 leaders: 1.814750e-02 +Nonblocking Multileader, 4 leaders: 1.339056e-02 +Nonblocking Locality Aware, 4 leaders: 2.034014e-03 +Nonblocking Multileader Locality, 4 leaders: 1.662267e-02 +Pairwise Multileader, 10 leaders: 9.261514e-03 +Pairwise Locality Aware, 10 leaders: 2.312307e-03 +Pairwise Multileader Locality, 10 leaders: 8.213514e-03 +Nonblocking Multileader, 10 leaders: 6.952974e-03 +Nonblocking Locality Aware, 10 leaders: 1.787597e-03 +Nonblocking Multileader Locality, 10 leaders: 7.849429e-03 +Pairwise Multileader, 20 leaders: 4.525534e-03 +Pairwise Locality Aware, 20 leaders: 2.165395e-03 +Pairwise Multileader Locality, 20 leaders: 4.361833e-03 +Nonblocking Multileader, 20 leaders: 3.121095e-03 +Nonblocking Locality Aware, 20 leaders: 2.167107e-03 +Nonblocking Multileader Locality, 20 leaders: 3.878881e-03 +Test 3 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 4.326671e-05 +Pairwise: 4.214396e-04 +NonBlocking: 1.969393e-04 +Pairwise Hierarchical: 1.874363e-04 +Pairwise Node Aware: 1.926424e-04 +Nonblocking Hierarchical: 1.870714e-04 +Nonblocking Node Aware: 1.757011e-04 +Pairwise Multileader, 4 leaders: 6.450240e-05 +Pairwise Locality Aware, 4 leaders: 7.454010e-05 +Pairwise Multileader Locality, 4 leaders: 5.950897e-05 +Nonblocking Multileader, 4 leaders: 4.903120e-05 +Nonblocking Locality Aware, 4 leaders: 5.141627e-05 +Nonblocking Multileader Locality, 4 leaders: 5.495906e-05 +Pairwise Multileader, 10 leaders: 8.578746e-05 +Pairwise Locality Aware, 10 leaders: 9.717504e-05 +Pairwise Multileader Locality, 10 leaders: 4.151690e-05 +Nonblocking Multileader, 10 leaders: 3.821994e-05 +Nonblocking Locality Aware, 10 leaders: 3.991546e-05 +Nonblocking Multileader Locality, 10 leaders: 3.636466e-05 +Pairwise Multileader, 20 leaders: 1.728210e-04 +Pairwise Locality Aware, 20 leaders: 1.782259e-04 +Pairwise Multileader Locality, 20 leaders: 5.633505e-05 +Nonblocking Multileader, 20 leaders: 5.674313e-05 +Nonblocking Locality Aware, 20 leaders: 5.424999e-05 +Nonblocking Multileader Locality, 20 leaders: 4.339480e-05 +Size 2 +PMPI: 4.958709e-05 +Pairwise: 4.181796e-04 +NonBlocking: 1.863843e-04 +Pairwise Hierarchical: 3.035541e-04 +Pairwise Node Aware: 1.833613e-04 +Nonblocking Hierarchical: 2.967089e-04 +Nonblocking Node Aware: 1.768689e-04 +Pairwise Multileader, 4 leaders: 9.017616e-05 +Pairwise Locality Aware, 4 leaders: 8.605338e-05 +Pairwise Multileader Locality, 4 leaders: 8.162772e-05 +Nonblocking Multileader, 4 leaders: 6.449617e-05 +Nonblocking Locality Aware, 4 leaders: 5.500756e-05 +Nonblocking Multileader Locality, 4 leaders: 7.884225e-05 +Pairwise Multileader, 10 leaders: 1.045097e-04 +Pairwise Locality Aware, 10 leaders: 1.176839e-04 +Pairwise Multileader Locality, 10 leaders: 5.503270e-05 +Nonblocking Multileader, 10 leaders: 4.604710e-05 +Nonblocking Locality Aware, 10 leaders: 6.033550e-05 +Nonblocking Multileader Locality, 10 leaders: 4.777834e-05 +Pairwise Multileader, 20 leaders: 2.010216e-04 +Pairwise Locality Aware, 20 leaders: 1.926911e-04 +Pairwise Multileader Locality, 20 leaders: 7.013667e-05 +Nonblocking Multileader, 20 leaders: 6.486706e-05 +Nonblocking Locality Aware, 20 leaders: 7.644139e-05 +Nonblocking Multileader Locality, 20 leaders: 5.137652e-05 +Size 4 +PMPI: 6.499196e-05 +Pairwise: 9.563599e-04 +NonBlocking: 6.494871e-03 +Pairwise Hierarchical: 4.516831e-04 +Pairwise Node Aware: 2.332597e-04 +Nonblocking Hierarchical: 4.363827e-04 +Nonblocking Node Aware: 1.788490e-04 +Pairwise Multileader, 4 leaders: 1.254244e-04 +Pairwise Locality Aware, 4 leaders: 9.052060e-05 +Pairwise Multileader Locality, 4 leaders: 1.136146e-04 +Nonblocking Multileader, 4 leaders: 8.964795e-05 +Nonblocking Locality Aware, 4 leaders: 5.664541e-05 +Nonblocking Multileader Locality, 4 leaders: 1.092010e-04 +Pairwise Multileader, 10 leaders: 1.046068e-04 +Pairwise Locality Aware, 10 leaders: 1.020520e-04 +Pairwise Multileader Locality, 10 leaders: 7.129740e-05 +Nonblocking Multileader, 10 leaders: 5.392484e-05 +Nonblocking Locality Aware, 10 leaders: 4.991153e-05 +Nonblocking Multileader Locality, 10 leaders: 6.179628e-05 +Pairwise Multileader, 20 leaders: 1.904517e-04 +Pairwise Locality Aware, 20 leaders: 1.679965e-04 +Pairwise Multileader Locality, 20 leaders: 7.549534e-05 +Nonblocking Multileader, 20 leaders: 7.043106e-05 +Nonblocking Locality Aware, 20 leaders: 7.171941e-05 +Nonblocking Multileader Locality, 20 leaders: 5.820349e-05 +Size 8 +PMPI: 7.922766e-05 +Pairwise: 8.575944e-04 +NonBlocking: 6.941839e-03 +Pairwise Hierarchical: 8.578880e-04 +Pairwise Node Aware: 2.546318e-04 +Nonblocking Hierarchical: 7.814513e-04 +Nonblocking Node Aware: 1.805952e-04 +Pairwise Multileader, 4 leaders: 2.259011e-04 +Pairwise Locality Aware, 4 leaders: 9.216191e-05 +Pairwise Multileader Locality, 4 leaders: 3.053767e-04 +Nonblocking Multileader, 4 leaders: 1.807232e-04 +Nonblocking Locality Aware, 4 leaders: 5.856385e-05 +Nonblocking Multileader Locality, 4 leaders: 2.568193e-04 +Pairwise Multileader, 10 leaders: 1.375767e-04 +Pairwise Locality Aware, 10 leaders: 1.022468e-04 +Pairwise Multileader Locality, 10 leaders: 1.010256e-04 +Nonblocking Multileader, 10 leaders: 7.492265e-05 +Nonblocking Locality Aware, 10 leaders: 7.015761e-05 +Nonblocking Multileader Locality, 10 leaders: 8.479950e-05 +Pairwise Multileader, 20 leaders: 1.837564e-04 +Pairwise Locality Aware, 20 leaders: 1.955740e-04 +Pairwise Multileader Locality, 20 leaders: 1.109404e-04 +Nonblocking Multileader, 20 leaders: 7.963666e-05 +Nonblocking Locality Aware, 20 leaders: 1.041348e-04 +Nonblocking Multileader Locality, 20 leaders: 8.352947e-05 +Size 16 +PMPI: 1.325136e-04 +Pairwise: 8.273090e-04 +NonBlocking: 9.608389e-03 +Pairwise Hierarchical: 1.569944e-03 +Pairwise Node Aware: 3.269946e-04 +Nonblocking Hierarchical: 1.475392e-03 +Nonblocking Node Aware: 1.839242e-04 +Pairwise Multileader, 4 leaders: 4.782079e-04 +Pairwise Locality Aware, 4 leaders: 1.355785e-04 +Pairwise Multileader Locality, 4 leaders: 5.242201e-04 +Nonblocking Multileader, 4 leaders: 3.655697e-04 +Nonblocking Locality Aware, 4 leaders: 1.389383e-04 +Nonblocking Multileader Locality, 4 leaders: 4.766813e-04 +Pairwise Multileader, 10 leaders: 2.305791e-04 +Pairwise Locality Aware, 10 leaders: 1.345894e-04 +Pairwise Multileader Locality, 10 leaders: 2.028851e-04 +Nonblocking Multileader, 10 leaders: 1.514769e-04 +Nonblocking Locality Aware, 10 leaders: 9.767779e-05 +Nonblocking Multileader Locality, 10 leaders: 1.779032e-04 +Pairwise Multileader, 20 leaders: 2.247883e-04 +Pairwise Locality Aware, 20 leaders: 1.802893e-04 +Pairwise Multileader Locality, 20 leaders: 1.526593e-04 +Nonblocking Multileader, 20 leaders: 1.149517e-04 +Nonblocking Locality Aware, 20 leaders: 1.163174e-04 +Nonblocking Multileader Locality, 20 leaders: 1.117438e-04 +Size 32 +PMPI: 2.669267e-04 +Pairwise: 8.369190e-04 +NonBlocking: 1.213299e-02 +Pairwise Hierarchical: 3.188041e-03 +Pairwise Node Aware: 4.276724e-04 +Nonblocking Hierarchical: 3.186900e-03 +Nonblocking Node Aware: 2.140056e-04 +Pairwise Multileader, 4 leaders: 1.077070e-03 +Pairwise Locality Aware, 4 leaders: 2.139589e-04 +Pairwise Multileader Locality, 4 leaders: 1.047100e-03 +Nonblocking Multileader, 4 leaders: 8.721954e-04 +Nonblocking Locality Aware, 4 leaders: 1.748140e-04 +Nonblocking Multileader Locality, 4 leaders: 9.633667e-04 +Pairwise Multileader, 10 leaders: 4.356279e-04 +Pairwise Locality Aware, 10 leaders: 1.818480e-04 +Pairwise Multileader Locality, 10 leaders: 4.567475e-04 +Nonblocking Multileader, 10 leaders: 3.152411e-04 +Nonblocking Locality Aware, 10 leaders: 2.361929e-04 +Nonblocking Multileader Locality, 10 leaders: 4.182567e-04 +Pairwise Multileader, 20 leaders: 3.256093e-04 +Pairwise Locality Aware, 20 leaders: 2.259447e-04 +Pairwise Multileader Locality, 20 leaders: 2.632912e-04 +Nonblocking Multileader, 20 leaders: 1.965429e-04 +Nonblocking Locality Aware, 20 leaders: 3.118096e-04 +Nonblocking Multileader Locality, 20 leaders: 1.945235e-04 +Size 64 +PMPI: 4.918178e-04 +Pairwise: 8.551610e-04 +NonBlocking: 1.269418e-02 +Pairwise Hierarchical: 7.009382e-03 +Pairwise Node Aware: 5.002700e-04 +Nonblocking Hierarchical: 6.682868e-03 +Nonblocking Node Aware: 3.166571e-04 +Pairwise Multileader, 4 leaders: 2.424219e-03 +Pairwise Locality Aware, 4 leaders: 3.353645e-04 +Pairwise Multileader Locality, 4 leaders: 2.376882e-03 +Nonblocking Multileader, 4 leaders: 1.973539e-03 +Nonblocking Locality Aware, 4 leaders: 2.762722e-04 +Nonblocking Multileader Locality, 4 leaders: 2.171329e-03 +Pairwise Multileader, 10 leaders: 9.625642e-04 +Pairwise Locality Aware, 10 leaders: 3.152816e-04 +Pairwise Multileader Locality, 10 leaders: 9.780044e-04 +Nonblocking Multileader, 10 leaders: 7.272884e-04 +Nonblocking Locality Aware, 10 leaders: 3.119121e-04 +Nonblocking Multileader Locality, 10 leaders: 8.803907e-04 +Pairwise Multileader, 20 leaders: 6.263634e-04 +Pairwise Locality Aware, 20 leaders: 3.743516e-04 +Pairwise Multileader Locality, 20 leaders: 5.257592e-04 +Nonblocking Multileader, 20 leaders: 3.660853e-04 +Nonblocking Locality Aware, 20 leaders: 3.726856e-04 +Nonblocking Multileader Locality, 20 leaders: 4.432228e-04 +Size 128 +PMPI: 8.340650e-04 +Pairwise: 1.118359e-03 +NonBlocking: 8.654171e-02 +Pairwise Hierarchical: 1.385158e-02 +Pairwise Node Aware: 8.507360e-04 +Nonblocking Hierarchical: 1.330965e-02 +Nonblocking Node Aware: 5.110502e-04 +Pairwise Multileader, 4 leaders: 4.360620e-03 +Pairwise Locality Aware, 4 leaders: 6.483268e-04 +Pairwise Multileader Locality, 4 leaders: 4.587500e-03 +Nonblocking Multileader, 4 leaders: 3.789399e-03 +Nonblocking Locality Aware, 4 leaders: 6.167606e-04 +Nonblocking Multileader Locality, 4 leaders: 4.246863e-03 +Pairwise Multileader, 10 leaders: 1.808768e-03 +Pairwise Locality Aware, 10 leaders: 6.689437e-04 +Pairwise Multileader Locality, 10 leaders: 1.791511e-03 +Nonblocking Multileader, 10 leaders: 1.398513e-03 +Nonblocking Locality Aware, 10 leaders: 5.615666e-04 +Nonblocking Multileader Locality, 10 leaders: 1.641548e-03 +Pairwise Multileader, 20 leaders: 1.075740e-03 +Pairwise Locality Aware, 20 leaders: 6.737900e-04 +Pairwise Multileader Locality, 20 leaders: 1.063559e-03 +Nonblocking Multileader, 20 leaders: 7.338734e-04 +Nonblocking Locality Aware, 20 leaders: 5.417743e-04 +Nonblocking Multileader Locality, 20 leaders: 9.996719e-04 +Size 256 +PMPI: 1.503097e-03 +Pairwise: 1.436752e-03 +NonBlocking: 7.435168e-02 +Pairwise Hierarchical: 2.861446e-02 +Pairwise Node Aware: 1.411092e-03 +Nonblocking Hierarchical: 2.897365e-02 +Nonblocking Node Aware: 9.244250e-04 +Pairwise Multileader, 4 leaders: 7.996853e-03 +Pairwise Locality Aware, 4 leaders: 1.184098e-03 +Pairwise Multileader Locality, 4 leaders: 8.287501e-03 +Nonblocking Multileader, 4 leaders: 6.626458e-03 +Nonblocking Locality Aware, 4 leaders: 1.099549e-03 +Nonblocking Multileader Locality, 4 leaders: 7.696544e-03 +Pairwise Multileader, 10 leaders: 4.425042e-03 +Pairwise Locality Aware, 10 leaders: 1.245465e-03 +Pairwise Multileader Locality, 10 leaders: 3.743074e-03 +Nonblocking Multileader, 10 leaders: 3.254699e-03 +Nonblocking Locality Aware, 10 leaders: 1.102237e-03 +Nonblocking Multileader Locality, 10 leaders: 3.612718e-03 +Pairwise Multileader, 20 leaders: 2.182740e-03 +Pairwise Locality Aware, 20 leaders: 1.161376e-03 +Pairwise Multileader Locality, 20 leaders: 2.258190e-03 +Nonblocking Multileader, 20 leaders: 1.559239e-03 +Nonblocking Locality Aware, 20 leaders: 1.037882e-03 +Nonblocking Multileader Locality, 20 leaders: 2.029369e-03 +Size 512 +PMPI: 2.184098e-03 +Pairwise: 2.324544e-03 +NonBlocking: 5.108632e-02 +Pairwise Hierarchical: 5.697611e-02 +Pairwise Node Aware: 2.334024e-03 +Nonblocking Hierarchical: 5.945374e-02 +Nonblocking Node Aware: 1.767727e-03 +Pairwise Multileader, 4 leaders: 1.756384e-02 +Pairwise Locality Aware, 4 leaders: 2.406036e-03 +Pairwise Multileader Locality, 4 leaders: 1.693270e-02 +Nonblocking Multileader, 4 leaders: 1.402018e-02 +Nonblocking Locality Aware, 4 leaders: 2.169632e-03 +Nonblocking Multileader Locality, 4 leaders: 1.584906e-02 +Pairwise Multileader, 10 leaders: 9.281894e-03 +Pairwise Locality Aware, 10 leaders: 2.210701e-03 +Pairwise Multileader Locality, 10 leaders: 8.285684e-03 +Nonblocking Multileader, 10 leaders: 6.939021e-03 +Nonblocking Locality Aware, 10 leaders: 1.920852e-03 +Nonblocking Multileader Locality, 10 leaders: 7.772503e-03 +Pairwise Multileader, 20 leaders: 4.472860e-03 +Pairwise Locality Aware, 20 leaders: 2.077766e-03 +Pairwise Multileader Locality, 20 leaders: 4.391520e-03 +Nonblocking Multileader, 20 leaders: 3.055604e-03 +Nonblocking Locality Aware, 20 leaders: 1.868476e-03 +Nonblocking Multileader Locality, 20 leaders: 4.006575e-03 diff --git a/runscripts/dane_results/alltoall_N2.2484423.err b/runscripts/dane_results/alltoall_N2.2484423.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/dane_results/alltoall_N2.2484423.out b/runscripts/dane_results/alltoall_N2.2484423.out new file mode 100644 index 000000000..90f215605 --- /dev/null +++ b/runscripts/dane_results/alltoall_N2.2484423.out @@ -0,0 +1,783 @@ +Test 1 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 3.981033e-05 +Pairwise: 3.552966e-04 +NonBlocking: 1.935242e-04 +Pairwise Hierarchical: 1.861620e-04 +Pairwise Node Aware: 1.833722e-04 +Nonblocking Hierarchical: 1.892647e-04 +Nonblocking Node Aware: 1.844633e-04 +Pairwise Multileader, 4 leaders: 6.194942e-05 +Pairwise Locality Aware, 4 leaders: 7.336113e-05 +Pairwise Multileader Locality, 4 leaders: 5.982052e-05 +Nonblocking Multileader, 4 leaders: 4.839142e-05 +Nonblocking Locality Aware, 4 leaders: 4.903204e-05 +Nonblocking Multileader Locality, 4 leaders: 5.461679e-05 +Pairwise Multileader, 10 leaders: 7.657156e-05 +Pairwise Locality Aware, 10 leaders: 8.640277e-05 +Pairwise Multileader Locality, 10 leaders: 3.994850e-05 +Nonblocking Multileader, 10 leaders: 3.701267e-05 +Nonblocking Locality Aware, 10 leaders: 4.120472e-05 +Nonblocking Multileader Locality, 10 leaders: 3.935169e-05 +Pairwise Multileader, 20 leaders: 1.515443e-04 +Pairwise Locality Aware, 20 leaders: 1.524144e-04 +Pairwise Multileader Locality, 20 leaders: 5.585190e-05 +Nonblocking Multileader, 20 leaders: 5.587883e-05 +Nonblocking Locality Aware, 20 leaders: 5.474362e-05 +Nonblocking Multileader Locality, 20 leaders: 4.241650e-05 +Size 2 +PMPI: 4.453471e-05 +Pairwise: 3.690559e-04 +NonBlocking: 1.905978e-04 +Pairwise Hierarchical: 2.260564e-04 +Pairwise Node Aware: 1.857448e-04 +Nonblocking Hierarchical: 2.371366e-04 +Nonblocking Node Aware: 1.865870e-04 +Pairwise Multileader, 4 leaders: 8.027393e-05 +Pairwise Locality Aware, 4 leaders: 7.558285e-05 +Pairwise Multileader Locality, 4 leaders: 8.315606e-05 +Nonblocking Multileader, 4 leaders: 6.500051e-05 +Nonblocking Locality Aware, 4 leaders: 5.184792e-05 +Nonblocking Multileader Locality, 4 leaders: 7.999099e-05 +Pairwise Multileader, 10 leaders: 9.352630e-05 +Pairwise Locality Aware, 10 leaders: 9.220459e-05 +Pairwise Multileader Locality, 10 leaders: 5.013580e-05 +Nonblocking Multileader, 10 leaders: 4.398309e-05 +Nonblocking Locality Aware, 10 leaders: 4.883250e-05 +Nonblocking Multileader Locality, 10 leaders: 4.723347e-05 +Pairwise Multileader, 20 leaders: 1.684905e-04 +Pairwise Locality Aware, 20 leaders: 1.425443e-04 +Pairwise Multileader Locality, 20 leaders: 6.324803e-05 +Nonblocking Multileader, 20 leaders: 5.964532e-05 +Nonblocking Locality Aware, 20 leaders: 6.024752e-05 +Nonblocking Multileader Locality, 20 leaders: 4.643631e-05 +Size 4 +PMPI: 5.339139e-05 +Pairwise: 6.975652e-04 +NonBlocking: 3.317824e-03 +Pairwise Hierarchical: 2.917024e-04 +Pairwise Node Aware: 2.319442e-04 +Nonblocking Hierarchical: 2.930318e-04 +Nonblocking Node Aware: 1.885148e-04 +Pairwise Multileader, 4 leaders: 1.129476e-04 +Pairwise Locality Aware, 4 leaders: 8.696280e-05 +Pairwise Multileader Locality, 4 leaders: 1.187685e-04 +Nonblocking Multileader, 4 leaders: 9.219630e-05 +Nonblocking Locality Aware, 4 leaders: 5.296733e-05 +Nonblocking Multileader Locality, 4 leaders: 1.110842e-04 +Pairwise Multileader, 10 leaders: 1.080886e-04 +Pairwise Locality Aware, 10 leaders: 1.112641e-04 +Pairwise Multileader Locality, 10 leaders: 6.909681e-05 +Nonblocking Multileader, 10 leaders: 5.711941e-05 +Nonblocking Locality Aware, 10 leaders: 5.231307e-05 +Nonblocking Multileader Locality, 10 leaders: 6.089874e-05 +Pairwise Multileader, 20 leaders: 1.671222e-04 +Pairwise Locality Aware, 20 leaders: 1.813529e-04 +Pairwise Multileader Locality, 20 leaders: 7.812968e-05 +Nonblocking Multileader, 20 leaders: 6.943513e-05 +Nonblocking Locality Aware, 20 leaders: 7.434138e-05 +Nonblocking Multileader Locality, 20 leaders: 6.062134e-05 +Size 8 +PMPI: 8.781898e-05 +Pairwise: 7.553066e-04 +NonBlocking: 6.449358e-03 +Pairwise Hierarchical: 8.180043e-04 +Pairwise Node Aware: 2.471056e-04 +Nonblocking Hierarchical: 7.792531e-04 +Nonblocking Node Aware: 1.869744e-04 +Pairwise Multileader, 4 leaders: 2.293004e-04 +Pairwise Locality Aware, 4 leaders: 9.117579e-05 +Pairwise Multileader Locality, 4 leaders: 2.947229e-04 +Nonblocking Multileader, 4 leaders: 1.810202e-04 +Nonblocking Locality Aware, 4 leaders: 6.234923e-05 +Nonblocking Multileader Locality, 4 leaders: 2.598161e-04 +Pairwise Multileader, 10 leaders: 1.293692e-04 +Pairwise Locality Aware, 10 leaders: 9.533055e-05 +Pairwise Multileader Locality, 10 leaders: 1.031675e-04 +Nonblocking Multileader, 10 leaders: 7.213095e-05 +Nonblocking Locality Aware, 10 leaders: 5.895659e-05 +Nonblocking Multileader Locality, 10 leaders: 8.461359e-05 +Pairwise Multileader, 20 leaders: 1.667340e-04 +Pairwise Locality Aware, 20 leaders: 1.705330e-04 +Pairwise Multileader Locality, 20 leaders: 1.092425e-04 +Nonblocking Multileader, 20 leaders: 7.808238e-05 +Nonblocking Locality Aware, 20 leaders: 8.011354e-05 +Nonblocking Multileader Locality, 20 leaders: 8.361113e-05 +Size 16 +PMPI: 1.345446e-04 +Pairwise: 7.206510e-04 +NonBlocking: 5.608747e-03 +Pairwise Hierarchical: 1.549619e-03 +Pairwise Node Aware: 3.105670e-04 +Nonblocking Hierarchical: 1.490158e-03 +Nonblocking Node Aware: 1.838121e-04 +Pairwise Multileader, 4 leaders: 4.564592e-04 +Pairwise Locality Aware, 4 leaders: 1.347917e-04 +Pairwise Multileader Locality, 4 leaders: 5.462404e-04 +Nonblocking Multileader, 4 leaders: 3.597454e-04 +Nonblocking Locality Aware, 4 leaders: 1.196709e-04 +Nonblocking Multileader Locality, 4 leaders: 4.843273e-04 +Pairwise Multileader, 10 leaders: 2.384435e-04 +Pairwise Locality Aware, 10 leaders: 1.254049e-04 +Pairwise Multileader Locality, 10 leaders: 2.022059e-04 +Nonblocking Multileader, 10 leaders: 1.513696e-04 +Nonblocking Locality Aware, 10 leaders: 8.039008e-05 +Nonblocking Multileader Locality, 10 leaders: 1.756061e-04 +Pairwise Multileader, 20 leaders: 2.137305e-04 +Pairwise Locality Aware, 20 leaders: 1.648128e-04 +Pairwise Multileader Locality, 20 leaders: 1.646745e-04 +Nonblocking Multileader, 20 leaders: 1.161321e-04 +Nonblocking Locality Aware, 20 leaders: 1.234945e-04 +Nonblocking Multileader Locality, 20 leaders: 1.259931e-04 +Size 32 +PMPI: 2.585556e-04 +Pairwise: 8.039697e-04 +NonBlocking: 1.259010e-02 +Pairwise Hierarchical: 3.045975e-03 +Pairwise Node Aware: 4.418820e-04 +Nonblocking Hierarchical: 2.946688e-03 +Nonblocking Node Aware: 2.403964e-04 +Pairwise Multileader, 4 leaders: 9.560401e-04 +Pairwise Locality Aware, 4 leaders: 2.058313e-04 +Pairwise Multileader Locality, 4 leaders: 9.703639e-04 +Nonblocking Multileader, 4 leaders: 8.359352e-04 +Nonblocking Locality Aware, 4 leaders: 1.666046e-04 +Nonblocking Multileader Locality, 4 leaders: 9.130854e-04 +Pairwise Multileader, 10 leaders: 5.134520e-04 +Pairwise Locality Aware, 10 leaders: 1.932632e-04 +Pairwise Multileader Locality, 10 leaders: 4.539675e-04 +Nonblocking Multileader, 10 leaders: 3.156696e-04 +Nonblocking Locality Aware, 10 leaders: 2.020848e-04 +Nonblocking Multileader Locality, 10 leaders: 4.012335e-04 +Pairwise Multileader, 20 leaders: 3.087479e-04 +Pairwise Locality Aware, 20 leaders: 2.223731e-04 +Pairwise Multileader Locality, 20 leaders: 2.564843e-04 +Nonblocking Multileader, 20 leaders: 1.842016e-04 +Nonblocking Locality Aware, 20 leaders: 2.167137e-04 +Nonblocking Multileader Locality, 20 leaders: 2.185047e-04 +Size 64 +PMPI: 4.334599e-04 +Pairwise: 7.637248e-04 +NonBlocking: 8.642785e-02 +Pairwise Hierarchical: 6.885252e-03 +Pairwise Node Aware: 4.602937e-04 +Nonblocking Hierarchical: 6.638690e-03 +Nonblocking Node Aware: 2.933072e-04 +Pairwise Multileader, 4 leaders: 2.401410e-03 +Pairwise Locality Aware, 4 leaders: 3.400467e-04 +Pairwise Multileader Locality, 4 leaders: 2.403223e-03 +Nonblocking Multileader, 4 leaders: 1.948774e-03 +Nonblocking Locality Aware, 4 leaders: 2.810889e-04 +Nonblocking Multileader Locality, 4 leaders: 2.217174e-03 +Pairwise Multileader, 10 leaders: 9.235117e-04 +Pairwise Locality Aware, 10 leaders: 3.444697e-04 +Pairwise Multileader Locality, 10 leaders: 9.887246e-04 +Nonblocking Multileader, 10 leaders: 7.161008e-04 +Nonblocking Locality Aware, 10 leaders: 2.365269e-04 +Nonblocking Multileader Locality, 10 leaders: 8.807560e-04 +Pairwise Multileader, 20 leaders: 5.541451e-04 +Pairwise Locality Aware, 20 leaders: 3.152483e-04 +Pairwise Multileader Locality, 20 leaders: 4.876734e-04 +Nonblocking Multileader, 20 leaders: 3.341524e-04 +Nonblocking Locality Aware, 20 leaders: 2.627466e-04 +Nonblocking Multileader Locality, 20 leaders: 4.233459e-04 +Size 128 +PMPI: 8.114325e-04 +Pairwise: 8.947251e-04 +NonBlocking: 2.030868e-02 +Pairwise Hierarchical: 1.341488e-02 +Pairwise Node Aware: 7.473390e-04 +Nonblocking Hierarchical: 1.323640e-02 +Nonblocking Node Aware: 4.872642e-04 +Pairwise Multileader, 4 leaders: 4.470851e-03 +Pairwise Locality Aware, 4 leaders: 5.722129e-04 +Pairwise Multileader Locality, 4 leaders: 4.664080e-03 +Nonblocking Multileader, 4 leaders: 3.718953e-03 +Nonblocking Locality Aware, 4 leaders: 4.796816e-04 +Nonblocking Multileader Locality, 4 leaders: 4.287198e-03 +Pairwise Multileader, 10 leaders: 1.807165e-03 +Pairwise Locality Aware, 10 leaders: 5.617021e-04 +Pairwise Multileader Locality, 10 leaders: 1.874884e-03 +Nonblocking Multileader, 10 leaders: 1.466885e-03 +Nonblocking Locality Aware, 10 leaders: 4.742798e-04 +Nonblocking Multileader Locality, 10 leaders: 1.723258e-03 +Pairwise Multileader, 20 leaders: 1.107226e-03 +Pairwise Locality Aware, 20 leaders: 5.509118e-04 +Pairwise Multileader Locality, 20 leaders: 1.053092e-03 +Nonblocking Multileader, 20 leaders: 7.516261e-04 +Nonblocking Locality Aware, 20 leaders: 4.380482e-04 +Nonblocking Multileader Locality, 20 leaders: 9.989666e-04 +Size 256 +PMPI: 1.483166e-03 +Pairwise: 1.262883e-03 +NonBlocking: 2.554217e-02 +Pairwise Hierarchical: 2.854503e-02 +Pairwise Node Aware: 1.269194e-03 +Nonblocking Hierarchical: 2.881218e-02 +Nonblocking Node Aware: 8.941364e-04 +Pairwise Multileader, 4 leaders: 8.045638e-03 +Pairwise Locality Aware, 4 leaders: 1.070865e-03 +Pairwise Multileader Locality, 4 leaders: 8.418356e-03 +Nonblocking Multileader, 4 leaders: 6.591755e-03 +Nonblocking Locality Aware, 4 leaders: 8.983449e-04 +Nonblocking Multileader Locality, 4 leaders: 7.759697e-03 +Pairwise Multileader, 10 leaders: 4.374100e-03 +Pairwise Locality Aware, 10 leaders: 1.098943e-03 +Pairwise Multileader Locality, 10 leaders: 3.748519e-03 +Nonblocking Multileader, 10 leaders: 3.203749e-03 +Nonblocking Locality Aware, 10 leaders: 9.884860e-04 +Nonblocking Multileader Locality, 10 leaders: 3.536862e-03 +Pairwise Multileader, 20 leaders: 2.184865e-03 +Pairwise Locality Aware, 20 leaders: 1.022980e-03 +Pairwise Multileader Locality, 20 leaders: 2.229082e-03 +Nonblocking Multileader, 20 leaders: 1.591847e-03 +Nonblocking Locality Aware, 20 leaders: 8.826187e-04 +Nonblocking Multileader Locality, 20 leaders: 1.984345e-03 +Size 512 +PMPI: 2.134147e-03 +Pairwise: 2.031809e-03 +NonBlocking: 5.104460e-03 +Pairwise Hierarchical: 5.746070e-02 +Pairwise Node Aware: 2.193880e-03 +Nonblocking Hierarchical: 5.954505e-02 +Nonblocking Node Aware: 1.759310e-03 +Pairwise Multileader, 4 leaders: 1.754344e-02 +Pairwise Locality Aware, 4 leaders: 2.117047e-03 +Pairwise Multileader Locality, 4 leaders: 1.765836e-02 +Nonblocking Multileader, 4 leaders: 1.424319e-02 +Nonblocking Locality Aware, 4 leaders: 1.930580e-03 +Nonblocking Multileader Locality, 4 leaders: 1.576154e-02 +Pairwise Multileader, 10 leaders: 9.232665e-03 +Pairwise Locality Aware, 10 leaders: 2.097816e-03 +Pairwise Multileader Locality, 10 leaders: 8.091770e-03 +Nonblocking Multileader, 10 leaders: 6.969385e-03 +Nonblocking Locality Aware, 10 leaders: 1.871905e-03 +Nonblocking Multileader Locality, 10 leaders: 7.680688e-03 +Pairwise Multileader, 20 leaders: 4.553077e-03 +Pairwise Locality Aware, 20 leaders: 2.064717e-03 +Pairwise Multileader Locality, 20 leaders: 4.369465e-03 +Nonblocking Multileader, 20 leaders: 3.124243e-03 +Nonblocking Locality Aware, 20 leaders: 1.836565e-03 +Nonblocking Multileader Locality, 20 leaders: 3.860284e-03 +Test 2 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 3.913165e-05 +Pairwise: 3.550015e-04 +NonBlocking: 1.821182e-04 +Pairwise Hierarchical: 1.823975e-04 +Pairwise Node Aware: 1.762372e-04 +Nonblocking Hierarchical: 1.861578e-04 +Nonblocking Node Aware: 1.854180e-04 +Pairwise Multileader, 4 leaders: 6.764998e-05 +Pairwise Locality Aware, 4 leaders: 7.474301e-05 +Pairwise Multileader Locality, 4 leaders: 5.841543e-05 +Nonblocking Multileader, 4 leaders: 4.702758e-05 +Nonblocking Locality Aware, 4 leaders: 4.862604e-05 +Nonblocking Multileader Locality, 4 leaders: 5.445931e-05 +Pairwise Multileader, 10 leaders: 7.520202e-05 +Pairwise Locality Aware, 10 leaders: 8.330789e-05 +Pairwise Multileader Locality, 10 leaders: 3.908483e-05 +Nonblocking Multileader, 10 leaders: 3.667671e-05 +Nonblocking Locality Aware, 10 leaders: 3.907710e-05 +Nonblocking Multileader Locality, 10 leaders: 3.543790e-05 +Pairwise Multileader, 20 leaders: 1.781697e-04 +Pairwise Locality Aware, 20 leaders: 1.838477e-04 +Pairwise Multileader Locality, 20 leaders: 5.592940e-05 +Nonblocking Multileader, 20 leaders: 5.525906e-05 +Nonblocking Locality Aware, 20 leaders: 4.983389e-05 +Nonblocking Multileader Locality, 20 leaders: 4.153436e-05 +Size 2 +PMPI: 4.344247e-05 +Pairwise: 3.685157e-04 +NonBlocking: 1.857913e-04 +Pairwise Hierarchical: 2.193368e-04 +Pairwise Node Aware: 1.789882e-04 +Nonblocking Hierarchical: 2.179995e-04 +Nonblocking Node Aware: 1.822192e-04 +Pairwise Multileader, 4 leaders: 7.694189e-05 +Pairwise Locality Aware, 4 leaders: 7.395558e-05 +Pairwise Multileader Locality, 4 leaders: 7.907704e-05 +Nonblocking Multileader, 4 leaders: 6.094471e-05 +Nonblocking Locality Aware, 4 leaders: 5.029859e-05 +Nonblocking Multileader Locality, 4 leaders: 7.675193e-05 +Pairwise Multileader, 10 leaders: 8.308942e-05 +Pairwise Locality Aware, 10 leaders: 8.674265e-05 +Pairwise Multileader Locality, 10 leaders: 4.953072e-05 +Nonblocking Multileader, 10 leaders: 4.272556e-05 +Nonblocking Locality Aware, 10 leaders: 4.244699e-05 +Nonblocking Multileader Locality, 10 leaders: 4.279131e-05 +Pairwise Multileader, 20 leaders: 1.465989e-04 +Pairwise Locality Aware, 20 leaders: 1.504018e-04 +Pairwise Multileader Locality, 20 leaders: 6.354862e-05 +Nonblocking Multileader, 20 leaders: 6.001333e-05 +Nonblocking Locality Aware, 20 leaders: 5.600286e-05 +Nonblocking Multileader Locality, 20 leaders: 4.702730e-05 +Size 4 +PMPI: 5.124183e-05 +Pairwise: 6.697971e-04 +NonBlocking: 2.775339e-03 +Pairwise Hierarchical: 4.449262e-04 +Pairwise Node Aware: 2.328125e-04 +Nonblocking Hierarchical: 4.370894e-04 +Nonblocking Node Aware: 1.868867e-04 +Pairwise Multileader, 4 leaders: 1.123087e-04 +Pairwise Locality Aware, 4 leaders: 8.251533e-05 +Pairwise Multileader Locality, 4 leaders: 1.249539e-04 +Nonblocking Multileader, 4 leaders: 9.160174e-05 +Nonblocking Locality Aware, 4 leaders: 5.762939e-05 +Nonblocking Multileader Locality, 4 leaders: 1.096412e-04 +Pairwise Multileader, 10 leaders: 9.459541e-05 +Pairwise Locality Aware, 10 leaders: 9.480085e-05 +Pairwise Multileader Locality, 10 leaders: 6.908152e-05 +Nonblocking Multileader, 10 leaders: 5.144657e-05 +Nonblocking Locality Aware, 10 leaders: 5.757831e-05 +Nonblocking Multileader Locality, 10 leaders: 6.891868e-05 +Pairwise Multileader, 20 leaders: 1.576824e-04 +Pairwise Locality Aware, 20 leaders: 1.560975e-04 +Pairwise Multileader Locality, 20 leaders: 7.670574e-05 +Nonblocking Multileader, 20 leaders: 6.475866e-05 +Nonblocking Locality Aware, 20 leaders: 7.348242e-05 +Nonblocking Multileader Locality, 20 leaders: 5.986548e-05 +Size 8 +PMPI: 8.078483e-05 +Pairwise: 6.861884e-04 +NonBlocking: 3.775518e-03 +Pairwise Hierarchical: 7.963193e-04 +Pairwise Node Aware: 2.638670e-04 +Nonblocking Hierarchical: 7.592528e-04 +Nonblocking Node Aware: 1.844916e-04 +Pairwise Multileader, 4 leaders: 2.259881e-04 +Pairwise Locality Aware, 4 leaders: 9.691857e-05 +Pairwise Multileader Locality, 4 leaders: 2.962913e-04 +Nonblocking Multileader, 4 leaders: 1.859053e-04 +Nonblocking Locality Aware, 4 leaders: 1.383817e-04 +Nonblocking Multileader Locality, 4 leaders: 2.657137e-04 +Pairwise Multileader, 10 leaders: 1.172257e-04 +Pairwise Locality Aware, 10 leaders: 9.988538e-05 +Pairwise Multileader Locality, 10 leaders: 1.044660e-04 +Nonblocking Multileader, 10 leaders: 7.460722e-05 +Nonblocking Locality Aware, 10 leaders: 6.746281e-05 +Nonblocking Multileader Locality, 10 leaders: 9.017375e-05 +Pairwise Multileader, 20 leaders: 1.691401e-04 +Pairwise Locality Aware, 20 leaders: 1.678271e-04 +Pairwise Multileader Locality, 20 leaders: 1.086238e-04 +Nonblocking Multileader, 20 leaders: 8.377316e-05 +Nonblocking Locality Aware, 20 leaders: 1.121781e-04 +Nonblocking Multileader Locality, 20 leaders: 8.513960e-05 +Size 16 +PMPI: 1.531588e-04 +Pairwise: 7.438983e-04 +NonBlocking: 1.407674e-02 +Pairwise Hierarchical: 1.472987e-03 +Pairwise Node Aware: 3.268372e-04 +Nonblocking Hierarchical: 1.440120e-03 +Nonblocking Node Aware: 1.850449e-04 +Pairwise Multileader, 4 leaders: 4.466200e-04 +Pairwise Locality Aware, 4 leaders: 1.354388e-04 +Pairwise Multileader Locality, 4 leaders: 5.347065e-04 +Nonblocking Multileader, 4 leaders: 3.677192e-04 +Nonblocking Locality Aware, 4 leaders: 1.548563e-04 +Nonblocking Multileader Locality, 4 leaders: 4.801780e-04 +Pairwise Multileader, 10 leaders: 2.282985e-04 +Pairwise Locality Aware, 10 leaders: 1.279583e-04 +Pairwise Multileader Locality, 10 leaders: 2.043053e-04 +Nonblocking Multileader, 10 leaders: 1.515307e-04 +Nonblocking Locality Aware, 10 leaders: 1.019874e-04 +Nonblocking Multileader Locality, 10 leaders: 1.677594e-04 +Pairwise Multileader, 20 leaders: 2.075234e-04 +Pairwise Locality Aware, 20 leaders: 1.779641e-04 +Pairwise Multileader Locality, 20 leaders: 1.752713e-04 +Nonblocking Multileader, 20 leaders: 1.180852e-04 +Nonblocking Locality Aware, 20 leaders: 1.413047e-04 +Nonblocking Multileader Locality, 20 leaders: 1.167861e-04 +Size 32 +PMPI: 2.523348e-04 +Pairwise: 7.773013e-04 +NonBlocking: 2.490179e-02 +Pairwise Hierarchical: 3.071608e-03 +Pairwise Node Aware: 4.893201e-04 +Nonblocking Hierarchical: 2.958764e-03 +Nonblocking Node Aware: 2.354149e-04 +Pairwise Multileader, 4 leaders: 1.193141e-03 +Pairwise Locality Aware, 4 leaders: 2.384262e-04 +Pairwise Multileader Locality, 4 leaders: 1.061040e-03 +Nonblocking Multileader, 4 leaders: 8.653025e-04 +Nonblocking Locality Aware, 4 leaders: 1.849407e-04 +Nonblocking Multileader Locality, 4 leaders: 9.271015e-04 +Pairwise Multileader, 10 leaders: 4.390312e-04 +Pairwise Locality Aware, 10 leaders: 2.073788e-04 +Pairwise Multileader Locality, 10 leaders: 4.603597e-04 +Nonblocking Multileader, 10 leaders: 3.204179e-04 +Nonblocking Locality Aware, 10 leaders: 2.375579e-04 +Nonblocking Multileader Locality, 10 leaders: 4.326424e-04 +Pairwise Multileader, 20 leaders: 3.277807e-04 +Pairwise Locality Aware, 20 leaders: 2.482664e-04 +Pairwise Multileader Locality, 20 leaders: 2.846759e-04 +Nonblocking Multileader, 20 leaders: 1.770144e-04 +Nonblocking Locality Aware, 20 leaders: 3.424280e-04 +Nonblocking Multileader Locality, 20 leaders: 2.076198e-04 +Size 64 +PMPI: 4.564021e-04 +Pairwise: 7.631652e-04 +NonBlocking: 4.952891e-02 +Pairwise Hierarchical: 6.963114e-03 +Pairwise Node Aware: 4.712598e-04 +Nonblocking Hierarchical: 6.669673e-03 +Nonblocking Node Aware: 3.151396e-04 +Pairwise Multileader, 4 leaders: 2.383128e-03 +Pairwise Locality Aware, 4 leaders: 3.322028e-04 +Pairwise Multileader Locality, 4 leaders: 2.398872e-03 +Nonblocking Multileader, 4 leaders: 1.963361e-03 +Nonblocking Locality Aware, 4 leaders: 2.499774e-04 +Nonblocking Multileader Locality, 4 leaders: 2.203038e-03 +Pairwise Multileader, 10 leaders: 9.351773e-04 +Pairwise Locality Aware, 10 leaders: 3.342109e-04 +Pairwise Multileader Locality, 10 leaders: 9.520605e-04 +Nonblocking Multileader, 10 leaders: 7.163405e-04 +Nonblocking Locality Aware, 10 leaders: 2.872145e-04 +Nonblocking Multileader Locality, 10 leaders: 8.553974e-04 +Pairwise Multileader, 20 leaders: 6.067885e-04 +Pairwise Locality Aware, 20 leaders: 3.236734e-04 +Pairwise Multileader Locality, 20 leaders: 5.504123e-04 +Nonblocking Multileader, 20 leaders: 3.899120e-04 +Nonblocking Locality Aware, 20 leaders: 3.003014e-04 +Nonblocking Multileader Locality, 20 leaders: 4.708105e-04 +Size 128 +PMPI: 8.427298e-04 +Pairwise: 9.361251e-04 +NonBlocking: 8.656836e-02 +Pairwise Hierarchical: 1.395215e-02 +Pairwise Node Aware: 7.865596e-04 +Nonblocking Hierarchical: 1.336836e-02 +Nonblocking Node Aware: 5.553072e-04 +Pairwise Multileader, 4 leaders: 4.537025e-03 +Pairwise Locality Aware, 4 leaders: 6.152105e-04 +Pairwise Multileader Locality, 4 leaders: 4.775072e-03 +Nonblocking Multileader, 4 leaders: 3.838842e-03 +Nonblocking Locality Aware, 4 leaders: 5.469993e-04 +Nonblocking Multileader Locality, 4 leaders: 4.330079e-03 +Pairwise Multileader, 10 leaders: 1.860452e-03 +Pairwise Locality Aware, 10 leaders: 6.255374e-04 +Pairwise Multileader Locality, 10 leaders: 1.908798e-03 +Nonblocking Multileader, 10 leaders: 1.488362e-03 +Nonblocking Locality Aware, 10 leaders: 5.072121e-04 +Nonblocking Multileader Locality, 10 leaders: 1.761995e-03 +Pairwise Multileader, 20 leaders: 1.255541e-03 +Pairwise Locality Aware, 20 leaders: 6.374983e-04 +Pairwise Multileader Locality, 20 leaders: 1.097882e-03 +Nonblocking Multileader, 20 leaders: 7.967489e-04 +Nonblocking Locality Aware, 20 leaders: 5.583417e-04 +Nonblocking Multileader Locality, 20 leaders: 1.062774e-03 +Size 256 +PMPI: 1.535419e-03 +Pairwise: 1.366168e-03 +NonBlocking: 6.223068e-02 +Pairwise Hierarchical: 2.922300e-02 +Pairwise Node Aware: 1.325662e-03 +Nonblocking Hierarchical: 2.861292e-02 +Nonblocking Node Aware: 1.028994e-03 +Pairwise Multileader, 4 leaders: 7.530655e-03 +Pairwise Locality Aware, 4 leaders: 1.130342e-03 +Pairwise Multileader Locality, 4 leaders: 9.366535e-03 +Nonblocking Multileader, 4 leaders: 6.621888e-03 +Nonblocking Locality Aware, 4 leaders: 1.004499e-03 +Nonblocking Multileader Locality, 4 leaders: 7.846777e-03 +Pairwise Multileader, 10 leaders: 4.434044e-03 +Pairwise Locality Aware, 10 leaders: 1.171174e-03 +Pairwise Multileader Locality, 10 leaders: 3.799330e-03 +Nonblocking Multileader, 10 leaders: 3.296978e-03 +Nonblocking Locality Aware, 10 leaders: 1.027379e-03 +Nonblocking Multileader Locality, 10 leaders: 3.669739e-03 +Pairwise Multileader, 20 leaders: 2.402968e-03 +Pairwise Locality Aware, 20 leaders: 1.179413e-03 +Pairwise Multileader Locality, 20 leaders: 2.251600e-03 +Nonblocking Multileader, 20 leaders: 1.660029e-03 +Nonblocking Locality Aware, 20 leaders: 1.119503e-03 +Nonblocking Multileader Locality, 20 leaders: 1.991291e-03 +Size 512 +PMPI: 2.258153e-03 +Pairwise: 2.608918e-03 +NonBlocking: 3.416884e-03 +Pairwise Hierarchical: 5.686555e-02 +Pairwise Node Aware: 2.268409e-03 +Nonblocking Hierarchical: 5.902807e-02 +Nonblocking Node Aware: 2.104612e-03 +Pairwise Multileader, 4 leaders: 1.712161e-02 +Pairwise Locality Aware, 4 leaders: 2.114295e-03 +Pairwise Multileader Locality, 4 leaders: 1.707310e-02 +Nonblocking Multileader, 4 leaders: 1.398240e-02 +Nonblocking Locality Aware, 4 leaders: 1.819889e-03 +Nonblocking Multileader Locality, 4 leaders: 1.570212e-02 +Pairwise Multileader, 10 leaders: 9.293999e-03 +Pairwise Locality Aware, 10 leaders: 2.103129e-03 +Pairwise Multileader Locality, 10 leaders: 8.099314e-03 +Nonblocking Multileader, 10 leaders: 7.184742e-03 +Nonblocking Locality Aware, 10 leaders: 2.018192e-03 +Nonblocking Multileader Locality, 10 leaders: 7.682547e-03 +Pairwise Multileader, 20 leaders: 4.737007e-03 +Pairwise Locality Aware, 20 leaders: 2.309032e-03 +Pairwise Multileader Locality, 20 leaders: 4.523231e-03 +Nonblocking Multileader, 20 leaders: 3.369475e-03 +Nonblocking Locality Aware, 20 leaders: 1.982418e-03 +Nonblocking Multileader Locality, 20 leaders: 3.924728e-03 +Test 3 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 4.110724e-05 +Pairwise: 3.892607e-04 +NonBlocking: 2.004555e-04 +Pairwise Hierarchical: 1.839619e-04 +Pairwise Node Aware: 1.764327e-04 +Nonblocking Hierarchical: 1.872055e-04 +Nonblocking Node Aware: 1.823413e-04 +Pairwise Multileader, 4 leaders: 6.265580e-05 +Pairwise Locality Aware, 4 leaders: 7.442157e-05 +Pairwise Multileader Locality, 4 leaders: 5.897831e-05 +Nonblocking Multileader, 4 leaders: 4.758160e-05 +Nonblocking Locality Aware, 4 leaders: 4.782725e-05 +Nonblocking Multileader Locality, 4 leaders: 5.493733e-05 +Pairwise Multileader, 10 leaders: 7.901974e-05 +Pairwise Locality Aware, 10 leaders: 8.691754e-05 +Pairwise Multileader Locality, 10 leaders: 3.993197e-05 +Nonblocking Multileader, 10 leaders: 3.708191e-05 +Nonblocking Locality Aware, 10 leaders: 4.079952e-05 +Nonblocking Multileader Locality, 10 leaders: 3.691412e-05 +Pairwise Multileader, 20 leaders: 1.514163e-04 +Pairwise Locality Aware, 20 leaders: 1.485347e-04 +Pairwise Multileader Locality, 20 leaders: 5.553511e-05 +Nonblocking Multileader, 20 leaders: 5.477677e-05 +Nonblocking Locality Aware, 20 leaders: 4.950356e-05 +Nonblocking Multileader Locality, 20 leaders: 4.149386e-05 +Size 2 +PMPI: 4.274871e-05 +Pairwise: 3.601825e-04 +NonBlocking: 1.856997e-04 +Pairwise Hierarchical: 3.072591e-04 +Pairwise Node Aware: 1.779300e-04 +Nonblocking Hierarchical: 2.981339e-04 +Nonblocking Node Aware: 1.823336e-04 +Pairwise Multileader, 4 leaders: 7.681969e-05 +Pairwise Locality Aware, 4 leaders: 7.311377e-05 +Pairwise Multileader Locality, 4 leaders: 8.156704e-05 +Nonblocking Multileader, 4 leaders: 6.050142e-05 +Nonblocking Locality Aware, 4 leaders: 4.967683e-05 +Nonblocking Multileader Locality, 4 leaders: 7.694098e-05 +Pairwise Multileader, 10 leaders: 8.308692e-05 +Pairwise Locality Aware, 10 leaders: 8.847015e-05 +Pairwise Multileader Locality, 10 leaders: 5.026085e-05 +Nonblocking Multileader, 10 leaders: 4.180072e-05 +Nonblocking Locality Aware, 10 leaders: 4.324832e-05 +Nonblocking Multileader Locality, 10 leaders: 4.358151e-05 +Pairwise Multileader, 20 leaders: 1.515906e-04 +Pairwise Locality Aware, 20 leaders: 1.428102e-04 +Pairwise Multileader Locality, 20 leaders: 6.355577e-05 +Nonblocking Multileader, 20 leaders: 5.992443e-05 +Nonblocking Locality Aware, 20 leaders: 5.767124e-05 +Nonblocking Multileader Locality, 20 leaders: 4.638752e-05 +Size 4 +PMPI: 5.107420e-05 +Pairwise: 6.943830e-04 +NonBlocking: 4.185449e-03 +Pairwise Hierarchical: 4.253495e-04 +Pairwise Node Aware: 2.415826e-04 +Nonblocking Hierarchical: 4.358923e-04 +Nonblocking Node Aware: 1.854916e-04 +Pairwise Multileader, 4 leaders: 1.122224e-04 +Pairwise Locality Aware, 4 leaders: 8.139067e-05 +Pairwise Multileader Locality, 4 leaders: 1.146938e-04 +Nonblocking Multileader, 4 leaders: 9.136236e-05 +Nonblocking Locality Aware, 4 leaders: 5.154192e-05 +Nonblocking Multileader Locality, 4 leaders: 1.083203e-04 +Pairwise Multileader, 10 leaders: 9.352217e-05 +Pairwise Locality Aware, 10 leaders: 8.736797e-05 +Pairwise Multileader Locality, 10 leaders: 6.764945e-05 +Nonblocking Multileader, 10 leaders: 5.298289e-05 +Nonblocking Locality Aware, 10 leaders: 4.775729e-05 +Nonblocking Multileader Locality, 10 leaders: 5.989611e-05 +Pairwise Multileader, 20 leaders: 1.528071e-04 +Pairwise Locality Aware, 20 leaders: 1.535355e-04 +Pairwise Multileader Locality, 20 leaders: 7.954048e-05 +Nonblocking Multileader, 20 leaders: 6.466280e-05 +Nonblocking Locality Aware, 20 leaders: 6.909758e-05 +Nonblocking Multileader Locality, 20 leaders: 5.909710e-05 +Size 8 +PMPI: 7.292948e-05 +Pairwise: 6.787691e-04 +NonBlocking: 3.530354e-03 +Pairwise Hierarchical: 8.268877e-04 +Pairwise Node Aware: 2.594440e-04 +Nonblocking Hierarchical: 7.815404e-04 +Nonblocking Node Aware: 1.849810e-04 +Pairwise Multileader, 4 leaders: 2.225414e-04 +Pairwise Locality Aware, 4 leaders: 9.177321e-05 +Pairwise Multileader Locality, 4 leaders: 2.936598e-04 +Nonblocking Multileader, 4 leaders: 1.841194e-04 +Nonblocking Locality Aware, 4 leaders: 1.081938e-04 +Nonblocking Multileader Locality, 4 leaders: 2.511589e-04 +Pairwise Multileader, 10 leaders: 1.148810e-04 +Pairwise Locality Aware, 10 leaders: 9.577386e-05 +Pairwise Multileader Locality, 10 leaders: 1.010740e-04 +Nonblocking Multileader, 10 leaders: 6.972405e-05 +Nonblocking Locality Aware, 10 leaders: 5.403793e-05 +Nonblocking Multileader Locality, 10 leaders: 8.525934e-05 +Pairwise Multileader, 20 leaders: 1.682010e-04 +Pairwise Locality Aware, 20 leaders: 1.515692e-04 +Pairwise Multileader Locality, 20 leaders: 1.033310e-04 +Nonblocking Multileader, 20 leaders: 7.843541e-05 +Nonblocking Locality Aware, 20 leaders: 6.942216e-05 +Nonblocking Multileader Locality, 20 leaders: 7.840696e-05 +Size 16 +PMPI: 1.310534e-04 +Pairwise: 6.997451e-04 +NonBlocking: 2.358693e-04 +Pairwise Hierarchical: 1.563213e-03 +Pairwise Node Aware: 3.057717e-04 +Nonblocking Hierarchical: 1.495929e-03 +Nonblocking Node Aware: 1.884100e-04 +Pairwise Multileader, 4 leaders: 4.509827e-04 +Pairwise Locality Aware, 4 leaders: 1.201562e-04 +Pairwise Multileader Locality, 4 leaders: 5.492909e-04 +Nonblocking Multileader, 4 leaders: 3.627717e-04 +Nonblocking Locality Aware, 4 leaders: 9.478866e-05 +Nonblocking Multileader Locality, 4 leaders: 4.868481e-04 +Pairwise Multileader, 10 leaders: 2.299922e-04 +Pairwise Locality Aware, 10 leaders: 1.110729e-04 +Pairwise Multileader Locality, 10 leaders: 1.932211e-04 +Nonblocking Multileader, 10 leaders: 1.489134e-04 +Nonblocking Locality Aware, 10 leaders: 8.902285e-05 +Nonblocking Multileader Locality, 10 leaders: 1.651188e-04 +Pairwise Multileader, 20 leaders: 1.904702e-04 +Pairwise Locality Aware, 20 leaders: 1.668410e-04 +Pairwise Multileader Locality, 20 leaders: 1.522033e-04 +Nonblocking Multileader, 20 leaders: 1.009346e-04 +Nonblocking Locality Aware, 20 leaders: 9.999539e-05 +Nonblocking Multileader Locality, 20 leaders: 1.142666e-04 +Size 32 +PMPI: 2.361368e-04 +Pairwise: 6.933844e-04 +NonBlocking: 6.320318e-03 +Pairwise Hierarchical: 3.084779e-03 +Pairwise Node Aware: 4.461541e-04 +Nonblocking Hierarchical: 2.953750e-03 +Nonblocking Node Aware: 2.345150e-04 +Pairwise Multileader, 4 leaders: 1.002698e-03 +Pairwise Locality Aware, 4 leaders: 1.922997e-04 +Pairwise Multileader Locality, 4 leaders: 1.049272e-03 +Nonblocking Multileader, 4 leaders: 8.669549e-04 +Nonblocking Locality Aware, 4 leaders: 1.713482e-04 +Nonblocking Multileader Locality, 4 leaders: 9.285816e-04 +Pairwise Multileader, 10 leaders: 4.257471e-04 +Pairwise Locality Aware, 10 leaders: 1.714975e-04 +Pairwise Multileader Locality, 10 leaders: 4.313325e-04 +Nonblocking Multileader, 10 leaders: 3.114409e-04 +Nonblocking Locality Aware, 10 leaders: 2.190280e-04 +Nonblocking Multileader Locality, 10 leaders: 3.964712e-04 +Pairwise Multileader, 20 leaders: 2.984280e-04 +Pairwise Locality Aware, 20 leaders: 1.981535e-04 +Pairwise Multileader Locality, 20 leaders: 2.503554e-04 +Nonblocking Multileader, 20 leaders: 1.819311e-04 +Nonblocking Locality Aware, 20 leaders: 3.186776e-04 +Nonblocking Multileader Locality, 20 leaders: 1.982940e-04 +Size 64 +PMPI: 4.352041e-04 +Pairwise: 7.203781e-04 +NonBlocking: 3.129724e-02 +Pairwise Hierarchical: 6.972025e-03 +Pairwise Node Aware: 4.738751e-04 +Nonblocking Hierarchical: 6.650955e-03 +Nonblocking Node Aware: 2.852650e-04 +Pairwise Multileader, 4 leaders: 2.407001e-03 +Pairwise Locality Aware, 4 leaders: 3.141145e-04 +Pairwise Multileader Locality, 4 leaders: 2.413632e-03 +Nonblocking Multileader, 4 leaders: 1.957454e-03 +Nonblocking Locality Aware, 4 leaders: 2.458097e-04 +Nonblocking Multileader Locality, 4 leaders: 2.246852e-03 +Pairwise Multileader, 10 leaders: 9.039887e-04 +Pairwise Locality Aware, 10 leaders: 3.087322e-04 +Pairwise Multileader Locality, 10 leaders: 9.404346e-04 +Nonblocking Multileader, 10 leaders: 6.990307e-04 +Nonblocking Locality Aware, 10 leaders: 2.717510e-04 +Nonblocking Multileader Locality, 10 leaders: 8.511331e-04 +Pairwise Multileader, 20 leaders: 5.553695e-04 +Pairwise Locality Aware, 20 leaders: 3.336031e-04 +Pairwise Multileader Locality, 20 leaders: 5.313058e-04 +Nonblocking Multileader, 20 leaders: 3.476932e-04 +Nonblocking Locality Aware, 20 leaders: 3.075257e-04 +Nonblocking Multileader Locality, 20 leaders: 4.482298e-04 +Size 128 +PMPI: 8.197389e-04 +Pairwise: 8.459391e-04 +NonBlocking: 1.344997e-02 +Pairwise Hierarchical: 1.378345e-02 +Pairwise Node Aware: 7.774066e-04 +Nonblocking Hierarchical: 1.330803e-02 +Nonblocking Node Aware: 4.938063e-04 +Pairwise Multileader, 4 leaders: 4.520775e-03 +Pairwise Locality Aware, 4 leaders: 6.171380e-04 +Pairwise Multileader Locality, 4 leaders: 4.804889e-03 +Nonblocking Multileader, 4 leaders: 3.784302e-03 +Nonblocking Locality Aware, 4 leaders: 5.693049e-04 +Nonblocking Multileader Locality, 4 leaders: 4.359447e-03 +Pairwise Multileader, 10 leaders: 1.811039e-03 +Pairwise Locality Aware, 10 leaders: 6.367039e-04 +Pairwise Multileader Locality, 10 leaders: 1.892462e-03 +Nonblocking Multileader, 10 leaders: 1.446406e-03 +Nonblocking Locality Aware, 10 leaders: 5.320059e-04 +Nonblocking Multileader Locality, 10 leaders: 1.775142e-03 +Pairwise Multileader, 20 leaders: 1.096644e-03 +Pairwise Locality Aware, 20 leaders: 5.607782e-04 +Pairwise Multileader Locality, 20 leaders: 1.121894e-03 +Nonblocking Multileader, 20 leaders: 7.731085e-04 +Nonblocking Locality Aware, 20 leaders: 4.593659e-04 +Nonblocking Multileader Locality, 20 leaders: 1.058954e-03 +Size 256 +PMPI: 1.597249e-03 +Pairwise: 1.207656e-03 +NonBlocking: 5.005802e-02 +Pairwise Hierarchical: 2.861269e-02 +Pairwise Node Aware: 1.262832e-03 +Nonblocking Hierarchical: 2.859794e-02 +Nonblocking Node Aware: 9.369269e-04 +Pairwise Multileader, 4 leaders: 7.967625e-03 +Pairwise Locality Aware, 4 leaders: 1.023068e-03 +Pairwise Multileader Locality, 4 leaders: 8.409800e-03 +Nonblocking Multileader, 4 leaders: 6.720649e-03 +Nonblocking Locality Aware, 4 leaders: 8.320064e-04 +Nonblocking Multileader Locality, 4 leaders: 7.853326e-03 +Pairwise Multileader, 10 leaders: 4.395178e-03 +Pairwise Locality Aware, 10 leaders: 1.084980e-03 +Pairwise Multileader Locality, 10 leaders: 3.731113e-03 +Nonblocking Multileader, 10 leaders: 3.242608e-03 +Nonblocking Locality Aware, 10 leaders: 1.069304e-03 +Nonblocking Multileader Locality, 10 leaders: 3.549180e-03 +Pairwise Multileader, 20 leaders: 2.276947e-03 +Pairwise Locality Aware, 20 leaders: 1.205025e-03 +Pairwise Multileader Locality, 20 leaders: 2.244881e-03 +Nonblocking Multileader, 20 leaders: 1.621858e-03 +Nonblocking Locality Aware, 20 leaders: 1.115687e-03 +Nonblocking Multileader Locality, 20 leaders: 1.975512e-03 +Size 512 +PMPI: 1.998725e-03 +Pairwise: 2.090363e-03 +NonBlocking: 2.304175e-03 +Pairwise Hierarchical: 5.754473e-02 +Pairwise Node Aware: 2.305499e-03 +Nonblocking Hierarchical: 5.947567e-02 +Nonblocking Node Aware: 1.775080e-03 +Pairwise Multileader, 4 leaders: 1.757918e-02 +Pairwise Locality Aware, 4 leaders: 2.155444e-03 +Pairwise Multileader Locality, 4 leaders: 1.703654e-02 +Nonblocking Multileader, 4 leaders: 1.400620e-02 +Nonblocking Locality Aware, 4 leaders: 1.951569e-03 +Nonblocking Multileader Locality, 4 leaders: 1.568634e-02 +Pairwise Multileader, 10 leaders: 9.349479e-03 +Pairwise Locality Aware, 10 leaders: 2.135996e-03 +Pairwise Multileader Locality, 10 leaders: 8.208732e-03 +Nonblocking Multileader, 10 leaders: 7.005656e-03 +Nonblocking Locality Aware, 10 leaders: 1.878527e-03 +Nonblocking Multileader Locality, 10 leaders: 7.862028e-03 +Pairwise Multileader, 20 leaders: 4.677348e-03 +Pairwise Locality Aware, 20 leaders: 2.115716e-03 +Pairwise Multileader Locality, 20 leaders: 4.393525e-03 +Nonblocking Multileader, 20 leaders: 3.300636e-03 +Nonblocking Locality Aware, 20 leaders: 1.847838e-03 +Nonblocking Multileader Locality, 20 leaders: 3.912086e-03 diff --git a/runscripts/dane_results/alltoall_N32.2373829.err b/runscripts/dane_results/alltoall_N32.2373829.err new file mode 100644 index 000000000..12f25f22c --- /dev/null +++ b/runscripts/dane_results/alltoall_N32.2373829.err @@ -0,0 +1,795 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fbdb6010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fbdb6010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa907010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa907010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2373829.0 ON dane302 CANCELLED AT 2025-04-29T18:51:16 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1212: task 2203: Killed +srun: error: dane1210: task 2103: Killed +srun: error: dane1197: task 1903: Killed +srun: error: dane888: task 1405: Killed +srun: error: dane877: task 1305: Killed +srun: error: dane551: tasks 505,509: Killed +srun: error: dane1356: tasks 3005,3009,3013: Killed +srun: error: dane1134: tasks 1505,1597: Killed +srun: error: dane852: task 1105: Killed +srun: error: dane324: tasks 205,209: Killed +srun: error: dane563: tasks 605,609,613: Killed +srun: error: dane494: tasks 403,405,409,413,417: Killed +srun: error: dane827: tasks 805,809: Killed +srun: error: dane1261: tasks 2500-2599: Killed +srun: error: dane1167: tasks 1700-1799: Killed +srun: error: dane1210: tasks 2100-2102,2104-2199: Killed +srun: error: dane1200: tasks 2000-2099: Killed +srun: error: dane1224: tasks 2300-2399: Killed +srun: error: dane1300: tasks 2700-2799: Killed +srun: error: dane1425: tasks 3100-3199: Killed +srun: error: dane1356: tasks 3000-3004,3006-3008,3010-3012,3014-3099: Killed +srun: error: dane877: tasks 1300-1304,1306-1399: Killed +srun: error: dane494: tasks 400-402,404,406-408,410-412,414-416,418-499: Killed +srun: error: dane563: tasks 600-604,606-608,610-612,614-699: Killed +srun: error: dane888: tasks 1400-1404,1406-1499: Killed +srun: error: dane1286: tasks 2600-2699: Killed +srun: error: dane551: tasks 500-504,506-508,510-599: Killed +srun: error: dane846: tasks 1000-1099: Killed +srun: error: dane1134: tasks 1500-1504,1506-1596,1598-1599: Killed +srun: error: dane1212: tasks 2200-2202,2204-2299: Killed +srun: error: dane1315: tasks 2900-2999: Killed +srun: error: dane1314: tasks 2800-2899: Killed +srun: error: dane324: tasks 200-204,206-208,210-299: Killed +srun: error: dane1241: tasks 2400-2499: Killed +srun: error: dane1163: tasks 1600-1699: Killed +srun: error: dane852: tasks 1100-1104,1106-1199: Killed +srun: error: dane827: tasks 800-804,806-808,810-899: Killed +srun: error: dane1197: tasks 1900-1902,1904-1999: Killed +srun: error: dane579: tasks 700-799: Killed +srun: error: dane1189: tasks 1800-1899: Killed +srun: error: dane433: tasks 300-399: Killed +srun: error: dane302: tasks 0-99: Killed +srun: error: dane856: tasks 1200-1299: Killed +srun: error: dane845: tasks 900-999: Killed +srun: error: dane308: tasks 100-199: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa907010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa907010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fbdb6010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fbdb6010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2373829.1 ON dane302 CANCELLED AT 2025-04-29T19:00:57 *** +slurmstepd: error: *** STEP 2373829.1 ON dane302 CANCELLED AT 2025-04-29T19:00:57 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1261: tasks 2500-2599: Killed +srun: error: dane1167: tasks 1700-1799: Killed +srun: error: dane1210: tasks 2100-2199: Killed +srun: error: dane1200: tasks 2000-2099: Killed +srun: error: dane1224: tasks 2300-2399: Killed +srun: error: dane1300: tasks 2700-2799: Killed +srun: error: dane1356: tasks 3000-3099: Killed +srun: error: dane1425: tasks 3100-3199: Killed +srun: error: dane877: tasks 1300-1399: Killed +srun: error: dane563: tasks 600-699: Killed +srun: error: dane551: tasks 500-599: Killed +srun: error: dane888: tasks 1400-1499: Killed +srun: error: dane1286: tasks 2600-2699: Killed +srun: error: dane494: tasks 400-499: Killed +srun: error: dane846: tasks 1000-1099: Killed +srun: error: dane1212: tasks 2200-2299: Killed +srun: error: dane1134: tasks 1500-1599: Killed +srun: error: dane1315: tasks 2900-2999: Killed +srun: error: dane1314: tasks 2800-2899: Killed +srun: error: dane324: tasks 200-299: Killed +srun: error: dane1241: tasks 2400-2499: Killed +srun: error: dane1163: tasks 1600-1699: Killed +srun: error: dane852: tasks 1100-1199: Killed +srun: error: dane1197: tasks 1900-1999: Killed +srun: error: dane827: tasks 800-899: Killed +srun: error: dane579: tasks 700-799: Killed +srun: error: dane1189: tasks 1800-1899: Killed +srun: error: dane433: tasks 300-399: Killed +srun: error: dane302: tasks 0-99: Killed +srun: error: dane845: tasks 900-999: Killed +srun: error: dane308: tasks 100-199: Killed +srun: error: dane856: tasks 1200-1299: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fbdb6010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fbdb6010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa907010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa907010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2373829.2 ON dane302 CANCELLED AT 2025-04-29T19:10:28 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane852: task 1103: Killed +srun: error: dane846: task 1003: Killed +srun: error: dane563: task 617: Killed +srun: error: dane1210: task 2105: Killed +srun: error: dane1134: tasks 1509,1513,1517: Killed +srun: error: dane1197: tasks 1905,1909: Killed +srun: error: dane888: tasks 1405,1409,1413,1417,1421: Killed +srun: error: dane1261: tasks 2500-2599: Killed +srun: error: dane1210: tasks 2100-2104,2106-2199: Killed +srun: error: dane1224: tasks 2300-2399: Killed +srun: error: dane1167: tasks 1700-1799: Killed +srun: error: dane1300: tasks 2700-2799: Killed +srun: error: dane1425: tasks 3100-3199: Killed +srun: error: dane1356: tasks 3000-3099: Killed +srun: error: dane1200: tasks 2000-2099: Killed +srun: error: dane877: tasks 1300-1399: Killed +srun: error: dane494: tasks 400-499: Killed +srun: error: dane563: tasks 600-616,618-699: Killed +srun: error: dane1286: tasks 2600-2699: Killed +srun: error: dane551: tasks 500-599: Killed +srun: error: dane888: tasks 1400-1404,1406-1408,1410-1412,1414-1416,1418-1420,1422-1499: Killed +srun: error: dane846: tasks 1000-1002,1004-1099: Killed +srun: error: dane1212: tasks 2200-2299: Killed +srun: error: dane1134: tasks 1500-1508,1510-1512,1514-1516,1518-1599: Killed +srun: error: dane1315: tasks 2900-2999: Killed +srun: error: dane1314: tasks 2800-2899: Killed +srun: error: dane324: tasks 200-299: Killed +srun: error: dane1241: tasks 2400-2499: Killed +srun: error: dane1163: tasks 1600-1699: Killed +srun: error: dane852: tasks 1100-1102,1104-1199: Killed +srun: error: dane1197: tasks 1900-1904,1906-1908,1910-1999: Killed +srun: error: dane827: tasks 800-899: Killed +srun: error: dane1189: tasks 1800-1899: Killed +srun: error: dane579: tasks 700-799: Killed +srun: error: dane433: tasks 300-399: Killed +srun: error: dane302: tasks 0-99: Killed +srun: error: dane845: tasks 900-999: Killed +srun: error: dane856: tasks 1200-1299: Killed +srun: error: dane308: tasks 100-199: Killed diff --git a/runscripts/dane_results/alltoall_N32.2373829.out b/runscripts/dane_results/alltoall_N32.2373829.out new file mode 100644 index 000000000..13f990670 --- /dev/null +++ b/runscripts/dane_results/alltoall_N32.2373829.out @@ -0,0 +1,873 @@ +Test 1 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 9.987141e-04 +Pairwise: 1.477677e-02 +NonBlocking: 4.872264e-01 +Pairwise Hierarchical: 2.366412e-03 +Pairwise Node Aware: 6.120289e-04 +Nonblocking Hierarchical: 1.986900e-03 +Nonblocking Node Aware: 4.663542e-04 +Pairwise Multileader, 4 leaders: 1.812443e-03 +Pairwise Locality Aware, 4 leaders: 1.227157e-03 +Pairwise Multileader Locality, 4 leaders: 1.226778e-03 +Nonblocking Multileader, 4 leaders: 1.018285e-03 +Nonblocking Locality Aware, 4 leaders: 9.188083e-02 +Nonblocking Multileader Locality, 4 leaders: 6.603081e-04 +Pairwise Multileader, 10 leaders: 2.159945e-03 +Pairwise Locality Aware, 10 leaders: 3.434986e-03 +Pairwise Multileader Locality, 10 leaders: 4.965427e-04 +Nonblocking Multileader, 10 leaders: 2.327243e-03 +Nonblocking Locality Aware, 10 leaders: 9.902323e-02 +Nonblocking Multileader Locality, 10 leaders: 3.426347e-04 +Pairwise Multileader, 20 leaders: 6.759927e-03 +Pairwise Locality Aware, 20 leaders: 8.119227e-03 +Pairwise Multileader Locality, 20 leaders: 4.236292e-04 +Nonblocking Multileader, 20 leaders: 6.064480e-03 +Nonblocking Locality Aware, 20 leaders: 1.380658e-01 +Nonblocking Multileader Locality, 20 leaders: 2.379395e-04 +Size 2 +PMPI: 1.142909e-03 +Pairwise: 1.440055e-02 +NonBlocking: 4.236275e-01 +Pairwise Hierarchical: 5.259816e-03 +Pairwise Node Aware: 6.421791e-04 +Nonblocking Hierarchical: 4.314721e-03 +Nonblocking Node Aware: 5.918300e-04 +Pairwise Multileader, 4 leaders: 2.006367e-03 +Pairwise Locality Aware, 4 leaders: 1.879777e-03 +Pairwise Multileader Locality, 4 leaders: 2.654569e-03 +Nonblocking Multileader, 4 leaders: 1.110144e-03 +Nonblocking Locality Aware, 4 leaders: 7.438038e-02 +Nonblocking Multileader Locality, 4 leaders: 1.305433e-03 +Pairwise Multileader, 10 leaders: 3.948605e-03 +Pairwise Locality Aware, 10 leaders: 2.690176e-03 +Pairwise Multileader Locality, 10 leaders: 7.645219e-04 +Nonblocking Multileader, 10 leaders: 2.801697e-03 +Nonblocking Locality Aware, 10 leaders: 1.150562e-01 +Nonblocking Multileader Locality, 10 leaders: 5.677414e-04 +Pairwise Multileader, 20 leaders: 5.183516e-03 +Pairwise Locality Aware, 20 leaders: 3.517382e-03 +Pairwise Multileader Locality, 20 leaders: 5.639062e-04 +Nonblocking Multileader, 20 leaders: 1.306901e-02 +Nonblocking Locality Aware, 20 leaders: 9.690199e-02 +Nonblocking Multileader Locality, 20 leaders: 3.576226e-04 +Size 4 +PMPI: 1.843498e-03 +Pairwise: 2.230887e-02 +NonBlocking: 4.466466e-01 +Pairwise Hierarchical: 1.022003e-02 +Pairwise Node Aware: 8.585643e-04 +Nonblocking Hierarchical: 8.395864e-03 +Nonblocking Node Aware: 7.863918e-04 +Pairwise Multileader, 4 leaders: 4.075348e-03 +Pairwise Locality Aware, 4 leaders: 1.640331e-03 +Pairwise Multileader Locality, 4 leaders: 3.001545e-03 +Nonblocking Multileader, 4 leaders: 2.146977e-03 +Nonblocking Locality Aware, 4 leaders: 2.509201e-02 +Nonblocking Multileader Locality, 4 leaders: 2.408625e-03 +Pairwise Multileader, 10 leaders: 3.678591e-03 +Pairwise Locality Aware, 10 leaders: 3.374105e-03 +Pairwise Multileader Locality, 10 leaders: 1.449589e-03 +Nonblocking Multileader, 10 leaders: 3.120023e-02 +Nonblocking Locality Aware, 10 leaders: 1.116698e-01 +Nonblocking Multileader Locality, 10 leaders: 9.215596e-04 +Pairwise Multileader, 20 leaders: 7.159295e-03 +Pairwise Locality Aware, 20 leaders: 7.178254e-03 +Pairwise Multileader Locality, 20 leaders: 8.185905e-04 +Nonblocking Multileader, 20 leaders: 1.098725e-03 +Nonblocking Locality Aware, 20 leaders: 1.156656e-01 +Nonblocking Multileader Locality, 20 leaders: 6.416609e-04 +Size 8 +PMPI: 3.566116e-03 +Pairwise: 2.628536e-02 +NonBlocking: 3.270516e-01 +Pairwise Hierarchical: 2.183310e-02 +Pairwise Node Aware: 1.368907e-03 +Nonblocking Hierarchical: 1.767550e-02 +Nonblocking Node Aware: 1.395921e-03 +Pairwise Multileader, 4 leaders: 5.959153e-03 +Pairwise Locality Aware, 4 leaders: 1.776112e-03 +Pairwise Multileader Locality, 4 leaders: 6.641314e-03 +Nonblocking Multileader, 4 leaders: 3.960893e-03 +Nonblocking Locality Aware, 4 leaders: 9.274251e-02 +Nonblocking Multileader Locality, 4 leaders: 5.050858e-03 +Pairwise Multileader, 10 leaders: 3.758400e-03 +Pairwise Locality Aware, 10 leaders: 3.872152e-03 +Pairwise Multileader Locality, 10 leaders: 2.478667e-03 +Nonblocking Multileader, 10 leaders: 5.279327e-03 +Nonblocking Locality Aware, 10 leaders: 1.157435e-01 +Nonblocking Multileader Locality, 10 leaders: 2.036285e-03 +Pairwise Multileader, 20 leaders: 6.930623e-03 +Pairwise Locality Aware, 20 leaders: 5.273743e-03 +Pairwise Multileader Locality, 20 leaders: 1.512375e-03 +Nonblocking Multileader, 20 leaders: 1.239836e-01 +Nonblocking Locality Aware, 20 leaders: 1.158479e-01 +Nonblocking Multileader Locality, 20 leaders: 1.390251e-03 +Size 16 +PMPI: 6.151751e-03 +Pairwise: 3.246787e-02 +NonBlocking: 3.282200e-01 +Pairwise Hierarchical: 4.539837e-02 +Pairwise Node Aware: 2.635764e-03 +Nonblocking Hierarchical: 3.528764e-02 +Nonblocking Node Aware: 2.551442e-03 +Pairwise Multileader, 4 leaders: 8.768510e-03 +Pairwise Locality Aware, 4 leaders: 2.878690e-03 +Pairwise Multileader Locality, 4 leaders: 1.148276e-02 +Nonblocking Multileader, 4 leaders: 6.838871e-03 +Nonblocking Locality Aware, 4 leaders: 7.557662e-02 +Nonblocking Multileader Locality, 4 leaders: 8.682683e-03 +Pairwise Multileader, 10 leaders: 7.876830e-03 +Pairwise Locality Aware, 10 leaders: 4.917157e-03 +Pairwise Multileader Locality, 10 leaders: 4.764210e-03 +Nonblocking Multileader, 10 leaders: 1.685303e-02 +Nonblocking Locality Aware, 10 leaders: 1.391731e-01 +Nonblocking Multileader Locality, 10 leaders: 3.718422e-03 +Pairwise Multileader, 20 leaders: 1.037964e-02 +Pairwise Locality Aware, 20 leaders: 1.058927e-02 +Pairwise Multileader Locality, 20 leaders: 3.044430e-03 +Nonblocking Multileader, 20 leaders: 8.039399e-02 +Nonblocking Locality Aware, 20 leaders: 1.164363e-01 +Nonblocking Multileader Locality, 20 leaders: 2.324643e-03 +Size 32 +PMPI: 1.277708e-02 +Pairwise: 2.165330e-02 +NonBlocking: 3.705901e-01 +Pairwise Hierarchical: 8.635325e-02 +Pairwise Node Aware: 4.299558e-03 +Nonblocking Hierarchical: 6.595182e-02 +Nonblocking Node Aware: 4.880632e-03 +Pairwise Multileader, 4 leaders: 2.762798e-02 +Pairwise Locality Aware, 4 leaders: 4.448120e-03 +Pairwise Multileader Locality, 4 leaders: 2.371811e-02 +Nonblocking Multileader, 4 leaders: 1.815706e-02 +Nonblocking Locality Aware, 4 leaders: 1.630374e-02 +Nonblocking Multileader Locality, 4 leaders: 1.813174e-02 +Pairwise Multileader, 10 leaders: 1.289372e-02 +Pairwise Locality Aware, 10 leaders: 5.688085e-03 +Pairwise Multileader Locality, 10 leaders: 1.081982e-02 +Nonblocking Multileader, 10 leaders: 1.172251e-02 +Nonblocking Locality Aware, 10 leaders: 1.183645e-01 +Nonblocking Multileader Locality, 10 leaders: 8.597408e-03 +Pairwise Multileader, 20 leaders: 1.081426e-02 +Pairwise Locality Aware, 20 leaders: 7.583293e-03 +Pairwise Multileader Locality, 20 leaders: 5.593592e-03 +Nonblocking Multileader, 20 leaders: 7.067172e-02 +Nonblocking Locality Aware, 20 leaders: 1.229659e-01 +Nonblocking Multileader Locality, 20 leaders: 4.898342e-03 +Size 64 +PMPI: 2.210544e-02 +Pairwise: 3.003343e-02 +NonBlocking: 2.248981e-01 +Pairwise Hierarchical: 1.661955e-01 +Pairwise Node Aware: 8.826996e-03 +Nonblocking Hierarchical: 1.266732e-01 +Nonblocking Node Aware: 7.519022e-03 +Pairwise Multileader, 4 leaders: 5.292815e-02 +Pairwise Locality Aware, 4 leaders: 8.288288e-03 +Pairwise Multileader Locality, 4 leaders: 5.723548e-02 +Nonblocking Multileader, 4 leaders: 3.784666e-02 +Nonblocking Locality Aware, 4 leaders: 1.042274e-02 +Nonblocking Multileader Locality, 4 leaders: 4.268004e-02 +Pairwise Multileader, 10 leaders: 2.569062e-02 +Pairwise Locality Aware, 10 leaders: 1.017345e-02 +Pairwise Multileader Locality, 10 leaders: 2.515463e-02 +Nonblocking Multileader, 10 leaders: 1.796538e-02 +Nonblocking Locality Aware, 10 leaders: 1.197047e-01 +Nonblocking Multileader Locality, 10 leaders: 1.992824e-02 +Pairwise Multileader, 20 leaders: 1.780547e-02 +Pairwise Locality Aware, 20 leaders: 1.183611e-02 +Pairwise Multileader Locality, 20 leaders: 1.233130e-02 +Nonblocking Multileader, 20 leaders: 8.814880e-03 +Nonblocking Locality Aware, 20 leaders: 1.195875e-01 +Nonblocking Multileader Locality, 20 leaders: 9.524260e-03 +Size 128 +PMPI: 4.301755e-02 +Pairwise: 3.715509e-02 +NonBlocking: 2.783748e-01 +Pairwise Hierarchical: 3.226321e-01 +Pairwise Node Aware: 1.995016e-02 +Nonblocking Hierarchical: 2.507497e-01 +Nonblocking Node Aware: 1.745272e-02 +Pairwise Multileader, 4 leaders: 9.763577e-02 +Pairwise Locality Aware, 4 leaders: 1.834186e-02 +Pairwise Multileader Locality, 4 leaders: 1.082115e-01 +Nonblocking Multileader, 4 leaders: 6.930908e-02 +Nonblocking Locality Aware, 4 leaders: 2.133106e-02 +Nonblocking Multileader Locality, 4 leaders: 7.962463e-02 +Pairwise Multileader, 10 leaders: 5.317948e-02 +Pairwise Locality Aware, 10 leaders: 2.045656e-02 +Pairwise Multileader Locality, 10 leaders: 5.339877e-02 +Nonblocking Multileader, 10 leaders: 7.722007e-02 +Nonblocking Locality Aware, 10 leaders: 1.224798e-01 +Nonblocking Multileader Locality, 10 leaders: 4.157351e-02 +Pairwise Multileader, 20 leaders: 3.801920e-02 +Pairwise Locality Aware, 20 leaders: 2.273059e-02 +Pairwise Multileader Locality, 20 leaders: 2.755981e-02 +Nonblocking Multileader, 20 leaders: 2.309638e-02 +Nonblocking Locality Aware, 20 leaders: 1.484382e-01 +Nonblocking Multileader Locality, 20 leaders: 2.085439e-02 +Size 256 +PMPI: 9.423167e-02 +Pairwise: 6.499840e-02 +NonBlocking: 2.879829e-01 +Pairwise Hierarchical: 6.492646e-01 +Pairwise Node Aware: 3.433067e-02 +Nonblocking Hierarchical: 5.413325e-01 +Nonblocking Node Aware: 3.043192e-02 +Pairwise Multileader, 4 leaders: 1.852159e-01 +Pairwise Locality Aware, 4 leaders: 4.147817e-02 +Pairwise Multileader Locality, 4 leaders: 2.074987e-01 +Nonblocking Multileader, 4 leaders: 1.319308e-01 +Nonblocking Locality Aware, 4 leaders: 3.231109e-02 +Nonblocking Multileader Locality, 4 leaders: 1.566307e-01 +Pairwise Multileader, 10 leaders: 1.180939e-01 +Pairwise Locality Aware, 10 leaders: 4.179147e-02 +Pairwise Multileader Locality, 10 leaders: 1.093720e-01 +Nonblocking Multileader, 10 leaders: 7.708731e-02 +Nonblocking Locality Aware, 10 leaders: 6.721100e-02 +Nonblocking Multileader Locality, 10 leaders: 8.884488e-02 +Pairwise Multileader, 20 leaders: 7.676414e-02 +Pairwise Locality Aware, 20 leaders: 4.603701e-02 +Pairwise Multileader Locality, 20 leaders: 5.802523e-02 +Nonblocking Multileader, 20 leaders: 4.011528e-02 +Nonblocking Locality Aware, 20 leaders: 1.300712e-01 +Nonblocking Multileader Locality, 20 leaders: 4.399672e-02 +Size 512 +PMPI: 4.108565e-01 +Pairwise: 1.022325e-01 +NonBlocking: 2.974673e-01 +Pairwise Hierarchical: 1.270482e+00 +Pairwise Node Aware: 7.036144e-02 +Nonblocking Hierarchical: 1.153354e+00 +Nonblocking Node Aware: 5.164266e-02 +Pairwise Multileader, 4 leaders: 3.459961e-01 +Pairwise Locality Aware, 4 leaders: 7.695385e-02 +Pairwise Multileader Locality, 4 leaders: 4.029377e-01 +Nonblocking Multileader, 4 leaders: 2.577038e-01 +Nonblocking Locality Aware, 4 leaders: 6.362557e-02 +Nonblocking Multileader Locality, 4 leaders: 3.068834e-01 +Pairwise Multileader, 10 leaders: 2.143051e-01 +Pairwise Locality Aware, 10 leaders: 8.907860e-02 +Pairwise Multileader Locality, 10 leaders: 2.149401e-01 +Nonblocking Multileader, 10 leaders: 1.468131e-01 +Nonblocking Locality Aware, 10 leaders: 6.416818e-02 +Nonblocking Multileader Locality, 10 leaders: 1.749836e-01 +Pairwise Multileader, 20 leaders: 1.554995e-01 +Pairwise Locality Aware, 20 leaders: 1.015760e-01 +Pairwise Multileader Locality, 20 leaders: 1.285901e-01 +Nonblocking Multileader, 20 leaders: 3.992841e-01 +Nonblocking Locality Aware, 20 leaders: 1.539006e-01 +Nonblocking Multileader Locality, 20 leaders: 1.078586e-01 +Size 1024 +PMPI: 1.814925e-01 +Pairwise: 1.792387e-01 +NonBlocking: 3.896955e-01 +Pairwise Hierarchical: 2.544602e+00 +Pairwise Node Aware: 1.486816e-01 +Nonblocking Hierarchical: 2.231158e+00 +Nonblocking Node Aware: 1.109409e-01 +Pairwise Multileader, 4 leaders: 7.062261e-01 +Pairwise Locality Aware, 4 leaders: 1.714898e-01 +Pairwise Multileader Locality, 4 leaders: 7.913553e-01 +Nonblocking Multileader, 4 leaders: 5.664537e-01 +Nonblocking Locality Aware, 4 leaders: 1.022692e-01 +Nonblocking Multileader Locality, 4 leaders: 6.572106e-01 +Pairwise Multileader, 10 leaders: 4.297853e-01 +Pairwise Locality Aware, 10 leaders: 1.817294e-01 +Pairwise Multileader Locality, 10 leaders: 4.480381e-01 +Nonblocking Multileader, 10 leaders: 2.894198e-01 +Nonblocking Locality Aware, 10 leaders: 2.097811e-01 +Nonblocking Multileader Locality, 10 leaders: 3.461892e-01 +Pairwise Multileader, 20 leaders: 3.152582e-01 +Pairwise Locality Aware, 20 leaders: 2.073156e-01 +Pairwise Multileader Locality, 20 leaders: 2.510474e-01 +Nonblocking Multileader, 20 leaders: 1.774195e-01 +Nonblocking Locality Aware, 20 leaders: 1.676270e-01 +Nonblocking Multileader Locality, 20 leaders: 2.134141e-01 +Size 2048 +PMPI: 3.884969e-01 +Pairwise: 3.801809e-01 +NonBlocking: 6.544173e-01 +Test 2 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.080722e-03 +Pairwise: 1.035897e-02 +NonBlocking: 1.695020e-01 +Pairwise Hierarchical: 2.401990e-03 +Pairwise Node Aware: 6.811216e-04 +Nonblocking Hierarchical: 2.012671e-03 +Nonblocking Node Aware: 4.092257e-04 +Pairwise Multileader, 4 leaders: 2.197859e-03 +Pairwise Locality Aware, 4 leaders: 1.296797e-03 +Pairwise Multileader Locality, 4 leaders: 9.141573e-04 +Nonblocking Multileader, 4 leaders: 7.031550e-04 +Nonblocking Locality Aware, 4 leaders: 4.956952e-02 +Nonblocking Multileader Locality, 4 leaders: 6.577428e-04 +Pairwise Multileader, 10 leaders: 2.769332e-03 +Pairwise Locality Aware, 10 leaders: 2.429917e-03 +Pairwise Multileader Locality, 10 leaders: 5.182430e-04 +Nonblocking Multileader, 10 leaders: 2.457708e-03 +Nonblocking Locality Aware, 10 leaders: 1.150112e-01 +Nonblocking Multileader Locality, 10 leaders: 3.450359e-04 +Pairwise Multileader, 20 leaders: 3.798003e-03 +Pairwise Locality Aware, 20 leaders: 6.247639e-03 +Pairwise Multileader Locality, 20 leaders: 4.088278e-04 +Nonblocking Multileader, 20 leaders: 1.097731e-02 +Nonblocking Locality Aware, 20 leaders: 1.380943e-01 +Nonblocking Multileader Locality, 20 leaders: 2.628751e-04 +Size 2 +PMPI: 1.132873e-03 +Pairwise: 1.352897e-02 +NonBlocking: 2.465791e-01 +Pairwise Hierarchical: 5.353326e-03 +Pairwise Node Aware: 6.162248e-04 +Nonblocking Hierarchical: 4.416348e-03 +Nonblocking Node Aware: 5.639969e-04 +Pairwise Multileader, 4 leaders: 2.189947e-03 +Pairwise Locality Aware, 4 leaders: 1.418689e-03 +Pairwise Multileader Locality, 4 leaders: 1.681969e-03 +Nonblocking Multileader, 4 leaders: 1.134756e-03 +Nonblocking Locality Aware, 4 leaders: 8.671572e-02 +Nonblocking Multileader Locality, 4 leaders: 1.321067e-03 +Pairwise Multileader, 10 leaders: 2.571176e-03 +Pairwise Locality Aware, 10 leaders: 3.037562e-03 +Pairwise Multileader Locality, 10 leaders: 8.328870e-04 +Nonblocking Multileader, 10 leaders: 8.228858e-03 +Nonblocking Locality Aware, 10 leaders: 1.151668e-01 +Nonblocking Multileader Locality, 10 leaders: 5.415159e-04 +Pairwise Multileader, 20 leaders: 5.295303e-03 +Pairwise Locality Aware, 20 leaders: 4.420235e-03 +Pairwise Multileader Locality, 20 leaders: 6.596498e-04 +Nonblocking Multileader, 20 leaders: 5.558625e-03 +Nonblocking Locality Aware, 20 leaders: 1.837443e-01 +Nonblocking Multileader Locality, 20 leaders: 4.244055e-04 +Size 4 +PMPI: 1.912373e-03 +Pairwise: 3.381021e-02 +NonBlocking: 3.529121e-01 +Pairwise Hierarchical: 1.038070e-02 +Pairwise Node Aware: 1.150020e-03 +Nonblocking Hierarchical: 8.457626e-03 +Nonblocking Node Aware: 8.183682e-04 +Pairwise Multileader, 4 leaders: 3.280544e-03 +Pairwise Locality Aware, 4 leaders: 1.429882e-03 +Pairwise Multileader Locality, 4 leaders: 2.999509e-03 +Nonblocking Multileader, 4 leaders: 2.192171e-03 +Nonblocking Locality Aware, 4 leaders: 7.454925e-02 +Nonblocking Multileader Locality, 4 leaders: 2.455257e-03 +Pairwise Multileader, 10 leaders: 3.769150e-03 +Pairwise Locality Aware, 10 leaders: 3.263934e-03 +Pairwise Multileader Locality, 10 leaders: 1.432656e-03 +Nonblocking Multileader, 10 leaders: 2.946536e-02 +Nonblocking Locality Aware, 10 leaders: 9.246005e-02 +Nonblocking Multileader Locality, 10 leaders: 9.815610e-04 +Pairwise Multileader, 20 leaders: 5.478854e-03 +Pairwise Locality Aware, 20 leaders: 6.926309e-03 +Pairwise Multileader Locality, 20 leaders: 9.221665e-04 +Nonblocking Multileader, 20 leaders: 3.729604e-02 +Nonblocking Locality Aware, 20 leaders: 1.385583e-01 +Nonblocking Multileader Locality, 20 leaders: 8.734253e-04 +Size 8 +PMPI: 3.654102e-03 +Pairwise: 2.814045e-02 +NonBlocking: 3.817931e-01 +Pairwise Hierarchical: 2.250295e-02 +Pairwise Node Aware: 1.312772e-03 +Nonblocking Hierarchical: 1.721162e-02 +Nonblocking Node Aware: 1.202117e-03 +Pairwise Multileader, 4 leaders: 6.018849e-03 +Pairwise Locality Aware, 4 leaders: 1.809473e-03 +Pairwise Multileader Locality, 4 leaders: 6.775774e-03 +Nonblocking Multileader, 4 leaders: 4.191610e-03 +Nonblocking Locality Aware, 4 leaders: 8.737296e-02 +Nonblocking Multileader Locality, 4 leaders: 5.117234e-03 +Pairwise Multileader, 10 leaders: 4.665801e-03 +Pairwise Locality Aware, 10 leaders: 3.564731e-03 +Pairwise Multileader Locality, 10 leaders: 2.505127e-03 +Nonblocking Multileader, 10 leaders: 6.622483e-03 +Nonblocking Locality Aware, 10 leaders: 1.157244e-01 +Nonblocking Multileader Locality, 10 leaders: 1.720281e-03 +Pairwise Multileader, 20 leaders: 5.628076e-03 +Pairwise Locality Aware, 20 leaders: 5.818781e-03 +Pairwise Multileader Locality, 20 leaders: 1.462692e-03 +Nonblocking Multileader, 20 leaders: 7.392028e-02 +Nonblocking Locality Aware, 20 leaders: 1.841776e-01 +Nonblocking Multileader Locality, 20 leaders: 1.047292e-03 +Size 16 +PMPI: 6.355871e-03 +Pairwise: 2.111610e-02 +NonBlocking: 4.862471e-01 +Pairwise Hierarchical: 4.542690e-02 +Pairwise Node Aware: 2.368325e-03 +Nonblocking Hierarchical: 3.529765e-02 +Nonblocking Node Aware: 2.546339e-03 +Pairwise Multileader, 4 leaders: 8.780592e-03 +Pairwise Locality Aware, 4 leaders: 2.533274e-03 +Pairwise Multileader Locality, 4 leaders: 1.157554e-02 +Nonblocking Multileader, 4 leaders: 6.785109e-03 +Nonblocking Locality Aware, 4 leaders: 9.337684e-02 +Nonblocking Multileader Locality, 4 leaders: 8.862506e-03 +Pairwise Multileader, 10 leaders: 6.655283e-03 +Pairwise Locality Aware, 10 leaders: 4.385906e-03 +Pairwise Multileader Locality, 10 leaders: 4.558250e-03 +Nonblocking Multileader, 10 leaders: 7.033857e-03 +Nonblocking Locality Aware, 10 leaders: 9.394475e-02 +Nonblocking Multileader Locality, 10 leaders: 3.744602e-03 +Pairwise Multileader, 20 leaders: 6.999545e-03 +Pairwise Locality Aware, 20 leaders: 5.964138e-03 +Pairwise Multileader Locality, 20 leaders: 3.043711e-03 +Nonblocking Multileader, 20 leaders: 8.779601e-02 +Nonblocking Locality Aware, 20 leaders: 1.161365e-01 +Nonblocking Multileader Locality, 20 leaders: 2.442798e-03 +Size 32 +PMPI: 1.257659e-02 +Pairwise: 2.231139e-02 +NonBlocking: 1.738363e-01 +Pairwise Hierarchical: 8.617649e-02 +Pairwise Node Aware: 4.367388e-03 +Nonblocking Hierarchical: 6.615292e-02 +Nonblocking Node Aware: 4.859708e-03 +Pairwise Multileader, 4 leaders: 2.769089e-02 +Pairwise Locality Aware, 4 leaders: 4.029427e-03 +Pairwise Multileader Locality, 4 leaders: 2.549106e-02 +Nonblocking Multileader, 4 leaders: 1.812481e-02 +Nonblocking Locality Aware, 4 leaders: 5.269045e-03 +Nonblocking Multileader Locality, 4 leaders: 1.825383e-02 +Pairwise Multileader, 10 leaders: 1.260892e-02 +Pairwise Locality Aware, 10 leaders: 5.661550e-03 +Pairwise Multileader Locality, 10 leaders: 1.090529e-02 +Nonblocking Multileader, 10 leaders: 8.005290e-03 +Nonblocking Locality Aware, 10 leaders: 1.402026e-01 +Nonblocking Multileader Locality, 10 leaders: 8.525167e-03 +Pairwise Multileader, 20 leaders: 9.979454e-03 +Pairwise Locality Aware, 20 leaders: 8.484492e-03 +Pairwise Multileader Locality, 20 leaders: 5.573719e-03 +Nonblocking Multileader, 20 leaders: 7.619720e-02 +Nonblocking Locality Aware, 20 leaders: 9.857426e-02 +Nonblocking Multileader Locality, 20 leaders: 4.990786e-03 +Size 64 +PMPI: 2.210315e-02 +Pairwise: 3.166936e-02 +NonBlocking: 1.759988e-01 +Pairwise Hierarchical: 1.674501e-01 +Pairwise Node Aware: 9.081208e-03 +Nonblocking Hierarchical: 1.269834e-01 +Nonblocking Node Aware: 7.341510e-03 +Pairwise Multileader, 4 leaders: 5.350917e-02 +Pairwise Locality Aware, 4 leaders: 8.193811e-03 +Pairwise Multileader Locality, 4 leaders: 5.807481e-02 +Nonblocking Multileader, 4 leaders: 3.798488e-02 +Nonblocking Locality Aware, 4 leaders: 9.762339e-03 +Nonblocking Multileader Locality, 4 leaders: 4.308408e-02 +Pairwise Multileader, 10 leaders: 2.660709e-02 +Pairwise Locality Aware, 10 leaders: 1.013870e-02 +Pairwise Multileader Locality, 10 leaders: 2.554758e-02 +Nonblocking Multileader, 10 leaders: 1.832670e-02 +Nonblocking Locality Aware, 10 leaders: 1.194124e-01 +Nonblocking Multileader Locality, 10 leaders: 1.994356e-02 +Pairwise Multileader, 20 leaders: 1.872626e-02 +Pairwise Locality Aware, 20 leaders: 1.091387e-02 +Pairwise Multileader Locality, 20 leaders: 1.228601e-02 +Nonblocking Multileader, 20 leaders: 1.169817e-02 +Nonblocking Locality Aware, 20 leaders: 1.190617e-01 +Nonblocking Multileader Locality, 20 leaders: 9.445211e-03 +Size 128 +PMPI: 4.206993e-02 +Pairwise: 4.121440e-02 +NonBlocking: 1.784690e-01 +Pairwise Hierarchical: 3.252661e-01 +Pairwise Node Aware: 1.925285e-02 +Nonblocking Hierarchical: 2.502857e-01 +Nonblocking Node Aware: 1.664309e-02 +Pairwise Multileader, 4 leaders: 9.801644e-02 +Pairwise Locality Aware, 4 leaders: 1.792165e-02 +Pairwise Multileader Locality, 4 leaders: 1.079918e-01 +Nonblocking Multileader, 4 leaders: 6.895253e-02 +Nonblocking Locality Aware, 4 leaders: 2.096790e-02 +Nonblocking Multileader Locality, 4 leaders: 7.997254e-02 +Pairwise Multileader, 10 leaders: 5.304625e-02 +Pairwise Locality Aware, 10 leaders: 1.773682e-02 +Pairwise Multileader Locality, 10 leaders: 5.307624e-02 +Nonblocking Multileader, 10 leaders: 3.634677e-02 +Nonblocking Locality Aware, 10 leaders: 5.929693e-02 +Nonblocking Multileader Locality, 10 leaders: 4.331923e-02 +Pairwise Multileader, 20 leaders: 3.554074e-02 +Pairwise Locality Aware, 20 leaders: 1.973995e-02 +Pairwise Multileader Locality, 20 leaders: 2.785882e-02 +Nonblocking Multileader, 20 leaders: 2.493249e-02 +Nonblocking Locality Aware, 20 leaders: 1.189244e-01 +Nonblocking Multileader Locality, 20 leaders: 2.118766e-02 +Size 256 +PMPI: 9.432507e-02 +Pairwise: 5.485313e-02 +NonBlocking: 3.469405e-01 +Pairwise Hierarchical: 6.411762e-01 +Pairwise Node Aware: 3.491076e-02 +Nonblocking Hierarchical: 5.382146e-01 +Nonblocking Node Aware: 2.998143e-02 +Pairwise Multileader, 4 leaders: 1.823430e-01 +Pairwise Locality Aware, 4 leaders: 3.836505e-02 +Pairwise Multileader Locality, 4 leaders: 2.059860e-01 +Nonblocking Multileader, 4 leaders: 1.318739e-01 +Nonblocking Locality Aware, 4 leaders: 2.519668e-02 +Nonblocking Multileader Locality, 4 leaders: 1.559385e-01 +Pairwise Multileader, 10 leaders: 1.146460e-01 +Pairwise Locality Aware, 10 leaders: 4.227466e-02 +Pairwise Multileader Locality, 10 leaders: 1.087841e-01 +Nonblocking Multileader, 10 leaders: 7.749574e-02 +Nonblocking Locality Aware, 10 leaders: 7.561370e-02 +Nonblocking Multileader Locality, 10 leaders: 8.915824e-02 +Pairwise Multileader, 20 leaders: 7.467525e-02 +Pairwise Locality Aware, 20 leaders: 4.419845e-02 +Pairwise Multileader Locality, 20 leaders: 5.721761e-02 +Nonblocking Multileader, 20 leaders: 4.221911e-02 +Nonblocking Locality Aware, 20 leaders: 1.306581e-01 +Nonblocking Multileader Locality, 20 leaders: 4.450555e-02 +Size 512 +PMPI: 2.514283e-01 +Pairwise: 9.815725e-02 +NonBlocking: 2.980273e-01 +Pairwise Hierarchical: 1.278520e+00 +Pairwise Node Aware: 6.930612e-02 +Nonblocking Hierarchical: 1.149918e+00 +Nonblocking Node Aware: 5.119667e-02 +Pairwise Multileader, 4 leaders: 3.471684e-01 +Pairwise Locality Aware, 4 leaders: 7.710276e-02 +Pairwise Multileader Locality, 4 leaders: 3.998047e-01 +Nonblocking Multileader, 4 leaders: 2.622265e-01 +Nonblocking Locality Aware, 4 leaders: 6.409648e-02 +Nonblocking Multileader Locality, 4 leaders: 3.066580e-01 +Pairwise Multileader, 10 leaders: 2.161047e-01 +Pairwise Locality Aware, 10 leaders: 8.533365e-02 +Pairwise Multileader Locality, 10 leaders: 2.154076e-01 +Nonblocking Multileader, 10 leaders: 1.460860e-01 +Nonblocking Locality Aware, 10 leaders: 6.019352e-02 +Nonblocking Multileader Locality, 10 leaders: 1.756188e-01 +Pairwise Multileader, 20 leaders: 1.590525e-01 +Pairwise Locality Aware, 20 leaders: 9.622019e-02 +Pairwise Multileader Locality, 20 leaders: 1.294821e-01 +Nonblocking Multileader, 20 leaders: 6.390400e-01 +Nonblocking Locality Aware, 20 leaders: 1.396733e-01 +Nonblocking Multileader Locality, 20 leaders: 1.088319e-01 +Size 1024 +PMPI: 1.760945e-01 +Pairwise: 1.883008e-01 +NonBlocking: 3.977451e-01 +Pairwise Hierarchical: 2.522271e+00 +Pairwise Node Aware: 1.459551e-01 +Nonblocking Hierarchical: 2.216354e+00 +Nonblocking Node Aware: 1.098529e-01 +Pairwise Multileader, 4 leaders: 7.021618e-01 +Pairwise Locality Aware, 4 leaders: 1.644194e-01 +Pairwise Multileader Locality, 4 leaders: 7.742622e-01 +Nonblocking Multileader, 4 leaders: 5.439149e-01 +Nonblocking Locality Aware, 4 leaders: 1.009834e-01 +Nonblocking Multileader Locality, 4 leaders: 6.533747e-01 +Pairwise Multileader, 10 leaders: 4.294791e-01 +Pairwise Locality Aware, 10 leaders: 1.761084e-01 +Pairwise Multileader Locality, 10 leaders: 4.477596e-01 +Nonblocking Multileader, 10 leaders: 2.865746e-01 +Nonblocking Locality Aware, 10 leaders: 3.349199e-01 +Nonblocking Multileader Locality, 10 leaders: 3.467518e-01 +Pairwise Multileader, 20 leaders: 3.021685e-01 +Pairwise Locality Aware, 20 leaders: 1.827665e-01 +Pairwise Multileader Locality, 20 leaders: 2.537618e-01 +Nonblocking Multileader, 20 leaders: 1.784817e-01 +Nonblocking Locality Aware, 20 leaders: 9.916291e-02 +Nonblocking Multileader Locality, 20 leaders: 2.136892e-01 +Size 2048 +PMPI: 3.727578e-01 +Pairwise: 3.904754e-01 +NonBlocking: 5.505518e-01 +Test 3 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.055472e-03 +Pairwise: 1.237969e-02 +NonBlocking: 2.893233e-01 +Pairwise Hierarchical: 2.575624e-03 +Pairwise Node Aware: 6.216009e-04 +Nonblocking Hierarchical: 2.153207e-03 +Nonblocking Node Aware: 4.341105e-04 +Pairwise Multileader, 4 leaders: 1.454844e-03 +Pairwise Locality Aware, 4 leaders: 1.109186e-03 +Pairwise Multileader Locality, 4 leaders: 9.496204e-04 +Nonblocking Multileader, 4 leaders: 7.053253e-04 +Nonblocking Locality Aware, 4 leaders: 3.741290e-02 +Nonblocking Multileader Locality, 4 leaders: 6.758262e-04 +Pairwise Multileader, 10 leaders: 2.386179e-03 +Pairwise Locality Aware, 10 leaders: 2.488437e-03 +Pairwise Multileader Locality, 10 leaders: 5.492129e-04 +Nonblocking Multileader, 10 leaders: 4.856770e-04 +Nonblocking Locality Aware, 10 leaders: 9.205617e-02 +Nonblocking Multileader Locality, 10 leaders: 3.571796e-04 +Pairwise Multileader, 20 leaders: 5.290914e-03 +Pairwise Locality Aware, 20 leaders: 4.599698e-03 +Pairwise Multileader Locality, 20 leaders: 4.880779e-04 +Nonblocking Multileader, 20 leaders: 1.311717e-02 +Nonblocking Locality Aware, 20 leaders: 1.380594e-01 +Nonblocking Multileader Locality, 20 leaders: 2.868982e-04 +Size 2 +PMPI: 1.125617e-03 +Pairwise: 1.165567e-02 +NonBlocking: 4.471604e-01 +Pairwise Hierarchical: 5.383150e-03 +Pairwise Node Aware: 5.841172e-04 +Nonblocking Hierarchical: 4.505261e-03 +Nonblocking Node Aware: 5.997715e-04 +Pairwise Multileader, 4 leaders: 1.932968e-03 +Pairwise Locality Aware, 4 leaders: 1.059101e-03 +Pairwise Multileader Locality, 4 leaders: 1.724014e-03 +Nonblocking Multileader, 4 leaders: 1.120125e-03 +Nonblocking Locality Aware, 4 leaders: 6.912531e-02 +Nonblocking Multileader Locality, 4 leaders: 1.393526e-03 +Pairwise Multileader, 10 leaders: 2.526451e-03 +Pairwise Locality Aware, 10 leaders: 2.667369e-03 +Pairwise Multileader Locality, 10 leaders: 8.214162e-04 +Nonblocking Multileader, 10 leaders: 3.020615e-03 +Nonblocking Locality Aware, 10 leaders: 1.151520e-01 +Nonblocking Multileader Locality, 10 leaders: 5.806591e-04 +Pairwise Multileader, 20 leaders: 4.204395e-03 +Pairwise Locality Aware, 20 leaders: 5.032892e-03 +Pairwise Multileader Locality, 20 leaders: 5.525235e-04 +Nonblocking Multileader, 20 leaders: 1.240074e-02 +Nonblocking Locality Aware, 20 leaders: 1.610340e-01 +Nonblocking Multileader Locality, 20 leaders: 3.856801e-04 +Size 4 +PMPI: 1.714340e-03 +Pairwise: 2.280724e-02 +NonBlocking: 2.489475e-01 +Pairwise Hierarchical: 1.028087e-02 +Pairwise Node Aware: 9.075155e-04 +Nonblocking Hierarchical: 8.598595e-03 +Nonblocking Node Aware: 7.996575e-04 +Pairwise Multileader, 4 leaders: 3.235905e-03 +Pairwise Locality Aware, 4 leaders: 1.305026e-03 +Pairwise Multileader Locality, 4 leaders: 3.011501e-03 +Nonblocking Multileader, 4 leaders: 2.130433e-03 +Nonblocking Locality Aware, 4 leaders: 7.455441e-02 +Nonblocking Multileader Locality, 4 leaders: 2.443477e-03 +Pairwise Multileader, 10 leaders: 3.090014e-03 +Pairwise Locality Aware, 10 leaders: 2.934411e-03 +Pairwise Multileader Locality, 10 leaders: 1.403298e-03 +Nonblocking Multileader, 10 leaders: 4.357552e-02 +Nonblocking Locality Aware, 10 leaders: 9.934834e-02 +Nonblocking Multileader Locality, 10 leaders: 9.612260e-04 +Pairwise Multileader, 20 leaders: 5.871230e-03 +Pairwise Locality Aware, 20 leaders: 4.746162e-03 +Pairwise Multileader Locality, 20 leaders: 8.781857e-04 +Nonblocking Multileader, 20 leaders: 2.547297e-02 +Nonblocking Locality Aware, 20 leaders: 1.154987e-01 +Nonblocking Multileader Locality, 20 leaders: 6.486847e-04 +Size 8 +PMPI: 3.532991e-03 +Pairwise: 1.777645e-02 +NonBlocking: 5.256354e-01 +Pairwise Hierarchical: 2.243751e-02 +Pairwise Node Aware: 1.372765e-03 +Nonblocking Hierarchical: 1.731902e-02 +Nonblocking Node Aware: 1.367463e-03 +Pairwise Multileader, 4 leaders: 5.821601e-03 +Pairwise Locality Aware, 4 leaders: 1.717641e-03 +Pairwise Multileader Locality, 4 leaders: 6.636285e-03 +Nonblocking Multileader, 4 leaders: 4.035533e-03 +Nonblocking Locality Aware, 4 leaders: 8.738400e-02 +Nonblocking Multileader Locality, 4 leaders: 5.087065e-03 +Pairwise Multileader, 10 leaders: 4.086848e-03 +Pairwise Locality Aware, 10 leaders: 3.291264e-03 +Pairwise Multileader Locality, 10 leaders: 2.497118e-03 +Nonblocking Multileader, 10 leaders: 3.570547e-03 +Nonblocking Locality Aware, 10 leaders: 1.153059e-01 +Nonblocking Multileader Locality, 10 leaders: 1.945834e-03 +Pairwise Multileader, 20 leaders: 5.198665e-03 +Pairwise Locality Aware, 20 leaders: 5.784963e-03 +Pairwise Multileader Locality, 20 leaders: 1.553622e-03 +Nonblocking Multileader, 20 leaders: 7.686344e-02 +Nonblocking Locality Aware, 20 leaders: 1.487011e-01 +Nonblocking Multileader Locality, 20 leaders: 1.385980e-03 +Size 16 +PMPI: 6.324950e-03 +Pairwise: 2.737180e-02 +NonBlocking: 2.525646e-01 +Pairwise Hierarchical: 4.563905e-02 +Pairwise Node Aware: 2.509425e-03 +Nonblocking Hierarchical: 3.549740e-02 +Nonblocking Node Aware: 2.610342e-03 +Pairwise Multileader, 4 leaders: 8.631717e-03 +Pairwise Locality Aware, 4 leaders: 2.592347e-03 +Pairwise Multileader Locality, 4 leaders: 1.232501e-02 +Nonblocking Multileader, 4 leaders: 7.009405e-03 +Nonblocking Locality Aware, 4 leaders: 8.790948e-02 +Nonblocking Multileader Locality, 4 leaders: 8.875843e-03 +Pairwise Multileader, 10 leaders: 7.173871e-03 +Pairwise Locality Aware, 10 leaders: 3.605849e-03 +Pairwise Multileader Locality, 10 leaders: 4.535899e-03 +Nonblocking Multileader, 10 leaders: 6.429073e-03 +Nonblocking Locality Aware, 10 leaders: 1.160901e-01 +Nonblocking Multileader Locality, 10 leaders: 3.400248e-03 +Pairwise Multileader, 20 leaders: 6.558018e-03 +Pairwise Locality Aware, 20 leaders: 5.330070e-03 +Pairwise Multileader Locality, 20 leaders: 3.061517e-03 +Nonblocking Multileader, 20 leaders: 7.926949e-02 +Nonblocking Locality Aware, 20 leaders: 1.385879e-01 +Nonblocking Multileader Locality, 20 leaders: 2.362466e-03 +Size 32 +PMPI: 1.245397e-02 +Pairwise: 2.429081e-02 +NonBlocking: 3.288145e-01 +Pairwise Hierarchical: 8.665138e-02 +Pairwise Node Aware: 4.835037e-03 +Nonblocking Hierarchical: 6.652150e-02 +Nonblocking Node Aware: 5.604263e-03 +Pairwise Multileader, 4 leaders: 2.810149e-02 +Pairwise Locality Aware, 4 leaders: 4.032951e-03 +Pairwise Multileader Locality, 4 leaders: 2.379968e-02 +Nonblocking Multileader, 4 leaders: 1.801465e-02 +Nonblocking Locality Aware, 4 leaders: 4.203081e-03 +Nonblocking Multileader Locality, 4 leaders: 1.811737e-02 +Pairwise Multileader, 10 leaders: 1.238615e-02 +Pairwise Locality Aware, 10 leaders: 5.493776e-03 +Pairwise Multileader Locality, 10 leaders: 1.083376e-02 +Nonblocking Multileader, 10 leaders: 1.047293e-02 +Nonblocking Locality Aware, 10 leaders: 1.399615e-01 +Nonblocking Multileader Locality, 10 leaders: 8.570907e-03 +Pairwise Multileader, 20 leaders: 1.138497e-02 +Pairwise Locality Aware, 20 leaders: 8.267462e-03 +Pairwise Multileader Locality, 20 leaders: 5.733198e-03 +Nonblocking Multileader, 20 leaders: 7.688359e-02 +Nonblocking Locality Aware, 20 leaders: 1.175749e-01 +Nonblocking Multileader Locality, 20 leaders: 5.080189e-03 +Size 64 +PMPI: 2.247962e-02 +Pairwise: 2.473943e-02 +NonBlocking: 1.755293e-01 +Pairwise Hierarchical: 1.662957e-01 +Pairwise Node Aware: 1.017774e-02 +Nonblocking Hierarchical: 1.271985e-01 +Nonblocking Node Aware: 8.079537e-03 +Pairwise Multileader, 4 leaders: 5.522289e-02 +Pairwise Locality Aware, 4 leaders: 1.055062e-02 +Pairwise Multileader Locality, 4 leaders: 5.830359e-02 +Nonblocking Multileader, 4 leaders: 3.812707e-02 +Nonblocking Locality Aware, 4 leaders: 1.402367e-02 +Nonblocking Multileader Locality, 4 leaders: 4.249056e-02 +Pairwise Multileader, 10 leaders: 2.594869e-02 +Pairwise Locality Aware, 10 leaders: 1.024322e-02 +Pairwise Multileader Locality, 10 leaders: 2.526988e-02 +Nonblocking Multileader, 10 leaders: 2.869198e-02 +Nonblocking Locality Aware, 10 leaders: 1.210145e-01 +Nonblocking Multileader Locality, 10 leaders: 1.990510e-02 +Pairwise Multileader, 20 leaders: 1.743032e-02 +Pairwise Locality Aware, 20 leaders: 1.211378e-02 +Pairwise Multileader Locality, 20 leaders: 1.239042e-02 +Nonblocking Multileader, 20 leaders: 1.036931e-02 +Nonblocking Locality Aware, 20 leaders: 1.876516e-01 +Nonblocking Multileader Locality, 20 leaders: 9.647049e-03 +Size 128 +PMPI: 4.334803e-02 +Pairwise: 3.543226e-02 +NonBlocking: 1.791083e-01 +Pairwise Hierarchical: 3.234307e-01 +Pairwise Node Aware: 1.954792e-02 +Nonblocking Hierarchical: 2.501333e-01 +Nonblocking Node Aware: 1.733284e-02 +Pairwise Multileader, 4 leaders: 9.779508e-02 +Pairwise Locality Aware, 4 leaders: 1.696750e-02 +Pairwise Multileader Locality, 4 leaders: 1.073890e-01 +Nonblocking Multileader, 4 leaders: 6.943296e-02 +Nonblocking Locality Aware, 4 leaders: 2.553511e-02 +Nonblocking Multileader Locality, 4 leaders: 8.000631e-02 +Pairwise Multileader, 10 leaders: 5.227442e-02 +Pairwise Locality Aware, 10 leaders: 2.023717e-02 +Pairwise Multileader Locality, 10 leaders: 5.343180e-02 +Nonblocking Multileader, 10 leaders: 3.697933e-02 +Nonblocking Locality Aware, 10 leaders: 9.221216e-02 +Nonblocking Multileader Locality, 10 leaders: 4.172816e-02 +Pairwise Multileader, 20 leaders: 3.744261e-02 +Pairwise Locality Aware, 20 leaders: 1.982802e-02 +Pairwise Multileader Locality, 20 leaders: 2.778863e-02 +Nonblocking Multileader, 20 leaders: 3.678086e-02 +Nonblocking Locality Aware, 20 leaders: 1.177536e-01 +Nonblocking Multileader Locality, 20 leaders: 2.093806e-02 +Size 256 +PMPI: 9.162426e-02 +Pairwise: 6.332365e-02 +NonBlocking: 2.920020e-01 +Pairwise Hierarchical: 6.579750e-01 +Pairwise Node Aware: 3.493470e-02 +Nonblocking Hierarchical: 5.447489e-01 +Nonblocking Node Aware: 3.007947e-02 +Pairwise Multileader, 4 leaders: 1.873834e-01 +Pairwise Locality Aware, 4 leaders: 3.899452e-02 +Pairwise Multileader Locality, 4 leaders: 2.094650e-01 +Nonblocking Multileader, 4 leaders: 1.319658e-01 +Nonblocking Locality Aware, 4 leaders: 2.703367e-02 +Nonblocking Multileader Locality, 4 leaders: 1.571266e-01 +Pairwise Multileader, 10 leaders: 1.134088e-01 +Pairwise Locality Aware, 10 leaders: 3.913154e-02 +Pairwise Multileader Locality, 10 leaders: 1.089140e-01 +Nonblocking Multileader, 10 leaders: 7.669130e-02 +Nonblocking Locality Aware, 10 leaders: 6.516589e-02 +Nonblocking Multileader Locality, 10 leaders: 8.879379e-02 +Pairwise Multileader, 20 leaders: 7.208192e-02 +Pairwise Locality Aware, 20 leaders: 4.253120e-02 +Pairwise Multileader Locality, 20 leaders: 5.727737e-02 +Nonblocking Multileader, 20 leaders: 2.450993e-01 +Nonblocking Locality Aware, 20 leaders: 9.954989e-02 +Nonblocking Multileader Locality, 20 leaders: 4.403405e-02 +Size 512 +PMPI: 3.577576e-01 +Pairwise: 9.777383e-02 +NonBlocking: 3.158882e-01 +Pairwise Hierarchical: 1.298366e+00 +Pairwise Node Aware: 6.999356e-02 +Nonblocking Hierarchical: 1.147719e+00 +Nonblocking Node Aware: 5.177397e-02 +Pairwise Multileader, 4 leaders: 3.476993e-01 +Pairwise Locality Aware, 4 leaders: 7.592425e-02 +Pairwise Multileader Locality, 4 leaders: 4.005040e-01 +Nonblocking Multileader, 4 leaders: 2.596986e-01 +Nonblocking Locality Aware, 4 leaders: 1.167248e-01 +Nonblocking Multileader Locality, 4 leaders: 3.088130e-01 +Pairwise Multileader, 10 leaders: 2.151344e-01 +Pairwise Locality Aware, 10 leaders: 8.495353e-02 +Pairwise Multileader Locality, 10 leaders: 2.153634e-01 +Nonblocking Multileader, 10 leaders: 1.462995e-01 +Nonblocking Locality Aware, 10 leaders: 5.961992e-02 +Nonblocking Multileader Locality, 10 leaders: 1.741985e-01 +Pairwise Multileader, 20 leaders: 1.514422e-01 +Pairwise Locality Aware, 20 leaders: 9.108556e-02 +Pairwise Multileader Locality, 20 leaders: 1.285479e-01 +Nonblocking Multileader, 20 leaders: 9.042251e-02 +Nonblocking Locality Aware, 20 leaders: 1.164822e-01 +Nonblocking Multileader Locality, 20 leaders: 1.092774e-01 +Size 1024 +PMPI: 1.858435e-01 +Pairwise: 1.944137e-01 +NonBlocking: 3.981591e-01 +Pairwise Hierarchical: 2.552199e+00 +Pairwise Node Aware: 1.462142e-01 +Nonblocking Hierarchical: 2.343260e+00 +Nonblocking Node Aware: 1.099826e-01 +Pairwise Multileader, 4 leaders: 7.090888e-01 +Pairwise Locality Aware, 4 leaders: 1.599827e-01 +Pairwise Multileader Locality, 4 leaders: 8.167963e-01 +Nonblocking Multileader, 4 leaders: 5.386469e-01 +Nonblocking Locality Aware, 4 leaders: 1.036199e-01 +Nonblocking Multileader Locality, 4 leaders: 6.575249e-01 +Pairwise Multileader, 10 leaders: 4.290647e-01 +Pairwise Locality Aware, 10 leaders: 1.817014e-01 +Pairwise Multileader Locality, 10 leaders: 4.479465e-01 +Nonblocking Multileader, 10 leaders: 2.897537e-01 +Nonblocking Locality Aware, 10 leaders: 2.119727e-01 +Nonblocking Multileader Locality, 10 leaders: 3.430072e-01 +Pairwise Multileader, 20 leaders: 3.049606e-01 +Pairwise Locality Aware, 20 leaders: 1.814067e-01 +Pairwise Multileader Locality, 20 leaders: 2.502642e-01 +Nonblocking Multileader, 20 leaders: 1.796115e-01 +Nonblocking Locality Aware, 20 leaders: 9.772620e-02 +Nonblocking Multileader Locality, 20 leaders: 2.138454e-01 +Size 2048 +PMPI: 3.816671e-01 +Pairwise: 3.825434e-01 +NonBlocking: 6.821265e-01 diff --git a/runscripts/dane_results/alltoall_N32.2484414.err b/runscripts/dane_results/alltoall_N32.2484414.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/dane_results/alltoall_N32.2484414.out b/runscripts/dane_results/alltoall_N32.2484414.out new file mode 100644 index 000000000..2a382f6d6 --- /dev/null +++ b/runscripts/dane_results/alltoall_N32.2484414.out @@ -0,0 +1,783 @@ +Test 1 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.290998e-03 +Pairwise: 1.345257e-02 +NonBlocking: 2.879422e-01 +Pairwise Hierarchical: 2.499684e-03 +Pairwise Node Aware: 6.706824e-04 +Nonblocking Hierarchical: 1.987357e-03 +Nonblocking Node Aware: 4.406038e-04 +Pairwise Multileader, 4 leaders: 1.843863e-03 +Pairwise Locality Aware, 4 leaders: 1.247043e-03 +Pairwise Multileader Locality, 4 leaders: 1.008590e-03 +Nonblocking Multileader, 4 leaders: 9.999425e-04 +Nonblocking Locality Aware, 4 leaders: 8.667506e-02 +Nonblocking Multileader Locality, 4 leaders: 6.633480e-04 +Pairwise Multileader, 10 leaders: 3.710085e-03 +Pairwise Locality Aware, 10 leaders: 2.964471e-03 +Pairwise Multileader Locality, 10 leaders: 6.268287e-04 +Nonblocking Multileader, 10 leaders: 2.934349e-02 +Nonblocking Locality Aware, 10 leaders: 1.150010e-01 +Nonblocking Multileader Locality, 10 leaders: 3.495823e-04 +Pairwise Multileader, 20 leaders: 5.733461e-03 +Pairwise Locality Aware, 20 leaders: 6.058604e-03 +Pairwise Multileader Locality, 20 leaders: 5.267475e-04 +Nonblocking Multileader, 20 leaders: 2.077282e-02 +Nonblocking Locality Aware, 20 leaders: 1.152599e-01 +Nonblocking Multileader Locality, 20 leaders: 2.669874e-04 +Size 2 +PMPI: 1.140265e-03 +Pairwise: 1.501733e-02 +NonBlocking: 2.823506e-01 +Pairwise Hierarchical: 5.195415e-03 +Pairwise Node Aware: 7.301468e-04 +Nonblocking Hierarchical: 4.297416e-03 +Nonblocking Node Aware: 5.787378e-04 +Pairwise Multileader, 4 leaders: 2.239428e-03 +Pairwise Locality Aware, 4 leaders: 1.301089e-03 +Pairwise Multileader Locality, 4 leaders: 1.892310e-03 +Nonblocking Multileader, 4 leaders: 1.118529e-03 +Nonblocking Locality Aware, 4 leaders: 6.204779e-02 +Nonblocking Multileader Locality, 4 leaders: 1.237547e-03 +Pairwise Multileader, 10 leaders: 3.170179e-03 +Pairwise Locality Aware, 10 leaders: 3.578338e-03 +Pairwise Multileader Locality, 10 leaders: 8.582847e-04 +Nonblocking Multileader, 10 leaders: 1.295904e-02 +Nonblocking Locality Aware, 10 leaders: 1.150100e-01 +Nonblocking Multileader Locality, 10 leaders: 6.204013e-04 +Pairwise Multileader, 20 leaders: 4.566407e-03 +Pairwise Locality Aware, 20 leaders: 5.670755e-03 +Pairwise Multileader Locality, 20 leaders: 6.393316e-04 +Nonblocking Multileader, 20 leaders: 5.099049e-02 +Nonblocking Locality Aware, 20 leaders: 1.613323e-01 +Nonblocking Multileader Locality, 20 leaders: 3.464329e-04 +Size 4 +PMPI: 1.669983e-03 +Pairwise: 2.795479e-02 +NonBlocking: 2.881160e-01 +Pairwise Hierarchical: 1.040119e-02 +Pairwise Node Aware: 8.695935e-04 +Nonblocking Hierarchical: 8.484647e-03 +Nonblocking Node Aware: 7.661085e-04 +Pairwise Multileader, 4 leaders: 3.260850e-03 +Pairwise Locality Aware, 4 leaders: 1.284160e-03 +Pairwise Multileader Locality, 4 leaders: 2.827368e-03 +Nonblocking Multileader, 4 leaders: 2.998268e-03 +Nonblocking Locality Aware, 4 leaders: 7.466986e-02 +Nonblocking Multileader Locality, 4 leaders: 2.348842e-03 +Pairwise Multileader, 10 leaders: 3.515880e-03 +Pairwise Locality Aware, 10 leaders: 2.552658e-03 +Pairwise Multileader Locality, 10 leaders: 1.466511e-03 +Nonblocking Multileader, 10 leaders: 3.148790e-02 +Nonblocking Locality Aware, 10 leaders: 1.153635e-01 +Nonblocking Multileader Locality, 10 leaders: 9.091503e-04 +Pairwise Multileader, 20 leaders: 7.415909e-03 +Pairwise Locality Aware, 20 leaders: 5.149674e-03 +Pairwise Multileader Locality, 20 leaders: 9.763297e-04 +Nonblocking Multileader, 20 leaders: 6.774516e-02 +Nonblocking Locality Aware, 20 leaders: 1.160461e-01 +Nonblocking Multileader Locality, 20 leaders: 7.504667e-04 +Size 8 +PMPI: 3.504206e-03 +Pairwise: 2.351232e-02 +NonBlocking: 2.876712e-01 +Pairwise Hierarchical: 2.173694e-02 +Pairwise Node Aware: 1.481864e-03 +Nonblocking Hierarchical: 1.734277e-02 +Nonblocking Node Aware: 1.457395e-03 +Pairwise Multileader, 4 leaders: 6.397059e-03 +Pairwise Locality Aware, 4 leaders: 1.854300e-03 +Pairwise Multileader Locality, 4 leaders: 6.584031e-03 +Nonblocking Multileader, 4 leaders: 4.074506e-03 +Nonblocking Locality Aware, 4 leaders: 9.272773e-02 +Nonblocking Multileader Locality, 4 leaders: 5.018448e-03 +Pairwise Multileader, 10 leaders: 4.624642e-03 +Pairwise Locality Aware, 10 leaders: 4.057455e-03 +Pairwise Multileader Locality, 10 leaders: 2.593517e-03 +Nonblocking Multileader, 10 leaders: 1.424335e-02 +Nonblocking Locality Aware, 10 leaders: 1.120829e-01 +Nonblocking Multileader Locality, 10 leaders: 2.144582e-03 +Pairwise Multileader, 20 leaders: 1.506651e-02 +Pairwise Locality Aware, 20 leaders: 1.635952e-02 +Pairwise Multileader Locality, 20 leaders: 2.085312e-03 +Nonblocking Multileader, 20 leaders: 9.281744e-02 +Nonblocking Locality Aware, 20 leaders: 1.384884e-01 +Nonblocking Multileader Locality, 20 leaders: 1.338282e-03 +Size 16 +PMPI: 6.272099e-03 +Pairwise: 6.617737e-02 +NonBlocking: 3.821994e-01 +Pairwise Hierarchical: 4.597538e-02 +Pairwise Node Aware: 3.247408e-03 +Nonblocking Hierarchical: 3.505844e-02 +Nonblocking Node Aware: 2.878689e-03 +Pairwise Multileader, 4 leaders: 1.133056e-02 +Pairwise Locality Aware, 4 leaders: 2.773318e-03 +Pairwise Multileader Locality, 4 leaders: 1.129187e-02 +Nonblocking Multileader, 4 leaders: 6.847244e-03 +Nonblocking Locality Aware, 4 leaders: 9.367909e-02 +Nonblocking Multileader Locality, 4 leaders: 8.802624e-03 +Pairwise Multileader, 10 leaders: 7.814590e-03 +Pairwise Locality Aware, 10 leaders: 4.027630e-03 +Pairwise Multileader Locality, 10 leaders: 4.595332e-03 +Nonblocking Multileader, 10 leaders: 6.426311e-03 +Nonblocking Locality Aware, 10 leaders: 9.334448e-02 +Nonblocking Multileader Locality, 10 leaders: 3.412571e-03 +Pairwise Multileader, 20 leaders: 8.375289e-03 +Pairwise Locality Aware, 20 leaders: 9.564043e-03 +Pairwise Multileader Locality, 20 leaders: 2.950856e-03 +Nonblocking Multileader, 20 leaders: 7.063055e-02 +Nonblocking Locality Aware, 20 leaders: 1.005389e-01 +Nonblocking Multileader Locality, 20 leaders: 2.348690e-03 +Size 32 +PMPI: 1.012485e-02 +Pairwise: 2.439857e-02 +NonBlocking: 3.281298e-01 +Pairwise Hierarchical: 8.621267e-02 +Pairwise Node Aware: 4.744353e-03 +Nonblocking Hierarchical: 6.581990e-02 +Nonblocking Node Aware: 5.660954e-03 +Pairwise Multileader, 4 leaders: 2.832131e-02 +Pairwise Locality Aware, 4 leaders: 4.401672e-03 +Pairwise Multileader Locality, 4 leaders: 2.394646e-02 +Nonblocking Multileader, 4 leaders: 1.815520e-02 +Nonblocking Locality Aware, 4 leaders: 6.683484e-03 +Nonblocking Multileader Locality, 4 leaders: 1.815061e-02 +Pairwise Multileader, 10 leaders: 1.328028e-02 +Pairwise Locality Aware, 10 leaders: 6.057429e-03 +Pairwise Multileader Locality, 10 leaders: 1.063471e-02 +Nonblocking Multileader, 10 leaders: 9.291359e-03 +Nonblocking Locality Aware, 10 leaders: 1.403822e-01 +Nonblocking Multileader Locality, 10 leaders: 8.503213e-03 +Pairwise Multileader, 20 leaders: 1.113026e-02 +Pairwise Locality Aware, 20 leaders: 8.851412e-03 +Pairwise Multileader Locality, 20 leaders: 5.257616e-03 +Nonblocking Multileader, 20 leaders: 7.852423e-02 +Nonblocking Locality Aware, 20 leaders: 1.170545e-01 +Nonblocking Multileader Locality, 20 leaders: 4.456862e-03 +Size 64 +PMPI: 1.908727e-02 +Pairwise: 3.119517e-02 +NonBlocking: 2.494691e-01 +Pairwise Hierarchical: 1.657924e-01 +Pairwise Node Aware: 8.479932e-03 +Nonblocking Hierarchical: 1.251648e-01 +Nonblocking Node Aware: 7.607485e-03 +Pairwise Multileader, 4 leaders: 5.301440e-02 +Pairwise Locality Aware, 4 leaders: 8.644631e-03 +Pairwise Multileader Locality, 4 leaders: 5.624023e-02 +Nonblocking Multileader, 4 leaders: 3.726041e-02 +Nonblocking Locality Aware, 4 leaders: 1.380703e-02 +Nonblocking Multileader Locality, 4 leaders: 4.218491e-02 +Pairwise Multileader, 10 leaders: 4.188808e-02 +Pairwise Locality Aware, 10 leaders: 8.833799e-03 +Pairwise Multileader Locality, 10 leaders: 2.526673e-02 +Nonblocking Multileader, 10 leaders: 1.764591e-02 +Nonblocking Locality Aware, 10 leaders: 1.191751e-01 +Nonblocking Multileader Locality, 10 leaders: 1.996140e-02 +Pairwise Multileader, 20 leaders: 1.917070e-02 +Pairwise Locality Aware, 20 leaders: 1.114174e-02 +Pairwise Multileader Locality, 20 leaders: 1.224749e-02 +Nonblocking Multileader, 20 leaders: 1.402715e-02 +Nonblocking Locality Aware, 20 leaders: 1.389666e-01 +Nonblocking Multileader Locality, 20 leaders: 9.758428e-03 +Size 128 +PMPI: 3.744317e-02 +Pairwise: 3.219359e-02 +NonBlocking: 2.792763e-01 +Pairwise Hierarchical: 3.222289e-01 +Pairwise Node Aware: 1.812262e-02 +Nonblocking Hierarchical: 2.473743e-01 +Nonblocking Node Aware: 1.652314e-02 +Pairwise Multileader, 4 leaders: 9.867642e-02 +Pairwise Locality Aware, 4 leaders: 1.402139e-02 +Pairwise Multileader Locality, 4 leaders: 1.074205e-01 +Nonblocking Multileader, 4 leaders: 6.820319e-02 +Nonblocking Locality Aware, 4 leaders: 2.295829e-02 +Nonblocking Multileader Locality, 4 leaders: 7.902826e-02 +Pairwise Multileader, 10 leaders: 5.141512e-02 +Pairwise Locality Aware, 10 leaders: 1.828356e-02 +Pairwise Multileader Locality, 10 leaders: 5.288182e-02 +Nonblocking Multileader, 10 leaders: 3.598935e-02 +Nonblocking Locality Aware, 10 leaders: 9.396653e-02 +Nonblocking Multileader Locality, 10 leaders: 4.201840e-02 +Pairwise Multileader, 20 leaders: 4.019805e-02 +Pairwise Locality Aware, 20 leaders: 2.645730e-02 +Pairwise Multileader Locality, 20 leaders: 2.845998e-02 +Nonblocking Multileader, 20 leaders: 5.816259e-02 +Nonblocking Locality Aware, 20 leaders: 1.421314e-01 +Nonblocking Multileader Locality, 20 leaders: 2.122844e-02 +Size 256 +PMPI: 8.092728e-02 +Pairwise: 7.351035e-02 +NonBlocking: 3.416633e-01 +Pairwise Hierarchical: 6.479909e-01 +Pairwise Node Aware: 3.317995e-02 +Nonblocking Hierarchical: 5.329627e-01 +Nonblocking Node Aware: 2.998144e-02 +Pairwise Multileader, 4 leaders: 1.832733e-01 +Pairwise Locality Aware, 4 leaders: 3.219852e-02 +Pairwise Multileader Locality, 4 leaders: 2.070862e-01 +Nonblocking Multileader, 4 leaders: 1.312000e-01 +Nonblocking Locality Aware, 4 leaders: 2.859113e-02 +Nonblocking Multileader Locality, 4 leaders: 1.564569e-01 +Pairwise Multileader, 10 leaders: 3.609583e-01 +Pairwise Locality Aware, 10 leaders: 3.609413e-02 +Pairwise Multileader Locality, 10 leaders: 1.122209e-01 +Nonblocking Multileader, 10 leaders: 7.915475e-02 +Nonblocking Locality Aware, 10 leaders: 7.001752e-02 +Nonblocking Multileader Locality, 10 leaders: 8.883162e-02 +Pairwise Multileader, 20 leaders: 1.467625e-01 +Pairwise Locality Aware, 20 leaders: 4.174961e-02 +Pairwise Multileader Locality, 20 leaders: 5.871035e-02 +Nonblocking Multileader, 20 leaders: 1.075470e-01 +Nonblocking Locality Aware, 20 leaders: 1.373569e-01 +Nonblocking Multileader Locality, 20 leaders: 4.413281e-02 +Size 512 +PMPI: 2.022913e-01 +Pairwise: 1.041129e-01 +NonBlocking: 3.071507e-01 +Pairwise Hierarchical: 1.300821e+00 +Pairwise Node Aware: 6.319347e-02 +Nonblocking Hierarchical: 1.150739e+00 +Nonblocking Node Aware: 5.205187e-02 +Pairwise Multileader, 4 leaders: 3.410151e-01 +Pairwise Locality Aware, 4 leaders: 7.669301e-02 +Pairwise Multileader Locality, 4 leaders: 4.013120e-01 +Nonblocking Multileader, 4 leaders: 2.576495e-01 +Nonblocking Locality Aware, 4 leaders: 6.893088e-02 +Nonblocking Multileader Locality, 4 leaders: 3.065132e-01 +Pairwise Multileader, 10 leaders: 5.011387e-01 +Pairwise Locality Aware, 10 leaders: 8.023992e-02 +Pairwise Multileader Locality, 10 leaders: 2.209330e-01 +Nonblocking Multileader, 10 leaders: 1.430812e-01 +Nonblocking Locality Aware, 10 leaders: 6.441259e-02 +Nonblocking Multileader Locality, 10 leaders: 1.730836e-01 +Pairwise Multileader, 20 leaders: 1.611095e-01 +Pairwise Locality Aware, 20 leaders: 6.834682e-02 +Pairwise Multileader Locality, 20 leaders: 1.245207e-01 +Nonblocking Multileader, 20 leaders: 3.769843e-01 +Nonblocking Locality Aware, 20 leaders: 1.119293e-01 +Nonblocking Multileader Locality, 20 leaders: 1.067877e-01 +Test 2 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.009187e-03 +Pairwise: 1.526318e-02 +NonBlocking: 6.035320e-01 +Pairwise Hierarchical: 2.700822e-03 +Pairwise Node Aware: 6.767955e-04 +Nonblocking Hierarchical: 2.001083e-03 +Nonblocking Node Aware: 4.513460e-04 +Pairwise Multileader, 4 leaders: 1.573214e-03 +Pairwise Locality Aware, 4 leaders: 1.169939e-03 +Pairwise Multileader Locality, 4 leaders: 1.074684e-03 +Nonblocking Multileader, 4 leaders: 2.084203e-03 +Nonblocking Locality Aware, 4 leaders: 3.734137e-02 +Nonblocking Multileader Locality, 4 leaders: 6.793832e-04 +Pairwise Multileader, 10 leaders: 2.617668e-03 +Pairwise Locality Aware, 10 leaders: 4.344474e-03 +Pairwise Multileader Locality, 10 leaders: 7.721729e-04 +Nonblocking Multileader, 10 leaders: 1.067048e-02 +Nonblocking Locality Aware, 10 leaders: 1.149049e-01 +Nonblocking Multileader Locality, 10 leaders: 3.409344e-04 +Pairwise Multileader, 20 leaders: 5.794615e-03 +Pairwise Locality Aware, 20 leaders: 6.762160e-03 +Pairwise Multileader Locality, 20 leaders: 5.061263e-04 +Nonblocking Multileader, 20 leaders: 3.765589e-02 +Nonblocking Locality Aware, 20 leaders: 1.290762e-01 +Nonblocking Multileader Locality, 20 leaders: 2.776029e-04 +Size 2 +PMPI: 1.136952e-03 +Pairwise: 1.490608e-02 +NonBlocking: 2.494882e-01 +Pairwise Hierarchical: 5.422534e-03 +Pairwise Node Aware: 7.923166e-04 +Nonblocking Hierarchical: 4.337324e-03 +Nonblocking Node Aware: 6.698864e-04 +Pairwise Multileader, 4 leaders: 3.054540e-03 +Pairwise Locality Aware, 4 leaders: 1.477504e-03 +Pairwise Multileader Locality, 4 leaders: 1.866141e-03 +Nonblocking Multileader, 4 leaders: 1.138679e-03 +Nonblocking Locality Aware, 4 leaders: 7.457218e-02 +Nonblocking Multileader Locality, 4 leaders: 1.292682e-03 +Pairwise Multileader, 10 leaders: 3.978316e-03 +Pairwise Locality Aware, 10 leaders: 3.977010e-03 +Pairwise Multileader Locality, 10 leaders: 1.199690e-03 +Nonblocking Multileader, 10 leaders: 1.941707e-02 +Nonblocking Locality Aware, 10 leaders: 1.152928e-01 +Nonblocking Multileader Locality, 10 leaders: 5.327882e-04 +Pairwise Multileader, 20 leaders: 5.090752e-03 +Pairwise Locality Aware, 20 leaders: 1.042119e-02 +Pairwise Multileader Locality, 20 leaders: 6.041415e-04 +Nonblocking Multileader, 20 leaders: 4.001906e-02 +Nonblocking Locality Aware, 20 leaders: 2.069069e-01 +Nonblocking Multileader Locality, 20 leaders: 4.277256e-04 +Size 4 +PMPI: 1.706553e-03 +Pairwise: 4.388700e-02 +NonBlocking: 2.736794e-01 +Pairwise Hierarchical: 1.196444e-02 +Pairwise Node Aware: 1.410989e-03 +Nonblocking Hierarchical: 8.575057e-03 +Nonblocking Node Aware: 9.812685e-04 +Pairwise Multileader, 4 leaders: 8.227432e-03 +Pairwise Locality Aware, 4 leaders: 3.790544e-03 +Pairwise Multileader Locality, 4 leaders: 3.674488e-03 +Nonblocking Multileader, 4 leaders: 2.153895e-03 +Nonblocking Locality Aware, 4 leaders: 9.945404e-02 +Nonblocking Multileader Locality, 4 leaders: 2.546663e-03 +Pairwise Multileader, 10 leaders: 1.059246e-02 +Pairwise Locality Aware, 10 leaders: 1.022897e-02 +Pairwise Multileader Locality, 10 leaders: 2.173046e-03 +Nonblocking Multileader, 10 leaders: 8.718294e-02 +Nonblocking Locality Aware, 10 leaders: 1.152254e-01 +Nonblocking Multileader Locality, 10 leaders: 1.147792e-03 +Pairwise Multileader, 20 leaders: 1.695338e-02 +Pairwise Locality Aware, 20 leaders: 1.730273e-02 +Pairwise Multileader Locality, 20 leaders: 9.048466e-04 +Nonblocking Multileader, 20 leaders: 6.039809e-02 +Nonblocking Locality Aware, 20 leaders: 1.610323e-01 +Nonblocking Multileader Locality, 20 leaders: 5.657027e-04 +Size 8 +PMPI: 3.580745e-03 +Pairwise: 3.798893e-02 +NonBlocking: 2.890695e-01 +Pairwise Hierarchical: 2.201014e-02 +Pairwise Node Aware: 1.444841e-03 +Nonblocking Hierarchical: 1.737096e-02 +Nonblocking Node Aware: 1.386463e-03 +Pairwise Multileader, 4 leaders: 6.320855e-03 +Pairwise Locality Aware, 4 leaders: 3.752654e-03 +Pairwise Multileader Locality, 4 leaders: 7.002682e-03 +Nonblocking Multileader, 4 leaders: 4.211427e-03 +Nonblocking Locality Aware, 4 leaders: 8.712853e-02 +Nonblocking Multileader Locality, 4 leaders: 5.206797e-03 +Pairwise Multileader, 10 leaders: 4.170043e-03 +Pairwise Locality Aware, 10 leaders: 4.152968e-03 +Pairwise Multileader Locality, 10 leaders: 2.485704e-03 +Nonblocking Multileader, 10 leaders: 1.249901e-02 +Nonblocking Locality Aware, 10 leaders: 1.155327e-01 +Nonblocking Multileader Locality, 10 leaders: 1.902587e-03 +Pairwise Multileader, 20 leaders: 6.595439e-03 +Pairwise Locality Aware, 20 leaders: 6.460215e-03 +Pairwise Multileader Locality, 20 leaders: 1.569993e-03 +Nonblocking Multileader, 20 leaders: 7.494860e-02 +Nonblocking Locality Aware, 20 leaders: 1.384538e-01 +Nonblocking Multileader Locality, 20 leaders: 1.224053e-03 +Size 16 +PMPI: 5.168513e-03 +Pairwise: 2.861982e-02 +NonBlocking: 3.678005e-01 +Pairwise Hierarchical: 4.523162e-02 +Pairwise Node Aware: 2.439462e-03 +Nonblocking Hierarchical: 3.531213e-02 +Nonblocking Node Aware: 2.425345e-03 +Pairwise Multileader, 4 leaders: 9.118229e-03 +Pairwise Locality Aware, 4 leaders: 2.709092e-03 +Pairwise Multileader Locality, 4 leaders: 1.205094e-02 +Nonblocking Multileader, 4 leaders: 6.788515e-03 +Nonblocking Locality Aware, 4 leaders: 9.343992e-02 +Nonblocking Multileader Locality, 4 leaders: 8.757169e-03 +Pairwise Multileader, 10 leaders: 7.350831e-03 +Pairwise Locality Aware, 10 leaders: 4.287343e-03 +Pairwise Multileader Locality, 10 leaders: 4.525849e-03 +Nonblocking Multileader, 10 leaders: 9.890930e-03 +Nonblocking Locality Aware, 10 leaders: 1.159931e-01 +Nonblocking Multileader Locality, 10 leaders: 3.662600e-03 +Pairwise Multileader, 20 leaders: 7.755386e-03 +Pairwise Locality Aware, 20 leaders: 5.960379e-03 +Pairwise Multileader Locality, 20 leaders: 2.877322e-03 +Nonblocking Multileader, 20 leaders: 7.569386e-02 +Nonblocking Locality Aware, 20 leaders: 1.621004e-01 +Nonblocking Multileader Locality, 20 leaders: 2.195058e-03 +Size 32 +PMPI: 9.898644e-03 +Pairwise: 3.049728e-02 +NonBlocking: 2.729162e-01 +Pairwise Hierarchical: 8.666724e-02 +Pairwise Node Aware: 4.488654e-03 +Nonblocking Hierarchical: 6.596692e-02 +Nonblocking Node Aware: 5.578833e-03 +Pairwise Multileader, 4 leaders: 2.845952e-02 +Pairwise Locality Aware, 4 leaders: 4.197737e-03 +Pairwise Multileader Locality, 4 leaders: 2.446960e-02 +Nonblocking Multileader, 4 leaders: 1.818761e-02 +Nonblocking Locality Aware, 4 leaders: 4.739432e-02 +Nonblocking Multileader Locality, 4 leaders: 1.809747e-02 +Pairwise Multileader, 10 leaders: 1.349077e-02 +Pairwise Locality Aware, 10 leaders: 6.174485e-03 +Pairwise Multileader Locality, 10 leaders: 1.055996e-02 +Nonblocking Multileader, 10 leaders: 2.214983e-02 +Nonblocking Locality Aware, 10 leaders: 1.403371e-01 +Nonblocking Multileader Locality, 10 leaders: 8.527703e-03 +Pairwise Multileader, 20 leaders: 1.023795e-02 +Pairwise Locality Aware, 20 leaders: 9.283543e-03 +Pairwise Multileader Locality, 20 leaders: 5.334123e-03 +Nonblocking Multileader, 20 leaders: 8.337080e-02 +Nonblocking Locality Aware, 20 leaders: 1.314348e-01 +Nonblocking Multileader Locality, 20 leaders: 4.273648e-03 +Size 64 +PMPI: 1.992373e-02 +Pairwise: 3.611086e-02 +NonBlocking: 2.126182e-01 +Pairwise Hierarchical: 1.652282e-01 +Pairwise Node Aware: 9.539057e-03 +Nonblocking Hierarchical: 1.256537e-01 +Nonblocking Node Aware: 7.876835e-03 +Pairwise Multileader, 4 leaders: 5.353398e-02 +Pairwise Locality Aware, 4 leaders: 8.250266e-03 +Pairwise Multileader Locality, 4 leaders: 5.700351e-02 +Nonblocking Multileader, 4 leaders: 3.790313e-02 +Nonblocking Locality Aware, 4 leaders: 1.154428e-02 +Nonblocking Multileader Locality, 4 leaders: 4.236704e-02 +Pairwise Multileader, 10 leaders: 2.565653e-02 +Pairwise Locality Aware, 10 leaders: 9.504089e-03 +Pairwise Multileader Locality, 10 leaders: 2.557278e-02 +Nonblocking Multileader, 10 leaders: 1.822439e-02 +Nonblocking Locality Aware, 10 leaders: 1.404149e-01 +Nonblocking Multileader Locality, 10 leaders: 1.999657e-02 +Pairwise Multileader, 20 leaders: 1.912417e-02 +Pairwise Locality Aware, 20 leaders: 1.523557e-02 +Pairwise Multileader Locality, 20 leaders: 1.253552e-02 +Nonblocking Multileader, 20 leaders: 2.547345e-02 +Nonblocking Locality Aware, 20 leaders: 1.398961e-01 +Nonblocking Multileader Locality, 20 leaders: 9.917599e-03 +Size 128 +PMPI: 3.696981e-02 +Pairwise: 5.808048e-02 +NonBlocking: 3.337874e-01 +Pairwise Hierarchical: 3.226439e-01 +Pairwise Node Aware: 1.997665e-02 +Nonblocking Hierarchical: 2.512015e-01 +Nonblocking Node Aware: 1.764326e-02 +Pairwise Multileader, 4 leaders: 9.916575e-02 +Pairwise Locality Aware, 4 leaders: 1.541551e-02 +Pairwise Multileader Locality, 4 leaders: 1.092702e-01 +Nonblocking Multileader, 4 leaders: 6.903268e-02 +Nonblocking Locality Aware, 4 leaders: 2.307796e-02 +Nonblocking Multileader Locality, 4 leaders: 7.974140e-02 +Pairwise Multileader, 10 leaders: 5.390377e-02 +Pairwise Locality Aware, 10 leaders: 2.063304e-02 +Pairwise Multileader Locality, 10 leaders: 5.415763e-02 +Nonblocking Multileader, 10 leaders: 1.149149e-01 +Nonblocking Locality Aware, 10 leaders: 1.108698e-01 +Nonblocking Multileader Locality, 10 leaders: 4.154784e-02 +Pairwise Multileader, 20 leaders: 3.370081e-02 +Pairwise Locality Aware, 20 leaders: 1.955607e-02 +Pairwise Multileader Locality, 20 leaders: 2.798589e-02 +Nonblocking Multileader, 20 leaders: 2.197279e-02 +Nonblocking Locality Aware, 20 leaders: 1.426132e-01 +Nonblocking Multileader Locality, 20 leaders: 2.091976e-02 +Size 256 +PMPI: 8.098072e-02 +Pairwise: 6.351349e-02 +NonBlocking: 2.417460e-01 +Pairwise Hierarchical: 6.525288e-01 +Pairwise Node Aware: 2.991483e-02 +Nonblocking Hierarchical: 5.426058e-01 +Nonblocking Node Aware: 3.018074e-02 +Pairwise Multileader, 4 leaders: 1.867841e-01 +Pairwise Locality Aware, 4 leaders: 3.283386e-02 +Pairwise Multileader Locality, 4 leaders: 2.094131e-01 +Nonblocking Multileader, 4 leaders: 1.312283e-01 +Nonblocking Locality Aware, 4 leaders: 2.616844e-02 +Nonblocking Multileader Locality, 4 leaders: 1.561987e-01 +Pairwise Multileader, 10 leaders: 1.101976e-01 +Pairwise Locality Aware, 10 leaders: 3.734946e-02 +Pairwise Multileader Locality, 10 leaders: 1.094481e-01 +Nonblocking Multileader, 10 leaders: 7.594870e-02 +Nonblocking Locality Aware, 10 leaders: 7.499981e-02 +Nonblocking Multileader Locality, 10 leaders: 8.853461e-02 +Pairwise Multileader, 20 leaders: 6.885137e-02 +Pairwise Locality Aware, 20 leaders: 3.808340e-02 +Pairwise Multileader Locality, 20 leaders: 5.736306e-02 +Nonblocking Multileader, 20 leaders: 2.502910e-01 +Nonblocking Locality Aware, 20 leaders: 1.179084e-01 +Nonblocking Multileader Locality, 20 leaders: 4.389479e-02 +Size 512 +PMPI: 1.338927e-01 +Pairwise: 9.152178e-02 +NonBlocking: 3.140708e-01 +Pairwise Hierarchical: 1.275606e+00 +Pairwise Node Aware: 6.769870e-02 +Nonblocking Hierarchical: 1.135458e+00 +Nonblocking Node Aware: 5.205033e-02 +Pairwise Multileader, 4 leaders: 3.469404e-01 +Pairwise Locality Aware, 4 leaders: 7.394860e-02 +Pairwise Multileader Locality, 4 leaders: 4.009446e-01 +Nonblocking Multileader, 4 leaders: 2.594503e-01 +Nonblocking Locality Aware, 4 leaders: 6.715104e-02 +Nonblocking Multileader Locality, 4 leaders: 3.058266e-01 +Pairwise Multileader, 10 leaders: 2.018932e-01 +Pairwise Locality Aware, 10 leaders: 7.969077e-02 +Pairwise Multileader Locality, 10 leaders: 2.123373e-01 +Nonblocking Multileader, 10 leaders: 1.434208e-01 +Nonblocking Locality Aware, 10 leaders: 6.430903e-02 +Nonblocking Multileader Locality, 10 leaders: 1.719797e-01 +Pairwise Multileader, 20 leaders: 1.247628e-01 +Pairwise Locality Aware, 20 leaders: 7.300578e-02 +Pairwise Multileader Locality, 20 leaders: 1.263595e-01 +Nonblocking Multileader, 20 leaders: 2.645164e-01 +Nonblocking Locality Aware, 20 leaders: 1.119443e-01 +Nonblocking Multileader Locality, 20 leaders: 1.067928e-01 +Test 3 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.234714e-03 +Pairwise: 1.813855e-02 +NonBlocking: 4.063549e-01 +Pairwise Hierarchical: 3.405104e-03 +Pairwise Node Aware: 1.878680e-03 +Nonblocking Hierarchical: 2.237586e-03 +Nonblocking Node Aware: 4.456697e-04 +Pairwise Multileader, 4 leaders: 2.120114e-03 +Pairwise Locality Aware, 4 leaders: 1.797320e-03 +Pairwise Multileader Locality, 4 leaders: 1.122167e-03 +Nonblocking Multileader, 4 leaders: 1.611067e-03 +Nonblocking Locality Aware, 4 leaders: 8.681710e-02 +Nonblocking Multileader Locality, 4 leaders: 7.584473e-04 +Pairwise Multileader, 10 leaders: 6.230415e-03 +Pairwise Locality Aware, 10 leaders: 6.958339e-03 +Pairwise Multileader Locality, 10 leaders: 1.133648e-03 +Nonblocking Multileader, 10 leaders: 6.206701e-02 +Nonblocking Locality Aware, 10 leaders: 1.151409e-01 +Nonblocking Multileader Locality, 10 leaders: 4.516329e-04 +Pairwise Multileader, 20 leaders: 1.718198e-02 +Pairwise Locality Aware, 20 leaders: 1.463431e-02 +Pairwise Multileader Locality, 20 leaders: 1.063076e-03 +Nonblocking Multileader, 20 leaders: 4.997563e-02 +Nonblocking Locality Aware, 20 leaders: 1.613421e-01 +Nonblocking Multileader Locality, 20 leaders: 4.070015e-04 +Size 2 +PMPI: 1.426542e-03 +Pairwise: 5.087793e-02 +NonBlocking: 4.404482e-01 +Pairwise Hierarchical: 7.591308e-03 +Pairwise Node Aware: 8.821459e-04 +Nonblocking Hierarchical: 5.268567e-03 +Nonblocking Node Aware: 7.036788e-04 +Pairwise Multileader, 4 leaders: 2.890253e-03 +Pairwise Locality Aware, 4 leaders: 1.833530e-03 +Pairwise Multileader Locality, 4 leaders: 1.813347e-03 +Nonblocking Multileader, 4 leaders: 1.102029e-03 +Nonblocking Locality Aware, 4 leaders: 7.440837e-02 +Nonblocking Multileader Locality, 4 leaders: 1.382591e-03 +Pairwise Multileader, 10 leaders: 4.076867e-03 +Pairwise Locality Aware, 10 leaders: 4.048830e-03 +Pairwise Multileader Locality, 10 leaders: 1.063590e-03 +Nonblocking Multileader, 10 leaders: 2.660982e-02 +Nonblocking Locality Aware, 10 leaders: 1.152139e-01 +Nonblocking Multileader Locality, 10 leaders: 5.530828e-04 +Pairwise Multileader, 20 leaders: 8.063114e-03 +Pairwise Locality Aware, 20 leaders: 8.641598e-03 +Pairwise Multileader Locality, 20 leaders: 1.046152e-03 +Nonblocking Multileader, 20 leaders: 4.999010e-02 +Nonblocking Locality Aware, 20 leaders: 1.150469e-01 +Nonblocking Multileader Locality, 20 leaders: 3.994596e-04 +Size 4 +PMPI: 1.786199e-03 +Pairwise: 3.482356e-02 +NonBlocking: 3.257218e-01 +Pairwise Hierarchical: 1.098767e-02 +Pairwise Node Aware: 9.130507e-04 +Nonblocking Hierarchical: 8.885474e-03 +Nonblocking Node Aware: 7.556043e-04 +Pairwise Multileader, 4 leaders: 3.684799e-03 +Pairwise Locality Aware, 4 leaders: 1.778684e-03 +Pairwise Multileader Locality, 4 leaders: 3.237732e-03 +Nonblocking Multileader, 4 leaders: 4.167278e-03 +Nonblocking Locality Aware, 4 leaders: 6.210787e-02 +Nonblocking Multileader Locality, 4 leaders: 2.476600e-03 +Pairwise Multileader, 10 leaders: 4.495267e-03 +Pairwise Locality Aware, 10 leaders: 3.295391e-03 +Pairwise Multileader Locality, 10 leaders: 1.535796e-03 +Nonblocking Multileader, 10 leaders: 5.006986e-02 +Nonblocking Locality Aware, 10 leaders: 1.152079e-01 +Nonblocking Multileader Locality, 10 leaders: 1.114107e-03 +Pairwise Multileader, 20 leaders: 7.663934e-03 +Pairwise Locality Aware, 20 leaders: 5.227736e-03 +Pairwise Multileader Locality, 20 leaders: 9.045816e-04 +Nonblocking Multileader, 20 leaders: 3.784611e-02 +Nonblocking Locality Aware, 20 leaders: 1.380958e-01 +Nonblocking Multileader Locality, 20 leaders: 6.382478e-04 +Size 8 +PMPI: 3.598555e-03 +Pairwise: 2.786876e-02 +NonBlocking: 4.492108e-01 +Pairwise Hierarchical: 2.293298e-02 +Pairwise Node Aware: 1.411840e-03 +Nonblocking Hierarchical: 1.788187e-02 +Nonblocking Node Aware: 1.369722e-03 +Pairwise Multileader, 4 leaders: 5.974550e-03 +Pairwise Locality Aware, 4 leaders: 2.399434e-03 +Pairwise Multileader Locality, 4 leaders: 7.506110e-03 +Nonblocking Multileader, 4 leaders: 4.214370e-03 +Nonblocking Locality Aware, 4 leaders: 8.719723e-02 +Nonblocking Multileader Locality, 4 leaders: 5.136829e-03 +Pairwise Multileader, 10 leaders: 5.630392e-03 +Pairwise Locality Aware, 10 leaders: 6.558477e-03 +Pairwise Multileader Locality, 10 leaders: 2.522685e-03 +Nonblocking Multileader, 10 leaders: 1.874337e-03 +Nonblocking Locality Aware, 10 leaders: 1.154144e-01 +Nonblocking Multileader Locality, 10 leaders: 2.018877e-03 +Pairwise Multileader, 20 leaders: 8.362600e-03 +Pairwise Locality Aware, 20 leaders: 7.098438e-03 +Pairwise Multileader Locality, 20 leaders: 1.653315e-03 +Nonblocking Multileader, 20 leaders: 7.817509e-02 +Nonblocking Locality Aware, 20 leaders: 1.159039e-01 +Nonblocking Multileader Locality, 20 leaders: 1.314798e-03 +Size 16 +PMPI: 5.279147e-03 +Pairwise: 2.795232e-02 +NonBlocking: 2.881728e-01 +Pairwise Hierarchical: 4.604246e-02 +Pairwise Node Aware: 2.358464e-03 +Nonblocking Hierarchical: 3.537624e-02 +Nonblocking Node Aware: 2.323546e-03 +Pairwise Multileader, 4 leaders: 9.273648e-03 +Pairwise Locality Aware, 4 leaders: 2.569171e-03 +Pairwise Multileader Locality, 4 leaders: 1.214817e-02 +Nonblocking Multileader, 4 leaders: 6.923023e-03 +Nonblocking Locality Aware, 4 leaders: 1.003337e-01 +Nonblocking Multileader Locality, 4 leaders: 9.017997e-03 +Pairwise Multileader, 10 leaders: 7.750863e-03 +Pairwise Locality Aware, 10 leaders: 4.810063e-03 +Pairwise Multileader Locality, 10 leaders: 4.685395e-03 +Nonblocking Multileader, 10 leaders: 1.436395e-02 +Nonblocking Locality Aware, 10 leaders: 9.362847e-02 +Nonblocking Multileader Locality, 10 leaders: 3.444025e-03 +Pairwise Multileader, 20 leaders: 1.592299e-02 +Pairwise Locality Aware, 20 leaders: 6.869191e-03 +Pairwise Multileader Locality, 20 leaders: 3.039995e-03 +Nonblocking Multileader, 20 leaders: 8.784375e-02 +Nonblocking Locality Aware, 20 leaders: 1.623418e-01 +Nonblocking Multileader Locality, 20 leaders: 2.456158e-03 +Size 32 +PMPI: 1.067517e-02 +Pairwise: 2.726851e-02 +NonBlocking: 2.477988e-01 +Pairwise Hierarchical: 8.876392e-02 +Pairwise Node Aware: 4.348235e-03 +Nonblocking Hierarchical: 6.585434e-02 +Nonblocking Node Aware: 4.868353e-03 +Pairwise Multileader, 4 leaders: 2.871835e-02 +Pairwise Locality Aware, 4 leaders: 4.604423e-03 +Pairwise Multileader Locality, 4 leaders: 2.417299e-02 +Nonblocking Multileader, 4 leaders: 1.818614e-02 +Nonblocking Locality Aware, 4 leaders: 6.529984e-02 +Nonblocking Multileader Locality, 4 leaders: 1.826898e-02 +Pairwise Multileader, 10 leaders: 1.385997e-02 +Pairwise Locality Aware, 10 leaders: 5.528734e-03 +Pairwise Multileader Locality, 10 leaders: 1.065255e-02 +Nonblocking Multileader, 10 leaders: 2.095986e-02 +Nonblocking Locality Aware, 10 leaders: 1.405510e-01 +Nonblocking Multileader Locality, 10 leaders: 8.721678e-03 +Pairwise Multileader, 20 leaders: 1.128052e-02 +Pairwise Locality Aware, 20 leaders: 9.534064e-03 +Pairwise Multileader Locality, 20 leaders: 5.368642e-03 +Nonblocking Multileader, 20 leaders: 7.422410e-02 +Nonblocking Locality Aware, 20 leaders: 1.400187e-01 +Nonblocking Multileader Locality, 20 leaders: 4.382139e-03 +Size 64 +PMPI: 1.943283e-02 +Pairwise: 3.394305e-02 +NonBlocking: 2.752756e-01 +Pairwise Hierarchical: 1.671348e-01 +Pairwise Node Aware: 1.003882e-02 +Nonblocking Hierarchical: 1.269355e-01 +Nonblocking Node Aware: 7.840588e-03 +Pairwise Multileader, 4 leaders: 5.482957e-02 +Pairwise Locality Aware, 4 leaders: 7.037976e-03 +Pairwise Multileader Locality, 4 leaders: 5.679952e-02 +Nonblocking Multileader, 4 leaders: 3.735480e-02 +Nonblocking Locality Aware, 4 leaders: 8.668232e-03 +Nonblocking Multileader Locality, 4 leaders: 4.218524e-02 +Pairwise Multileader, 10 leaders: 2.629750e-02 +Pairwise Locality Aware, 10 leaders: 1.024763e-02 +Pairwise Multileader Locality, 10 leaders: 2.517583e-02 +Nonblocking Multileader, 10 leaders: 1.821982e-02 +Nonblocking Locality Aware, 10 leaders: 1.199324e-01 +Nonblocking Multileader Locality, 10 leaders: 2.014013e-02 +Pairwise Multileader, 20 leaders: 2.056589e-02 +Pairwise Locality Aware, 20 leaders: 1.251046e-02 +Pairwise Multileader Locality, 20 leaders: 1.237084e-02 +Nonblocking Multileader, 20 leaders: 2.287205e-02 +Nonblocking Locality Aware, 20 leaders: 1.407193e-01 +Nonblocking Multileader Locality, 20 leaders: 1.011123e-02 +Size 128 +PMPI: 3.907016e-02 +Pairwise: 4.555581e-02 +NonBlocking: 1.783218e-01 +Pairwise Hierarchical: 3.229204e-01 +Pairwise Node Aware: 1.872521e-02 +Nonblocking Hierarchical: 2.478994e-01 +Nonblocking Node Aware: 1.702685e-02 +Pairwise Multileader, 4 leaders: 9.905044e-02 +Pairwise Locality Aware, 4 leaders: 1.403495e-02 +Pairwise Multileader Locality, 4 leaders: 1.067992e-01 +Nonblocking Multileader, 4 leaders: 6.890386e-02 +Nonblocking Locality Aware, 4 leaders: 2.241835e-02 +Nonblocking Multileader Locality, 4 leaders: 7.895301e-02 +Pairwise Multileader, 10 leaders: 4.981706e-02 +Pairwise Locality Aware, 10 leaders: 1.692045e-02 +Pairwise Multileader Locality, 10 leaders: 5.304998e-02 +Nonblocking Multileader, 10 leaders: 1.130536e-01 +Nonblocking Locality Aware, 10 leaders: 9.437795e-02 +Nonblocking Multileader Locality, 10 leaders: 4.176786e-02 +Pairwise Multileader, 20 leaders: 3.442168e-02 +Pairwise Locality Aware, 20 leaders: 1.819354e-02 +Pairwise Multileader Locality, 20 leaders: 2.958947e-02 +Nonblocking Multileader, 20 leaders: 2.727464e-02 +Nonblocking Locality Aware, 20 leaders: 1.127430e-01 +Nonblocking Multileader Locality, 20 leaders: 2.114705e-02 +Size 256 +PMPI: 7.961389e-02 +Pairwise: 4.989072e-02 +NonBlocking: 1.882523e-01 +Pairwise Hierarchical: 6.565939e-01 +Pairwise Node Aware: 3.074332e-02 +Nonblocking Hierarchical: 5.430450e-01 +Nonblocking Node Aware: 3.220649e-02 +Pairwise Multileader, 4 leaders: 1.872801e-01 +Pairwise Locality Aware, 4 leaders: 3.827021e-02 +Pairwise Multileader Locality, 4 leaders: 2.093093e-01 +Nonblocking Multileader, 4 leaders: 1.315565e-01 +Nonblocking Locality Aware, 4 leaders: 2.678087e-02 +Nonblocking Multileader Locality, 4 leaders: 1.572783e-01 +Pairwise Multileader, 10 leaders: 1.376543e-01 +Pairwise Locality Aware, 10 leaders: 4.612104e-02 +Pairwise Multileader Locality, 10 leaders: 1.179889e-01 +Nonblocking Multileader, 10 leaders: 7.709146e-02 +Nonblocking Locality Aware, 10 leaders: 9.958092e-02 +Nonblocking Multileader Locality, 10 leaders: 9.026976e-02 +Pairwise Multileader, 20 leaders: 9.653367e-02 +Pairwise Locality Aware, 20 leaders: 5.663213e-02 +Pairwise Multileader Locality, 20 leaders: 5.947858e-02 +Nonblocking Multileader, 20 leaders: 1.160540e-01 +Nonblocking Locality Aware, 20 leaders: 1.561686e-01 +Nonblocking Multileader Locality, 20 leaders: 4.430919e-02 +Size 512 +PMPI: 1.905947e-01 +Pairwise: 9.889737e-02 +NonBlocking: 3.030962e-01 +Pairwise Hierarchical: 1.281593e+00 +Pairwise Node Aware: 6.003220e-02 +Nonblocking Hierarchical: 1.139340e+00 +Nonblocking Node Aware: 5.257150e-02 +Pairwise Multileader, 4 leaders: 3.465619e-01 +Pairwise Locality Aware, 4 leaders: 7.560847e-02 +Pairwise Multileader Locality, 4 leaders: 4.055356e-01 +Nonblocking Multileader, 4 leaders: 2.589835e-01 +Nonblocking Locality Aware, 4 leaders: 7.010552e-02 +Nonblocking Multileader Locality, 4 leaders: 3.085235e-01 +Pairwise Multileader, 10 leaders: 2.115237e-01 +Pairwise Locality Aware, 10 leaders: 8.440185e-02 +Pairwise Multileader Locality, 10 leaders: 2.147730e-01 +Nonblocking Multileader, 10 leaders: 1.439262e-01 +Nonblocking Locality Aware, 10 leaders: 6.953199e-02 +Nonblocking Multileader Locality, 10 leaders: 1.718604e-01 +Pairwise Multileader, 20 leaders: 1.411355e-01 +Pairwise Locality Aware, 20 leaders: 8.530498e-02 +Pairwise Multileader Locality, 20 leaders: 1.260233e-01 +Nonblocking Multileader, 20 leaders: 1.996876e-01 +Nonblocking Locality Aware, 20 leaders: 1.416455e-01 +Nonblocking Multileader Locality, 20 leaders: 1.080314e-01 diff --git a/runscripts/dane_results/alltoall_N32.2484415.err b/runscripts/dane_results/alltoall_N32.2484415.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/dane_results/alltoall_N32.2484415.out b/runscripts/dane_results/alltoall_N32.2484415.out new file mode 100644 index 000000000..e44ebfacd --- /dev/null +++ b/runscripts/dane_results/alltoall_N32.2484415.out @@ -0,0 +1,783 @@ +Test 1 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.031179e-03 +Pairwise: 1.423542e-02 +NonBlocking: 4.853996e-01 +Pairwise Hierarchical: 2.565957e-03 +Pairwise Node Aware: 7.295385e-04 +Nonblocking Hierarchical: 2.040944e-03 +Nonblocking Node Aware: 4.419680e-04 +Pairwise Multileader, 4 leaders: 2.097048e-03 +Pairwise Locality Aware, 4 leaders: 1.370589e-03 +Pairwise Multileader Locality, 4 leaders: 9.651171e-04 +Nonblocking Multileader, 4 leaders: 1.132771e-03 +Nonblocking Locality Aware, 4 leaders: 7.421906e-02 +Nonblocking Multileader Locality, 4 leaders: 6.783219e-04 +Pairwise Multileader, 10 leaders: 4.086917e-03 +Pairwise Locality Aware, 10 leaders: 2.660960e-03 +Pairwise Multileader Locality, 10 leaders: 5.499018e-04 +Nonblocking Multileader, 10 leaders: 2.106273e-02 +Nonblocking Locality Aware, 10 leaders: 1.114369e-01 +Nonblocking Multileader Locality, 10 leaders: 3.509550e-04 +Pairwise Multileader, 20 leaders: 7.329413e-03 +Pairwise Locality Aware, 20 leaders: 8.060639e-03 +Pairwise Multileader Locality, 20 leaders: 4.534260e-04 +Nonblocking Multileader, 20 leaders: 7.451927e-02 +Nonblocking Locality Aware, 20 leaders: 1.152374e-01 +Nonblocking Multileader Locality, 20 leaders: 2.430965e-04 +Size 2 +PMPI: 1.214682e-03 +Pairwise: 1.769957e-02 +NonBlocking: 3.306004e-01 +Pairwise Hierarchical: 5.522237e-03 +Pairwise Node Aware: 7.964693e-04 +Nonblocking Hierarchical: 4.374746e-03 +Nonblocking Node Aware: 5.938556e-04 +Pairwise Multileader, 4 leaders: 2.631934e-03 +Pairwise Locality Aware, 4 leaders: 2.004768e-03 +Pairwise Multileader Locality, 4 leaders: 1.702309e-03 +Nonblocking Multileader, 4 leaders: 1.851213e-03 +Nonblocking Locality Aware, 4 leaders: 9.903339e-02 +Nonblocking Multileader Locality, 4 leaders: 1.272158e-03 +Pairwise Multileader, 10 leaders: 4.590704e-03 +Pairwise Locality Aware, 10 leaders: 2.775183e-03 +Pairwise Multileader Locality, 10 leaders: 8.293301e-04 +Nonblocking Multileader, 10 leaders: 2.062778e-02 +Nonblocking Locality Aware, 10 leaders: 9.232804e-02 +Nonblocking Multileader Locality, 10 leaders: 5.353255e-04 +Pairwise Multileader, 20 leaders: 4.813585e-03 +Pairwise Locality Aware, 20 leaders: 6.429438e-03 +Pairwise Multileader Locality, 20 leaders: 6.493383e-04 +Nonblocking Multileader, 20 leaders: 5.768163e-02 +Nonblocking Locality Aware, 20 leaders: 1.151810e-01 +Nonblocking Multileader Locality, 20 leaders: 3.854015e-04 +Size 4 +PMPI: 1.810865e-03 +Pairwise: 2.245715e-02 +NonBlocking: 3.295868e-01 +Pairwise Hierarchical: 1.023727e-02 +Pairwise Node Aware: 9.630813e-04 +Nonblocking Hierarchical: 8.540200e-03 +Nonblocking Node Aware: 7.966090e-04 +Pairwise Multileader, 4 leaders: 3.327262e-03 +Pairwise Locality Aware, 4 leaders: 3.232234e-03 +Pairwise Multileader Locality, 4 leaders: 3.363403e-03 +Nonblocking Multileader, 4 leaders: 1.294524e-02 +Nonblocking Locality Aware, 4 leaders: 9.218546e-02 +Nonblocking Multileader Locality, 4 leaders: 2.463384e-03 +Pairwise Multileader, 10 leaders: 9.036964e-03 +Pairwise Locality Aware, 10 leaders: 7.455059e-03 +Pairwise Multileader Locality, 10 leaders: 2.029084e-03 +Nonblocking Multileader, 10 leaders: 5.024965e-02 +Nonblocking Locality Aware, 10 leaders: 1.152953e-01 +Nonblocking Multileader Locality, 10 leaders: 1.178286e-03 +Pairwise Multileader, 20 leaders: 2.085590e-02 +Pairwise Locality Aware, 20 leaders: 1.218094e-02 +Pairwise Multileader Locality, 20 leaders: 9.781195e-04 +Nonblocking Multileader, 20 leaders: 5.017704e-02 +Nonblocking Locality Aware, 20 leaders: 1.384876e-01 +Nonblocking Multileader Locality, 20 leaders: 7.958990e-04 +Size 8 +PMPI: 3.477955e-03 +Pairwise: 3.676530e-02 +NonBlocking: 6.473124e-01 +Pairwise Hierarchical: 2.221072e-02 +Pairwise Node Aware: 1.317292e-03 +Nonblocking Hierarchical: 1.723427e-02 +Nonblocking Node Aware: 1.202292e-03 +Pairwise Multileader, 4 leaders: 5.988585e-03 +Pairwise Locality Aware, 4 leaders: 2.587360e-03 +Pairwise Multileader Locality, 4 leaders: 7.159776e-03 +Nonblocking Multileader, 4 leaders: 4.227809e-03 +Nonblocking Locality Aware, 4 leaders: 8.731925e-02 +Nonblocking Multileader Locality, 4 leaders: 5.270421e-03 +Pairwise Multileader, 10 leaders: 4.519497e-03 +Pairwise Locality Aware, 10 leaders: 4.705880e-03 +Pairwise Multileader Locality, 10 leaders: 2.532113e-03 +Nonblocking Multileader, 10 leaders: 1.167645e-02 +Nonblocking Locality Aware, 10 leaders: 1.614462e-01 +Nonblocking Multileader Locality, 10 leaders: 1.814193e-03 +Pairwise Multileader, 20 leaders: 8.577405e-03 +Pairwise Locality Aware, 20 leaders: 6.365649e-03 +Pairwise Multileader Locality, 20 leaders: 1.543157e-03 +Nonblocking Multileader, 20 leaders: 9.934121e-02 +Nonblocking Locality Aware, 20 leaders: 1.615294e-01 +Nonblocking Multileader Locality, 20 leaders: 1.114722e-03 +Size 16 +PMPI: 5.818204e-03 +Pairwise: 3.258785e-02 +NonBlocking: 3.683681e-01 +Pairwise Hierarchical: 4.535231e-02 +Pairwise Node Aware: 2.616817e-03 +Nonblocking Hierarchical: 3.471196e-02 +Nonblocking Node Aware: 2.486801e-03 +Pairwise Multileader, 4 leaders: 9.045804e-03 +Pairwise Locality Aware, 4 leaders: 2.946874e-03 +Pairwise Multileader Locality, 4 leaders: 1.161735e-02 +Nonblocking Multileader, 4 leaders: 6.848948e-03 +Nonblocking Locality Aware, 4 leaders: 9.334343e-02 +Nonblocking Multileader Locality, 4 leaders: 8.642267e-03 +Pairwise Multileader, 10 leaders: 8.206034e-03 +Pairwise Locality Aware, 10 leaders: 4.395331e-03 +Pairwise Multileader Locality, 10 leaders: 4.602456e-03 +Nonblocking Multileader, 10 leaders: 1.250884e-02 +Nonblocking Locality Aware, 10 leaders: 1.623596e-01 +Nonblocking Multileader Locality, 10 leaders: 3.740556e-03 +Pairwise Multileader, 20 leaders: 7.989893e-03 +Pairwise Locality Aware, 20 leaders: 1.129892e-02 +Pairwise Multileader Locality, 20 leaders: 3.327143e-03 +Nonblocking Multileader, 20 leaders: 8.419139e-02 +Nonblocking Locality Aware, 20 leaders: 1.617583e-01 +Nonblocking Multileader Locality, 20 leaders: 2.324365e-03 +Size 32 +PMPI: 1.400945e-02 +Pairwise: 3.371854e-02 +NonBlocking: 2.868213e-01 +Pairwise Hierarchical: 8.642252e-02 +Pairwise Node Aware: 5.290232e-03 +Nonblocking Hierarchical: 6.636415e-02 +Nonblocking Node Aware: 5.886425e-03 +Pairwise Multileader, 4 leaders: 3.130896e-02 +Pairwise Locality Aware, 4 leaders: 6.992557e-03 +Pairwise Multileader Locality, 4 leaders: 2.516649e-02 +Nonblocking Multileader, 4 leaders: 1.832050e-02 +Nonblocking Locality Aware, 4 leaders: 5.198816e-03 +Nonblocking Multileader Locality, 4 leaders: 1.835922e-02 +Pairwise Multileader, 10 leaders: 1.924323e-02 +Pairwise Locality Aware, 10 leaders: 1.129909e-02 +Pairwise Multileader Locality, 10 leaders: 1.215672e-02 +Nonblocking Multileader, 10 leaders: 4.308376e-02 +Nonblocking Locality Aware, 10 leaders: 1.407119e-01 +Nonblocking Multileader Locality, 10 leaders: 8.743635e-03 +Pairwise Multileader, 20 leaders: 1.871104e-02 +Pairwise Locality Aware, 20 leaders: 1.585640e-02 +Pairwise Multileader Locality, 20 leaders: 5.513505e-03 +Nonblocking Multileader, 20 leaders: 8.269964e-02 +Nonblocking Locality Aware, 20 leaders: 1.166733e-01 +Nonblocking Multileader Locality, 20 leaders: 5.120169e-03 +Size 64 +PMPI: 2.053362e-02 +Pairwise: 3.219362e-02 +NonBlocking: 2.900091e-01 +Pairwise Hierarchical: 1.662770e-01 +Pairwise Node Aware: 1.165699e-02 +Nonblocking Hierarchical: 1.276117e-01 +Nonblocking Node Aware: 7.563993e-03 +Pairwise Multileader, 4 leaders: 5.385927e-02 +Pairwise Locality Aware, 4 leaders: 7.916721e-03 +Pairwise Multileader Locality, 4 leaders: 5.773935e-02 +Nonblocking Multileader, 4 leaders: 3.789733e-02 +Nonblocking Locality Aware, 4 leaders: 1.080300e-02 +Nonblocking Multileader Locality, 4 leaders: 4.458656e-02 +Pairwise Multileader, 10 leaders: 2.826310e-02 +Pairwise Locality Aware, 10 leaders: 1.187044e-02 +Pairwise Multileader Locality, 10 leaders: 2.552297e-02 +Nonblocking Multileader, 10 leaders: 1.783311e-02 +Nonblocking Locality Aware, 10 leaders: 1.129175e-01 +Nonblocking Multileader Locality, 10 leaders: 2.022613e-02 +Pairwise Multileader, 20 leaders: 2.010151e-02 +Pairwise Locality Aware, 20 leaders: 1.579307e-02 +Pairwise Multileader Locality, 20 leaders: 1.466221e-02 +Nonblocking Multileader, 20 leaders: 6.007002e-02 +Nonblocking Locality Aware, 20 leaders: 9.747781e-02 +Nonblocking Multileader Locality, 20 leaders: 9.520191e-03 +Size 128 +PMPI: 4.008866e-02 +Pairwise: 6.076275e-02 +NonBlocking: 2.756504e-01 +Pairwise Hierarchical: 3.238363e-01 +Pairwise Node Aware: 1.980893e-02 +Nonblocking Hierarchical: 2.507707e-01 +Nonblocking Node Aware: 1.653713e-02 +Pairwise Multileader, 4 leaders: 9.880668e-02 +Pairwise Locality Aware, 4 leaders: 1.430701e-02 +Pairwise Multileader Locality, 4 leaders: 1.070583e-01 +Nonblocking Multileader, 4 leaders: 6.888531e-02 +Nonblocking Locality Aware, 4 leaders: 2.082180e-02 +Nonblocking Multileader Locality, 4 leaders: 7.952925e-02 +Pairwise Multileader, 10 leaders: 5.302764e-02 +Pairwise Locality Aware, 10 leaders: 1.993315e-02 +Pairwise Multileader Locality, 10 leaders: 5.344031e-02 +Nonblocking Multileader, 10 leaders: 7.677582e-02 +Nonblocking Locality Aware, 10 leaders: 1.030023e-01 +Nonblocking Multileader Locality, 10 leaders: 4.232221e-02 +Pairwise Multileader, 20 leaders: 3.730537e-02 +Pairwise Locality Aware, 20 leaders: 2.131940e-02 +Pairwise Multileader Locality, 20 leaders: 2.850625e-02 +Nonblocking Multileader, 20 leaders: 2.531274e-02 +Nonblocking Locality Aware, 20 leaders: 1.182356e-01 +Nonblocking Multileader Locality, 20 leaders: 2.119417e-02 +Size 256 +PMPI: 8.100046e-02 +Pairwise: 6.027919e-02 +NonBlocking: 1.977498e-01 +Pairwise Hierarchical: 6.584268e-01 +Pairwise Node Aware: 3.232816e-02 +Nonblocking Hierarchical: 5.444180e-01 +Nonblocking Node Aware: 2.981581e-02 +Pairwise Multileader, 4 leaders: 1.891392e-01 +Pairwise Locality Aware, 4 leaders: 4.715900e-02 +Pairwise Multileader Locality, 4 leaders: 2.129697e-01 +Nonblocking Multileader, 4 leaders: 1.334393e-01 +Nonblocking Locality Aware, 4 leaders: 2.460819e-02 +Nonblocking Multileader Locality, 4 leaders: 1.587284e-01 +Pairwise Multileader, 10 leaders: 1.137216e-01 +Pairwise Locality Aware, 10 leaders: 4.099574e-02 +Pairwise Multileader Locality, 10 leaders: 1.095330e-01 +Nonblocking Multileader, 10 leaders: 7.694013e-02 +Nonblocking Locality Aware, 10 leaders: 9.972737e-02 +Nonblocking Multileader Locality, 10 leaders: 8.869678e-02 +Pairwise Multileader, 20 leaders: 6.739209e-02 +Pairwise Locality Aware, 20 leaders: 3.710801e-02 +Pairwise Multileader Locality, 20 leaders: 5.676389e-02 +Nonblocking Multileader, 20 leaders: 2.459587e-01 +Nonblocking Locality Aware, 20 leaders: 1.033823e-01 +Nonblocking Multileader Locality, 20 leaders: 4.413029e-02 +Size 512 +PMPI: 2.976887e-01 +Pairwise: 9.986945e-02 +NonBlocking: 3.019195e-01 +Pairwise Hierarchical: 1.271857e+00 +Pairwise Node Aware: 6.259374e-02 +Nonblocking Hierarchical: 1.118095e+00 +Nonblocking Node Aware: 5.052967e-02 +Pairwise Multileader, 4 leaders: 3.477271e-01 +Pairwise Locality Aware, 4 leaders: 6.589456e-02 +Pairwise Multileader Locality, 4 leaders: 3.997852e-01 +Nonblocking Multileader, 4 leaders: 2.599242e-01 +Nonblocking Locality Aware, 4 leaders: 5.146498e-02 +Nonblocking Multileader Locality, 4 leaders: 3.116322e-01 +Pairwise Multileader, 10 leaders: 2.095048e-01 +Pairwise Locality Aware, 10 leaders: 7.462988e-02 +Pairwise Multileader Locality, 10 leaders: 2.127715e-01 +Nonblocking Multileader, 10 leaders: 1.444032e-01 +Nonblocking Locality Aware, 10 leaders: 8.322448e-02 +Nonblocking Multileader Locality, 10 leaders: 1.749450e-01 +Pairwise Multileader, 20 leaders: 1.481314e-01 +Pairwise Locality Aware, 20 leaders: 7.812836e-02 +Pairwise Multileader Locality, 20 leaders: 1.257454e-01 +Nonblocking Multileader, 20 leaders: 6.342911e-01 +Nonblocking Locality Aware, 20 leaders: 1.247508e-01 +Nonblocking Multileader Locality, 20 leaders: 1.076431e-01 +Test 2 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.049165e-03 +Pairwise: 1.893153e-02 +NonBlocking: 2.505836e-01 +Pairwise Hierarchical: 2.595334e-03 +Pairwise Node Aware: 8.785030e-04 +Nonblocking Hierarchical: 2.031775e-03 +Nonblocking Node Aware: 4.615402e-04 +Pairwise Multileader, 4 leaders: 2.137600e-03 +Pairwise Locality Aware, 4 leaders: 1.462041e-03 +Pairwise Multileader Locality, 4 leaders: 9.812250e-04 +Nonblocking Multileader, 4 leaders: 1.022504e-03 +Nonblocking Locality Aware, 4 leaders: 6.215646e-02 +Nonblocking Multileader Locality, 4 leaders: 6.764198e-04 +Pairwise Multileader, 10 leaders: 4.261259e-03 +Pairwise Locality Aware, 10 leaders: 2.881695e-03 +Pairwise Multileader Locality, 10 leaders: 5.517741e-04 +Nonblocking Multileader, 10 leaders: 2.388913e-02 +Nonblocking Locality Aware, 10 leaders: 1.152419e-01 +Nonblocking Multileader Locality, 10 leaders: 3.434266e-04 +Pairwise Multileader, 20 leaders: 7.670838e-03 +Pairwise Locality Aware, 20 leaders: 7.758321e-03 +Pairwise Multileader Locality, 20 leaders: 5.645088e-04 +Nonblocking Multileader, 20 leaders: 6.933876e-02 +Nonblocking Locality Aware, 20 leaders: 1.608405e-01 +Nonblocking Multileader Locality, 20 leaders: 2.465119e-04 +Size 2 +PMPI: 1.158570e-03 +Pairwise: 1.905898e-02 +NonBlocking: 5.668861e-01 +Pairwise Hierarchical: 5.290643e-03 +Pairwise Node Aware: 6.615631e-04 +Nonblocking Hierarchical: 4.422067e-03 +Nonblocking Node Aware: 6.342530e-04 +Pairwise Multileader, 4 leaders: 2.406644e-03 +Pairwise Locality Aware, 4 leaders: 1.979888e-03 +Pairwise Multileader Locality, 4 leaders: 1.769323e-03 +Nonblocking Multileader, 4 leaders: 1.321304e-03 +Nonblocking Locality Aware, 4 leaders: 4.968025e-02 +Nonblocking Multileader Locality, 4 leaders: 1.287164e-03 +Pairwise Multileader, 10 leaders: 5.252419e-03 +Pairwise Locality Aware, 10 leaders: 3.401223e-03 +Pairwise Multileader Locality, 10 leaders: 1.017224e-03 +Nonblocking Multileader, 10 leaders: 3.478817e-02 +Nonblocking Locality Aware, 10 leaders: 1.158011e-01 +Nonblocking Multileader Locality, 10 leaders: 6.181000e-04 +Pairwise Multileader, 20 leaders: 8.900956e-03 +Pairwise Locality Aware, 20 leaders: 8.826323e-03 +Pairwise Multileader Locality, 20 leaders: 6.845286e-04 +Nonblocking Multileader, 20 leaders: 6.394955e-02 +Nonblocking Locality Aware, 20 leaders: 1.610781e-01 +Nonblocking Multileader Locality, 20 leaders: 4.561003e-04 +Size 4 +PMPI: 1.747373e-03 +Pairwise: 2.349866e-02 +NonBlocking: 4.874302e-01 +Pairwise Hierarchical: 1.088486e-02 +Pairwise Node Aware: 1.131410e-03 +Nonblocking Hierarchical: 8.517259e-03 +Nonblocking Node Aware: 8.891821e-04 +Pairwise Multileader, 4 leaders: 3.962849e-03 +Pairwise Locality Aware, 4 leaders: 1.765618e-03 +Pairwise Multileader Locality, 4 leaders: 3.144517e-03 +Nonblocking Multileader, 4 leaders: 2.153199e-03 +Nonblocking Locality Aware, 4 leaders: 8.712934e-02 +Nonblocking Multileader Locality, 4 leaders: 2.457715e-03 +Pairwise Multileader, 10 leaders: 3.673565e-03 +Pairwise Locality Aware, 10 leaders: 2.812267e-03 +Pairwise Multileader Locality, 10 leaders: 1.841628e-03 +Nonblocking Multileader, 10 leaders: 3.788790e-02 +Nonblocking Locality Aware, 10 leaders: 9.926862e-02 +Nonblocking Multileader Locality, 10 leaders: 1.185955e-03 +Pairwise Multileader, 20 leaders: 7.273728e-03 +Pairwise Locality Aware, 20 leaders: 7.339609e-03 +Pairwise Multileader Locality, 20 leaders: 1.105034e-03 +Nonblocking Multileader, 20 leaders: 9.241482e-02 +Nonblocking Locality Aware, 20 leaders: 2.067933e-01 +Nonblocking Multileader Locality, 20 leaders: 8.073801e-04 +Size 8 +PMPI: 3.526548e-03 +Pairwise: 3.772666e-02 +NonBlocking: 4.062656e-01 +Pairwise Hierarchical: 2.219684e-02 +Pairwise Node Aware: 1.287523e-03 +Nonblocking Hierarchical: 1.750292e-02 +Nonblocking Node Aware: 1.198336e-03 +Pairwise Multileader, 4 leaders: 6.228252e-03 +Pairwise Locality Aware, 4 leaders: 1.986208e-03 +Pairwise Multileader Locality, 4 leaders: 7.332732e-03 +Nonblocking Multileader, 4 leaders: 4.065353e-03 +Nonblocking Locality Aware, 4 leaders: 9.247933e-02 +Nonblocking Multileader Locality, 4 leaders: 5.209509e-03 +Pairwise Multileader, 10 leaders: 6.401189e-03 +Pairwise Locality Aware, 10 leaders: 4.003121e-03 +Pairwise Multileader Locality, 10 leaders: 2.789748e-03 +Nonblocking Multileader, 10 leaders: 1.983187e-02 +Nonblocking Locality Aware, 10 leaders: 1.157241e-01 +Nonblocking Multileader Locality, 10 leaders: 1.653418e-03 +Pairwise Multileader, 20 leaders: 9.551498e-03 +Pairwise Locality Aware, 20 leaders: 8.031669e-03 +Pairwise Multileader Locality, 20 leaders: 1.488022e-03 +Nonblocking Multileader, 20 leaders: 8.766273e-02 +Nonblocking Locality Aware, 20 leaders: 1.845661e-01 +Nonblocking Multileader Locality, 20 leaders: 1.180110e-03 +Size 16 +PMPI: 6.013062e-03 +Pairwise: 4.836879e-02 +NonBlocking: 2.484916e-01 +Pairwise Hierarchical: 4.689078e-02 +Pairwise Node Aware: 2.572692e-03 +Nonblocking Hierarchical: 3.515833e-02 +Nonblocking Node Aware: 2.317384e-03 +Pairwise Multileader, 4 leaders: 9.664851e-03 +Pairwise Locality Aware, 4 leaders: 3.648219e-03 +Pairwise Multileader Locality, 4 leaders: 1.161455e-02 +Nonblocking Multileader, 4 leaders: 7.131716e-03 +Nonblocking Locality Aware, 4 leaders: 1.161297e-01 +Nonblocking Multileader Locality, 4 leaders: 8.709481e-03 +Pairwise Multileader, 10 leaders: 7.667309e-03 +Pairwise Locality Aware, 10 leaders: 6.520432e-03 +Pairwise Multileader Locality, 10 leaders: 4.690345e-03 +Nonblocking Multileader, 10 leaders: 1.871675e-02 +Nonblocking Locality Aware, 10 leaders: 1.164972e-01 +Nonblocking Multileader Locality, 10 leaders: 3.277996e-03 +Pairwise Multileader, 20 leaders: 1.121472e-02 +Pairwise Locality Aware, 20 leaders: 1.075462e-02 +Pairwise Multileader Locality, 20 leaders: 3.760463e-03 +Nonblocking Multileader, 20 leaders: 5.128591e-02 +Nonblocking Locality Aware, 20 leaders: 1.493062e-01 +Nonblocking Multileader Locality, 20 leaders: 2.709868e-03 +Size 32 +PMPI: 1.290961e-02 +Pairwise: 4.757119e-02 +NonBlocking: 3.286529e-01 +Pairwise Hierarchical: 8.745726e-02 +Pairwise Node Aware: 5.250831e-03 +Nonblocking Hierarchical: 6.680626e-02 +Nonblocking Node Aware: 5.632625e-03 +Pairwise Multileader, 4 leaders: 2.946414e-02 +Pairwise Locality Aware, 4 leaders: 3.966078e-03 +Pairwise Multileader Locality, 4 leaders: 2.508791e-02 +Nonblocking Multileader, 4 leaders: 1.843792e-02 +Nonblocking Locality Aware, 4 leaders: 3.749773e-03 +Nonblocking Multileader Locality, 4 leaders: 1.846234e-02 +Pairwise Multileader, 10 leaders: 1.700076e-02 +Pairwise Locality Aware, 10 leaders: 7.801574e-03 +Pairwise Multileader Locality, 10 leaders: 1.135080e-02 +Nonblocking Multileader, 10 leaders: 1.579543e-02 +Nonblocking Locality Aware, 10 leaders: 1.178870e-01 +Nonblocking Multileader Locality, 10 leaders: 8.666184e-03 +Pairwise Multileader, 20 leaders: 1.618946e-02 +Pairwise Locality Aware, 20 leaders: 1.515243e-02 +Pairwise Multileader Locality, 20 leaders: 6.281992e-03 +Nonblocking Multileader, 20 leaders: 7.880639e-02 +Nonblocking Locality Aware, 20 leaders: 1.395578e-01 +Nonblocking Multileader Locality, 20 leaders: 7.550708e-03 +Size 64 +PMPI: 2.009893e-02 +Pairwise: 4.552261e-02 +NonBlocking: 3.016669e-01 +Pairwise Hierarchical: 1.667556e-01 +Pairwise Node Aware: 8.482354e-03 +Nonblocking Hierarchical: 1.268778e-01 +Nonblocking Node Aware: 6.809643e-03 +Pairwise Multileader, 4 leaders: 5.432159e-02 +Pairwise Locality Aware, 4 leaders: 9.981379e-03 +Pairwise Multileader Locality, 4 leaders: 5.787920e-02 +Nonblocking Multileader, 4 leaders: 3.797591e-02 +Nonblocking Locality Aware, 4 leaders: 1.200291e-02 +Nonblocking Multileader Locality, 4 leaders: 4.311246e-02 +Pairwise Multileader, 10 leaders: 2.734385e-02 +Pairwise Locality Aware, 10 leaders: 1.015701e-02 +Pairwise Multileader Locality, 10 leaders: 2.530236e-02 +Nonblocking Multileader, 10 leaders: 1.763279e-02 +Nonblocking Locality Aware, 10 leaders: 9.743493e-02 +Nonblocking Multileader Locality, 10 leaders: 2.036648e-02 +Pairwise Multileader, 20 leaders: 1.823634e-02 +Pairwise Locality Aware, 20 leaders: 2.236348e-02 +Pairwise Multileader Locality, 20 leaders: 1.414390e-02 +Nonblocking Multileader, 20 leaders: 4.736704e-02 +Nonblocking Locality Aware, 20 leaders: 1.426703e-01 +Nonblocking Multileader Locality, 20 leaders: 9.752523e-03 +Size 128 +PMPI: 4.918493e-02 +Pairwise: 8.669740e-02 +NonBlocking: 3.320340e-01 +Pairwise Hierarchical: 3.240131e-01 +Pairwise Node Aware: 2.170768e-02 +Nonblocking Hierarchical: 2.515006e-01 +Nonblocking Node Aware: 1.775450e-02 +Pairwise Multileader, 4 leaders: 1.022977e-01 +Pairwise Locality Aware, 4 leaders: 1.993703e-02 +Pairwise Multileader Locality, 4 leaders: 1.099895e-01 +Nonblocking Multileader, 4 leaders: 6.904357e-02 +Nonblocking Locality Aware, 4 leaders: 2.268432e-02 +Nonblocking Multileader Locality, 4 leaders: 8.145424e-02 +Pairwise Multileader, 10 leaders: 5.256818e-02 +Pairwise Locality Aware, 10 leaders: 1.849552e-02 +Pairwise Multileader Locality, 10 leaders: 5.343435e-02 +Nonblocking Multileader, 10 leaders: 7.523044e-02 +Nonblocking Locality Aware, 10 leaders: 9.637395e-02 +Nonblocking Multileader Locality, 10 leaders: 4.251849e-02 +Pairwise Multileader, 20 leaders: 3.852967e-02 +Pairwise Locality Aware, 20 leaders: 2.257122e-02 +Pairwise Multileader Locality, 20 leaders: 2.932644e-02 +Nonblocking Multileader, 20 leaders: 2.664192e-02 +Nonblocking Locality Aware, 20 leaders: 1.339025e-01 +Nonblocking Multileader Locality, 20 leaders: 2.097250e-02 +Size 256 +PMPI: 8.156055e-02 +Pairwise: 7.161872e-02 +NonBlocking: 1.892910e-01 +Pairwise Hierarchical: 6.518718e-01 +Pairwise Node Aware: 3.194948e-02 +Nonblocking Hierarchical: 5.367856e-01 +Nonblocking Node Aware: 2.951621e-02 +Pairwise Multileader, 4 leaders: 1.811177e-01 +Pairwise Locality Aware, 4 leaders: 3.564978e-02 +Pairwise Multileader Locality, 4 leaders: 2.061881e-01 +Nonblocking Multileader, 4 leaders: 1.334899e-01 +Nonblocking Locality Aware, 4 leaders: 3.208672e-02 +Nonblocking Multileader Locality, 4 leaders: 1.566388e-01 +Pairwise Multileader, 10 leaders: 1.294072e-01 +Pairwise Locality Aware, 10 leaders: 4.319127e-02 +Pairwise Multileader Locality, 10 leaders: 1.113321e-01 +Nonblocking Multileader, 10 leaders: 7.828955e-02 +Nonblocking Locality Aware, 10 leaders: 8.578420e-02 +Nonblocking Multileader Locality, 10 leaders: 9.082677e-02 +Pairwise Multileader, 20 leaders: 8.815206e-02 +Pairwise Locality Aware, 20 leaders: 4.397288e-02 +Pairwise Multileader Locality, 20 leaders: 5.811961e-02 +Nonblocking Multileader, 20 leaders: 9.433450e-02 +Nonblocking Locality Aware, 20 leaders: 1.772596e-01 +Nonblocking Multileader Locality, 20 leaders: 4.455805e-02 +Size 512 +PMPI: 4.064902e-01 +Pairwise: 1.061461e-01 +NonBlocking: 3.062073e-01 +Pairwise Hierarchical: 1.264005e+00 +Pairwise Node Aware: 6.234793e-02 +Nonblocking Hierarchical: 1.133451e+00 +Nonblocking Node Aware: 4.948413e-02 +Pairwise Multileader, 4 leaders: 3.480876e-01 +Pairwise Locality Aware, 4 leaders: 7.052115e-02 +Pairwise Multileader Locality, 4 leaders: 4.025704e-01 +Nonblocking Multileader, 4 leaders: 2.616629e-01 +Nonblocking Locality Aware, 4 leaders: 6.190129e-02 +Nonblocking Multileader Locality, 4 leaders: 3.092601e-01 +Pairwise Multileader, 10 leaders: 2.232602e-01 +Pairwise Locality Aware, 10 leaders: 7.888198e-02 +Pairwise Multileader Locality, 10 leaders: 2.151549e-01 +Nonblocking Multileader, 10 leaders: 1.453607e-01 +Nonblocking Locality Aware, 10 leaders: 6.836843e-02 +Nonblocking Multileader Locality, 10 leaders: 1.740150e-01 +Pairwise Multileader, 20 leaders: 1.455823e-01 +Pairwise Locality Aware, 20 leaders: 7.737803e-02 +Pairwise Multileader Locality, 20 leaders: 1.236385e-01 +Nonblocking Multileader, 20 leaders: 6.299690e-01 +Nonblocking Locality Aware, 20 leaders: 1.400798e-01 +Nonblocking Multileader Locality, 20 leaders: 1.074760e-01 +Test 3 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.153102e-03 +Pairwise: 1.645691e-02 +NonBlocking: 3.650682e-01 +Pairwise Hierarchical: 2.399660e-03 +Pairwise Node Aware: 9.568754e-04 +Nonblocking Hierarchical: 2.033980e-03 +Nonblocking Node Aware: 4.657329e-04 +Pairwise Multileader, 4 leaders: 2.020980e-03 +Pairwise Locality Aware, 4 leaders: 1.796872e-03 +Pairwise Multileader Locality, 4 leaders: 9.822603e-04 +Nonblocking Multileader, 4 leaders: 1.766621e-03 +Nonblocking Locality Aware, 4 leaders: 6.217876e-02 +Nonblocking Multileader Locality, 4 leaders: 6.982474e-04 +Pairwise Multileader, 10 leaders: 3.181760e-03 +Pairwise Locality Aware, 10 leaders: 4.561280e-03 +Pairwise Multileader Locality, 10 leaders: 7.427823e-04 +Nonblocking Multileader, 10 leaders: 4.692995e-04 +Nonblocking Locality Aware, 10 leaders: 1.149595e-01 +Nonblocking Multileader Locality, 10 leaders: 3.706357e-04 +Pairwise Multileader, 20 leaders: 8.417512e-03 +Pairwise Locality Aware, 20 leaders: 5.106184e-03 +Pairwise Multileader Locality, 20 leaders: 5.852401e-04 +Nonblocking Multileader, 20 leaders: 4.998101e-02 +Nonblocking Locality Aware, 20 leaders: 2.067842e-01 +Nonblocking Multileader Locality, 20 leaders: 3.716180e-04 +Size 2 +PMPI: 1.162589e-03 +Pairwise: 1.451732e-02 +NonBlocking: 5.679700e-01 +Pairwise Hierarchical: 5.238760e-03 +Pairwise Node Aware: 9.124184e-04 +Nonblocking Hierarchical: 4.354714e-03 +Nonblocking Node Aware: 6.947624e-04 +Pairwise Multileader, 4 leaders: 2.948307e-03 +Pairwise Locality Aware, 4 leaders: 1.937208e-03 +Pairwise Multileader Locality, 4 leaders: 1.738306e-03 +Nonblocking Multileader, 4 leaders: 1.112924e-03 +Nonblocking Locality Aware, 4 leaders: 6.199265e-02 +Nonblocking Multileader Locality, 4 leaders: 1.356609e-03 +Pairwise Multileader, 10 leaders: 3.813772e-03 +Pairwise Locality Aware, 10 leaders: 3.832468e-03 +Pairwise Multileader Locality, 10 leaders: 9.149528e-04 +Nonblocking Multileader, 10 leaders: 3.209884e-02 +Nonblocking Locality Aware, 10 leaders: 1.152767e-01 +Nonblocking Multileader Locality, 10 leaders: 5.593005e-04 +Pairwise Multileader, 20 leaders: 8.832791e-03 +Pairwise Locality Aware, 20 leaders: 8.093547e-03 +Pairwise Multileader Locality, 20 leaders: 6.118355e-04 +Nonblocking Multileader, 20 leaders: 6.766060e-02 +Nonblocking Locality Aware, 20 leaders: 1.158870e-01 +Nonblocking Multileader Locality, 20 leaders: 4.429934e-04 +Size 4 +PMPI: 1.843169e-03 +Pairwise: 3.978179e-02 +NonBlocking: 3.272722e-01 +Pairwise Hierarchical: 1.053564e-02 +Pairwise Node Aware: 9.272528e-04 +Nonblocking Hierarchical: 8.452367e-03 +Nonblocking Node Aware: 8.289043e-04 +Pairwise Multileader, 4 leaders: 4.717483e-03 +Pairwise Locality Aware, 4 leaders: 2.109515e-03 +Pairwise Multileader Locality, 4 leaders: 3.031328e-03 +Nonblocking Multileader, 4 leaders: 2.801663e-03 +Nonblocking Locality Aware, 4 leaders: 8.715718e-02 +Nonblocking Multileader Locality, 4 leaders: 2.473998e-03 +Pairwise Multileader, 10 leaders: 6.468529e-03 +Pairwise Locality Aware, 10 leaders: 5.019534e-03 +Pairwise Multileader Locality, 10 leaders: 1.653075e-03 +Nonblocking Multileader, 10 leaders: 1.100283e-03 +Nonblocking Locality Aware, 10 leaders: 1.155808e-01 +Nonblocking Multileader Locality, 10 leaders: 1.140307e-03 +Pairwise Multileader, 20 leaders: 6.894648e-03 +Pairwise Locality Aware, 20 leaders: 7.243246e-03 +Pairwise Multileader Locality, 20 leaders: 1.997451e-03 +Nonblocking Multileader, 20 leaders: 7.479805e-02 +Nonblocking Locality Aware, 20 leaders: 1.612030e-01 +Nonblocking Multileader Locality, 20 leaders: 6.341260e-04 +Size 8 +PMPI: 3.585121e-03 +Pairwise: 4.247834e-02 +NonBlocking: 4.869949e-01 +Pairwise Hierarchical: 2.243930e-02 +Pairwise Node Aware: 1.377235e-03 +Nonblocking Hierarchical: 1.731488e-02 +Nonblocking Node Aware: 1.294482e-03 +Pairwise Multileader, 4 leaders: 6.060954e-03 +Pairwise Locality Aware, 4 leaders: 3.948291e-03 +Pairwise Multileader Locality, 4 leaders: 7.164807e-03 +Nonblocking Multileader, 4 leaders: 3.981207e-03 +Nonblocking Locality Aware, 4 leaders: 8.722700e-02 +Nonblocking Multileader Locality, 4 leaders: 5.159468e-03 +Pairwise Multileader, 10 leaders: 5.185393e-03 +Pairwise Locality Aware, 10 leaders: 6.360214e-03 +Pairwise Multileader Locality, 10 leaders: 2.524228e-03 +Nonblocking Multileader, 10 leaders: 5.080632e-02 +Nonblocking Locality Aware, 10 leaders: 1.156809e-01 +Nonblocking Multileader Locality, 10 leaders: 1.978451e-03 +Pairwise Multileader, 20 leaders: 7.299667e-03 +Pairwise Locality Aware, 20 leaders: 1.020082e-02 +Pairwise Multileader Locality, 20 leaders: 1.651231e-03 +Nonblocking Multileader, 20 leaders: 8.814887e-02 +Nonblocking Locality Aware, 20 leaders: 1.383348e-01 +Nonblocking Multileader Locality, 20 leaders: 1.317109e-03 +Size 16 +PMPI: 5.670200e-03 +Pairwise: 4.206332e-02 +NonBlocking: 6.490302e-01 +Pairwise Hierarchical: 4.641430e-02 +Pairwise Node Aware: 2.731243e-03 +Nonblocking Hierarchical: 3.540907e-02 +Nonblocking Node Aware: 2.523578e-03 +Pairwise Multileader, 4 leaders: 1.081330e-02 +Pairwise Locality Aware, 4 leaders: 3.330836e-03 +Pairwise Multileader Locality, 4 leaders: 1.200160e-02 +Nonblocking Multileader, 4 leaders: 7.034771e-03 +Nonblocking Locality Aware, 4 leaders: 8.811459e-02 +Nonblocking Multileader Locality, 4 leaders: 8.781751e-03 +Pairwise Multileader, 10 leaders: 7.991684e-03 +Pairwise Locality Aware, 10 leaders: 6.769278e-03 +Pairwise Multileader Locality, 10 leaders: 4.672006e-03 +Nonblocking Multileader, 10 leaders: 8.250355e-03 +Nonblocking Locality Aware, 10 leaders: 1.163396e-01 +Nonblocking Multileader Locality, 10 leaders: 3.707720e-03 +Pairwise Multileader, 20 leaders: 9.463451e-03 +Pairwise Locality Aware, 20 leaders: 7.757104e-03 +Pairwise Multileader Locality, 20 leaders: 3.703609e-03 +Nonblocking Multileader, 20 leaders: 1.620527e-01 +Nonblocking Locality Aware, 20 leaders: 2.259137e-01 +Nonblocking Multileader Locality, 20 leaders: 2.742714e-03 +Size 32 +PMPI: 1.250957e-02 +Pairwise: 6.035694e-02 +NonBlocking: 4.489648e-01 +Pairwise Hierarchical: 8.906476e-02 +Pairwise Node Aware: 6.012539e-03 +Nonblocking Hierarchical: 6.699935e-02 +Nonblocking Node Aware: 5.324680e-03 +Pairwise Multileader, 4 leaders: 3.204303e-02 +Pairwise Locality Aware, 4 leaders: 1.087375e-02 +Pairwise Multileader Locality, 4 leaders: 2.570629e-02 +Nonblocking Multileader, 4 leaders: 1.835915e-02 +Nonblocking Locality Aware, 4 leaders: 6.837308e-03 +Nonblocking Multileader Locality, 4 leaders: 1.860886e-02 +Pairwise Multileader, 10 leaders: 1.707518e-02 +Pairwise Locality Aware, 10 leaders: 8.161715e-03 +Pairwise Multileader Locality, 10 leaders: 1.114836e-02 +Nonblocking Multileader, 10 leaders: 3.002664e-02 +Nonblocking Locality Aware, 10 leaders: 1.175895e-01 +Nonblocking Multileader Locality, 10 leaders: 8.655756e-03 +Pairwise Multileader, 20 leaders: 1.802239e-02 +Pairwise Locality Aware, 20 leaders: 1.213999e-02 +Pairwise Multileader Locality, 20 leaders: 5.191142e-03 +Nonblocking Multileader, 20 leaders: 7.645570e-02 +Nonblocking Locality Aware, 20 leaders: 1.629707e-01 +Nonblocking Multileader Locality, 20 leaders: 4.367272e-03 +Size 64 +PMPI: 1.952806e-02 +Pairwise: 4.658972e-02 +NonBlocking: 1.760848e-01 +Pairwise Hierarchical: 1.668194e-01 +Pairwise Node Aware: 9.679241e-03 +Nonblocking Hierarchical: 1.268328e-01 +Nonblocking Node Aware: 9.132335e-03 +Pairwise Multileader, 4 leaders: 5.452334e-02 +Pairwise Locality Aware, 4 leaders: 7.949159e-03 +Pairwise Multileader Locality, 4 leaders: 5.861443e-02 +Nonblocking Multileader, 4 leaders: 3.818540e-02 +Nonblocking Locality Aware, 4 leaders: 9.986015e-03 +Nonblocking Multileader Locality, 4 leaders: 4.318972e-02 +Pairwise Multileader, 10 leaders: 2.878274e-02 +Pairwise Locality Aware, 10 leaders: 1.446852e-02 +Pairwise Multileader Locality, 10 leaders: 2.770647e-02 +Nonblocking Multileader, 10 leaders: 1.817879e-02 +Nonblocking Locality Aware, 10 leaders: 1.194246e-01 +Nonblocking Multileader Locality, 10 leaders: 2.044934e-02 +Pairwise Multileader, 20 leaders: 2.051309e-02 +Pairwise Locality Aware, 20 leaders: 1.420075e-02 +Pairwise Multileader Locality, 20 leaders: 1.335590e-02 +Nonblocking Multileader, 20 leaders: 2.188253e-02 +Nonblocking Locality Aware, 20 leaders: 1.190928e-01 +Nonblocking Multileader Locality, 20 leaders: 2.579410e-02 +Size 128 +PMPI: 3.807801e-02 +Pairwise: 4.985721e-02 +NonBlocking: 2.797709e-01 +Pairwise Hierarchical: 3.245343e-01 +Pairwise Node Aware: 2.332413e-02 +Nonblocking Hierarchical: 2.468252e-01 +Nonblocking Node Aware: 1.924482e-02 +Pairwise Multileader, 4 leaders: 1.011921e-01 +Pairwise Locality Aware, 4 leaders: 1.685410e-02 +Pairwise Multileader Locality, 4 leaders: 1.074658e-01 +Nonblocking Multileader, 4 leaders: 6.909208e-02 +Nonblocking Locality Aware, 4 leaders: 2.375324e-02 +Nonblocking Multileader Locality, 4 leaders: 7.994144e-02 +Pairwise Multileader, 10 leaders: 5.064129e-02 +Pairwise Locality Aware, 10 leaders: 1.792886e-02 +Pairwise Multileader Locality, 10 leaders: 5.485290e-02 +Nonblocking Multileader, 10 leaders: 1.141865e-01 +Nonblocking Locality Aware, 10 leaders: 9.180464e-02 +Nonblocking Multileader Locality, 10 leaders: 4.246610e-02 +Pairwise Multileader, 20 leaders: 3.643639e-02 +Pairwise Locality Aware, 20 leaders: 2.857249e-02 +Pairwise Multileader Locality, 20 leaders: 2.842914e-02 +Nonblocking Multileader, 20 leaders: 4.620897e-02 +Nonblocking Locality Aware, 20 leaders: 1.175251e-01 +Nonblocking Multileader Locality, 20 leaders: 2.132773e-02 +Size 256 +PMPI: 8.312406e-02 +Pairwise: 8.300047e-02 +NonBlocking: 1.881992e-01 +Pairwise Hierarchical: 6.539959e-01 +Pairwise Node Aware: 4.023167e-02 +Nonblocking Hierarchical: 5.401099e-01 +Nonblocking Node Aware: 2.955713e-02 +Pairwise Multileader, 4 leaders: 1.895316e-01 +Pairwise Locality Aware, 4 leaders: 3.270753e-02 +Pairwise Multileader Locality, 4 leaders: 2.114129e-01 +Nonblocking Multileader, 4 leaders: 1.323768e-01 +Nonblocking Locality Aware, 4 leaders: 2.828389e-02 +Nonblocking Multileader Locality, 4 leaders: 1.561958e-01 +Pairwise Multileader, 10 leaders: 1.141202e-01 +Pairwise Locality Aware, 10 leaders: 3.813505e-02 +Pairwise Multileader Locality, 10 leaders: 1.090717e-01 +Nonblocking Multileader, 10 leaders: 7.817584e-02 +Nonblocking Locality Aware, 10 leaders: 6.686205e-02 +Nonblocking Multileader Locality, 10 leaders: 8.884180e-02 +Pairwise Multileader, 20 leaders: 7.058469e-02 +Pairwise Locality Aware, 20 leaders: 3.839629e-02 +Pairwise Multileader Locality, 20 leaders: 5.664490e-02 +Nonblocking Multileader, 20 leaders: 2.367542e-01 +Nonblocking Locality Aware, 20 leaders: 1.085656e-01 +Nonblocking Multileader Locality, 20 leaders: 4.417694e-02 +Size 512 +PMPI: 4.080657e-01 +Pairwise: 9.806752e-02 +NonBlocking: 3.029515e-01 +Pairwise Hierarchical: 1.289940e+00 +Pairwise Node Aware: 7.515630e-02 +Nonblocking Hierarchical: 1.154515e+00 +Nonblocking Node Aware: 4.987889e-02 +Pairwise Multileader, 4 leaders: 3.434039e-01 +Pairwise Locality Aware, 4 leaders: 7.035477e-02 +Pairwise Multileader Locality, 4 leaders: 4.044040e-01 +Nonblocking Multileader, 4 leaders: 2.585987e-01 +Nonblocking Locality Aware, 4 leaders: 5.735452e-02 +Nonblocking Multileader Locality, 4 leaders: 3.084415e-01 +Pairwise Multileader, 10 leaders: 2.070506e-01 +Pairwise Locality Aware, 10 leaders: 8.463742e-02 +Pairwise Multileader Locality, 10 leaders: 2.143442e-01 +Nonblocking Multileader, 10 leaders: 1.449636e-01 +Nonblocking Locality Aware, 10 leaders: 6.701689e-02 +Nonblocking Multileader Locality, 10 leaders: 1.737169e-01 +Pairwise Multileader, 20 leaders: 1.537770e-01 +Pairwise Locality Aware, 20 leaders: 8.470722e-02 +Pairwise Multileader Locality, 20 leaders: 1.245460e-01 +Nonblocking Multileader, 20 leaders: 2.024949e-01 +Nonblocking Locality Aware, 20 leaders: 1.328236e-01 +Nonblocking Multileader Locality, 20 leaders: 1.068925e-01 diff --git a/runscripts/dane_results/alltoall_N4.2373826.err b/runscripts/dane_results/alltoall_N4.2373826.err new file mode 100644 index 000000000..87fcdae6e --- /dev/null +++ b/runscripts/dane_results/alltoall_N4.2373826.err @@ -0,0 +1,104 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2373826.0 ON dane215 CANCELLED AT 2025-04-29T17:42:54 *** +slurmstepd: error: *** STEP 2373826.0 ON dane215 CANCELLED AT 2025-04-29T17:42:54 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1167: tasks 300-399: Killed +srun: error: dane1134: tasks 200-299: Killed +srun: error: dane644: tasks 100-199: Killed +srun: error: dane215: tasks 0-99: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2373826.1 ON dane215 CANCELLED AT 2025-04-29T17:53:35 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1167: tasks 300-399: Killed +srun: error: dane1134: tasks 200-299: Killed +srun: error: dane215: tasks 0-99: Killed +srun: error: dane644: tasks 100-199: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x15551069c010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x15551069c010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2373826.2 ON dane215 CANCELLED AT 2025-04-29T18:04:37 *** +slurmstepd: error: *** STEP 2373826.2 ON dane215 CANCELLED AT 2025-04-29T18:04:37 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1167: tasks 300-399: Killed +srun: error: dane1134: tasks 200-299: Killed +srun: error: dane644: tasks 100-199: Killed +srun: error: dane215: tasks 0-99: Killed diff --git a/runscripts/dane_results/alltoall_N4.2373826.out b/runscripts/dane_results/alltoall_N4.2373826.out new file mode 100644 index 000000000..f7d1067cf --- /dev/null +++ b/runscripts/dane_results/alltoall_N4.2373826.out @@ -0,0 +1,1107 @@ +Test 1 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 9.530671e-05 +Pairwise: 1.036941e-03 +NonBlocking: 3.580185e-04 +Pairwise Hierarchical: 3.265360e-04 +Pairwise Node Aware: 2.008013e-04 +Nonblocking Hierarchical: 3.078509e-04 +Nonblocking Node Aware: 1.932906e-04 +Pairwise Multileader, 4 leaders: 1.408247e-04 +Pairwise Locality Aware, 4 leaders: 1.216760e-04 +Pairwise Multileader Locality, 4 leaders: 1.263167e-04 +Nonblocking Multileader, 4 leaders: 9.036503e-05 +Nonblocking Locality Aware, 4 leaders: 6.953953e-05 +Nonblocking Multileader Locality, 4 leaders: 1.065706e-04 +Pairwise Multileader, 10 leaders: 3.080942e-04 +Pairwise Locality Aware, 10 leaders: 2.339328e-04 +Pairwise Multileader Locality, 10 leaders: 7.727052e-05 +Nonblocking Multileader, 10 leaders: 6.956228e-05 +Nonblocking Locality Aware, 10 leaders: 9.088037e-05 +Nonblocking Multileader Locality, 10 leaders: 6.432754e-05 +Pairwise Multileader, 20 leaders: 4.680366e-04 +Pairwise Locality Aware, 20 leaders: 4.869956e-04 +Pairwise Multileader Locality, 20 leaders: 8.837117e-05 +Nonblocking Multileader, 20 leaders: 1.024840e-04 +Nonblocking Locality Aware, 20 leaders: 1.694513e-04 +Nonblocking Multileader Locality, 20 leaders: 5.810551e-05 +Size 2 +PMPI: 1.111421e-04 +Pairwise: 1.156141e-03 +NonBlocking: 2.822575e-04 +Pairwise Hierarchical: 6.293579e-04 +Pairwise Node Aware: 2.530616e-04 +Nonblocking Hierarchical: 5.435577e-04 +Nonblocking Node Aware: 2.036681e-04 +Pairwise Multileader, 4 leaders: 1.923615e-04 +Pairwise Locality Aware, 4 leaders: 1.675778e-04 +Pairwise Multileader Locality, 4 leaders: 1.607532e-04 +Nonblocking Multileader, 4 leaders: 1.175809e-04 +Nonblocking Locality Aware, 4 leaders: 1.004217e-04 +Nonblocking Multileader Locality, 4 leaders: 1.312322e-04 +Pairwise Multileader, 10 leaders: 2.738444e-04 +Pairwise Locality Aware, 10 leaders: 2.464173e-04 +Pairwise Multileader Locality, 10 leaders: 9.920614e-05 +Nonblocking Multileader, 10 leaders: 7.812308e-05 +Nonblocking Locality Aware, 10 leaders: 1.120036e-04 +Nonblocking Multileader Locality, 10 leaders: 8.385008e-05 +Pairwise Multileader, 20 leaders: 4.426765e-04 +Pairwise Locality Aware, 20 leaders: 4.391884e-04 +Pairwise Multileader Locality, 20 leaders: 1.022261e-04 +Nonblocking Multileader, 20 leaders: 1.086225e-04 +Nonblocking Locality Aware, 20 leaders: 1.822219e-04 +Nonblocking Multileader Locality, 20 leaders: 7.012302e-05 +Size 4 +PMPI: 1.613643e-04 +Pairwise: 1.976453e-03 +NonBlocking: 7.429680e-02 +Pairwise Hierarchical: 9.563784e-04 +Pairwise Node Aware: 2.926005e-04 +Nonblocking Hierarchical: 8.598961e-04 +Nonblocking Node Aware: 2.010901e-04 +Pairwise Multileader, 4 leaders: 3.447069e-04 +Pairwise Locality Aware, 4 leaders: 1.359642e-04 +Pairwise Multileader Locality, 4 leaders: 3.169443e-04 +Nonblocking Multileader, 4 leaders: 2.199755e-04 +Nonblocking Locality Aware, 4 leaders: 1.117220e-04 +Nonblocking Multileader Locality, 4 leaders: 2.668247e-04 +Pairwise Multileader, 10 leaders: 2.791483e-04 +Pairwise Locality Aware, 10 leaders: 3.086008e-04 +Pairwise Multileader Locality, 10 leaders: 1.296998e-04 +Nonblocking Multileader, 10 leaders: 1.133639e-04 +Nonblocking Locality Aware, 10 leaders: 1.421083e-04 +Nonblocking Multileader Locality, 10 leaders: 1.057701e-04 +Pairwise Multileader, 20 leaders: 6.403244e-04 +Pairwise Locality Aware, 20 leaders: 4.184158e-04 +Pairwise Multileader Locality, 20 leaders: 1.283921e-04 +Nonblocking Multileader, 20 leaders: 1.432893e-04 +Nonblocking Locality Aware, 20 leaders: 2.374102e-04 +Nonblocking Multileader Locality, 20 leaders: 9.201107e-05 +Size 8 +PMPI: 2.455442e-04 +Pairwise: 2.464728e-03 +NonBlocking: 3.745928e-02 +Pairwise Hierarchical: 1.771444e-03 +Pairwise Node Aware: 3.638219e-04 +Nonblocking Hierarchical: 1.610456e-03 +Nonblocking Node Aware: 2.162790e-04 +Pairwise Multileader, 4 leaders: 5.122608e-04 +Pairwise Locality Aware, 4 leaders: 2.232956e-04 +Pairwise Multileader Locality, 4 leaders: 6.946717e-04 +Nonblocking Multileader, 4 leaders: 3.851964e-04 +Nonblocking Locality Aware, 4 leaders: 2.522452e-04 +Nonblocking Multileader Locality, 4 leaders: 5.559032e-04 +Pairwise Multileader, 10 leaders: 4.644014e-04 +Pairwise Locality Aware, 10 leaders: 3.132514e-04 +Pairwise Multileader Locality, 10 leaders: 2.503904e-04 +Nonblocking Multileader, 10 leaders: 1.791694e-04 +Nonblocking Locality Aware, 10 leaders: 1.738573e-04 +Nonblocking Multileader Locality, 10 leaders: 2.060145e-04 +Pairwise Multileader, 20 leaders: 6.014548e-04 +Pairwise Locality Aware, 20 leaders: 4.474731e-04 +Pairwise Multileader Locality, 20 leaders: 1.915130e-04 +Nonblocking Multileader, 20 leaders: 1.655264e-04 +Nonblocking Locality Aware, 20 leaders: 3.960757e-04 +Nonblocking Multileader Locality, 20 leaders: 1.319349e-04 +Size 16 +PMPI: 3.940048e-04 +Pairwise: 2.484164e-03 +NonBlocking: 4.623079e-02 +Pairwise Hierarchical: 3.417737e-03 +Pairwise Node Aware: 4.566965e-04 +Nonblocking Hierarchical: 3.170182e-03 +Nonblocking Node Aware: 2.818113e-04 +Pairwise Multileader, 4 leaders: 9.534163e-04 +Pairwise Locality Aware, 4 leaders: 2.692475e-04 +Pairwise Multileader Locality, 4 leaders: 1.231842e-03 +Nonblocking Multileader, 4 leaders: 7.424179e-04 +Nonblocking Locality Aware, 4 leaders: 2.159368e-04 +Nonblocking Multileader Locality, 4 leaders: 1.032297e-03 +Pairwise Multileader, 10 leaders: 6.083317e-04 +Pairwise Locality Aware, 10 leaders: 4.667023e-04 +Pairwise Multileader Locality, 10 leaders: 4.643257e-04 +Nonblocking Multileader, 10 leaders: 3.343723e-04 +Nonblocking Locality Aware, 10 leaders: 2.989606e-04 +Nonblocking Multileader Locality, 10 leaders: 4.254855e-04 +Pairwise Multileader, 20 leaders: 6.625026e-04 +Pairwise Locality Aware, 20 leaders: 6.076075e-04 +Pairwise Multileader Locality, 20 leaders: 3.049201e-04 +Nonblocking Multileader, 20 leaders: 2.814058e-04 +Nonblocking Locality Aware, 20 leaders: 3.027534e-04 +Nonblocking Multileader Locality, 20 leaders: 2.170260e-04 +Size 32 +PMPI: 7.509087e-04 +Pairwise: 2.532059e-03 +NonBlocking: 7.441809e-02 +Pairwise Hierarchical: 7.477144e-03 +Pairwise Node Aware: 6.371873e-04 +Nonblocking Hierarchical: 6.897994e-03 +Nonblocking Node Aware: 4.347551e-04 +Pairwise Multileader, 4 leaders: 2.920867e-03 +Pairwise Locality Aware, 4 leaders: 5.563146e-04 +Pairwise Multileader Locality, 4 leaders: 2.752103e-03 +Nonblocking Multileader, 4 leaders: 2.252392e-03 +Nonblocking Locality Aware, 4 leaders: 4.451927e-04 +Nonblocking Multileader Locality, 4 leaders: 2.369837e-03 +Pairwise Multileader, 10 leaders: 1.226157e-03 +Pairwise Locality Aware, 10 leaders: 5.232697e-04 +Pairwise Multileader Locality, 10 leaders: 1.121891e-03 +Nonblocking Multileader, 10 leaders: 7.548021e-04 +Nonblocking Locality Aware, 10 leaders: 4.786254e-04 +Nonblocking Multileader Locality, 10 leaders: 9.634168e-04 +Pairwise Multileader, 20 leaders: 9.631407e-04 +Pairwise Locality Aware, 20 leaders: 7.001998e-04 +Pairwise Multileader Locality, 20 leaders: 5.368448e-04 +Nonblocking Multileader, 20 leaders: 4.697463e-04 +Nonblocking Locality Aware, 20 leaders: 6.223257e-04 +Nonblocking Multileader Locality, 20 leaders: 4.411260e-04 +Size 64 +PMPI: 1.661677e-03 +Pairwise: 3.257263e-03 +NonBlocking: 9.909287e-02 +Pairwise Hierarchical: 1.708775e-02 +Pairwise Node Aware: 1.035306e-03 +Nonblocking Hierarchical: 1.501224e-02 +Nonblocking Node Aware: 6.425266e-04 +Pairwise Multileader, 4 leaders: 5.170338e-03 +Pairwise Locality Aware, 4 leaders: 9.086818e-04 +Pairwise Multileader Locality, 4 leaders: 5.148011e-03 +Nonblocking Multileader, 4 leaders: 4.061661e-03 +Nonblocking Locality Aware, 4 leaders: 7.963067e-04 +Nonblocking Multileader Locality, 4 leaders: 4.468937e-03 +Pairwise Multileader, 10 leaders: 2.202600e-03 +Pairwise Locality Aware, 10 leaders: 8.321810e-04 +Pairwise Multileader Locality, 10 leaders: 2.166939e-03 +Nonblocking Multileader, 10 leaders: 1.556621e-03 +Nonblocking Locality Aware, 10 leaders: 6.989559e-04 +Nonblocking Multileader Locality, 10 leaders: 1.862046e-03 +Pairwise Multileader, 20 leaders: 1.607486e-03 +Pairwise Locality Aware, 20 leaders: 1.080328e-03 +Pairwise Multileader Locality, 20 leaders: 1.242073e-03 +Nonblocking Multileader, 20 leaders: 7.576196e-04 +Nonblocking Locality Aware, 20 leaders: 5.854063e-04 +Nonblocking Multileader Locality, 20 leaders: 1.046700e-03 +Size 128 +PMPI: 2.920120e-03 +Pairwise: 2.557301e-03 +NonBlocking: 9.966119e-02 +Pairwise Hierarchical: 3.586190e-02 +Pairwise Node Aware: 1.945333e-03 +Nonblocking Hierarchical: 3.248923e-02 +Nonblocking Node Aware: 1.230349e-03 +Pairwise Multileader, 4 leaders: 9.759840e-03 +Pairwise Locality Aware, 4 leaders: 1.539607e-03 +Pairwise Multileader Locality, 4 leaders: 9.911000e-03 +Nonblocking Multileader, 4 leaders: 7.138012e-03 +Nonblocking Locality Aware, 4 leaders: 1.072761e-03 +Nonblocking Multileader Locality, 4 leaders: 8.106537e-03 +Pairwise Multileader, 10 leaders: 4.486377e-03 +Pairwise Locality Aware, 10 leaders: 1.602996e-03 +Pairwise Multileader Locality, 10 leaders: 4.184481e-03 +Nonblocking Multileader, 10 leaders: 3.144544e-03 +Nonblocking Locality Aware, 10 leaders: 1.404096e-03 +Nonblocking Multileader Locality, 10 leaders: 3.676275e-03 +Pairwise Multileader, 20 leaders: 2.718125e-03 +Pairwise Locality Aware, 20 leaders: 1.827704e-03 +Pairwise Multileader Locality, 20 leaders: 2.527880e-03 +Nonblocking Multileader, 20 leaders: 1.664681e-03 +Nonblocking Locality Aware, 20 leaders: 1.502625e-03 +Nonblocking Multileader Locality, 20 leaders: 2.139845e-03 +Size 256 +PMPI: 6.171665e-03 +Pairwise: 3.581984e-03 +NonBlocking: 1.007113e-01 +Pairwise Hierarchical: 6.690865e-02 +Pairwise Node Aware: 2.958200e-03 +Nonblocking Hierarchical: 6.338459e-02 +Nonblocking Node Aware: 2.057269e-03 +Pairwise Multileader, 4 leaders: 2.096588e-02 +Pairwise Locality Aware, 4 leaders: 3.360438e-03 +Pairwise Multileader Locality, 4 leaders: 1.975664e-02 +Nonblocking Multileader, 4 leaders: 1.493872e-02 +Nonblocking Locality Aware, 4 leaders: 2.637062e-03 +Nonblocking Multileader Locality, 4 leaders: 1.691241e-02 +Pairwise Multileader, 10 leaders: 1.209133e-02 +Pairwise Locality Aware, 10 leaders: 3.042155e-03 +Pairwise Multileader Locality, 10 leaders: 9.113374e-03 +Nonblocking Multileader, 10 leaders: 7.653216e-03 +Nonblocking Locality Aware, 10 leaders: 3.301074e-03 +Nonblocking Multileader Locality, 10 leaders: 8.083522e-03 +Pairwise Multileader, 20 leaders: 5.099060e-03 +Pairwise Locality Aware, 20 leaders: 2.861406e-03 +Pairwise Multileader Locality, 20 leaders: 4.795923e-03 +Nonblocking Multileader, 20 leaders: 3.214822e-03 +Nonblocking Locality Aware, 20 leaders: 2.331437e-03 +Nonblocking Multileader Locality, 20 leaders: 4.013658e-03 +Size 512 +PMPI: 7.317590e-03 +Pairwise: 6.544347e-03 +NonBlocking: 8.354503e-02 +Pairwise Hierarchical: 1.286043e-01 +Pairwise Node Aware: 5.914260e-03 +Nonblocking Hierarchical: 1.238736e-01 +Nonblocking Node Aware: 3.562812e-03 +Pairwise Multileader, 4 leaders: 4.655468e-02 +Pairwise Locality Aware, 4 leaders: 6.191635e-03 +Pairwise Multileader Locality, 4 leaders: 4.812716e-02 +Nonblocking Multileader, 4 leaders: 3.486669e-02 +Nonblocking Locality Aware, 4 leaders: 3.736659e-03 +Nonblocking Multileader Locality, 4 leaders: 4.065157e-02 +Pairwise Multileader, 10 leaders: 2.471404e-02 +Pairwise Locality Aware, 10 leaders: 6.899867e-03 +Pairwise Multileader Locality, 10 leaders: 2.163024e-02 +Nonblocking Multileader, 10 leaders: 1.663649e-02 +Nonblocking Locality Aware, 10 leaders: 5.446553e-03 +Nonblocking Multileader Locality, 10 leaders: 1.889717e-02 +Pairwise Multileader, 20 leaders: 1.174797e-02 +Pairwise Locality Aware, 20 leaders: 6.419056e-03 +Pairwise Multileader Locality, 20 leaders: 1.041273e-02 +Nonblocking Multileader, 20 leaders: 7.530139e-03 +Nonblocking Locality Aware, 20 leaders: 6.511661e-03 +Nonblocking Multileader Locality, 20 leaders: 8.782219e-03 +Size 1024 +PMPI: 1.351585e-02 +Pairwise: 1.114464e-02 +NonBlocking: 2.700522e-02 +Pairwise Hierarchical: 2.305630e-01 +Pairwise Node Aware: 1.376700e-02 +Nonblocking Hierarchical: 2.360868e-01 +Nonblocking Node Aware: 8.628947e-03 +Pairwise Multileader, 4 leaders: 8.483706e-02 +Pairwise Locality Aware, 4 leaders: 1.157176e-02 +Pairwise Multileader Locality, 4 leaders: 9.088932e-02 +Nonblocking Multileader, 4 leaders: 6.521492e-02 +Nonblocking Locality Aware, 4 leaders: 7.682098e-03 +Nonblocking Multileader Locality, 4 leaders: 7.696260e-02 +Pairwise Multileader, 10 leaders: 4.734861e-02 +Pairwise Locality Aware, 10 leaders: 1.346662e-02 +Pairwise Multileader Locality, 10 leaders: 4.724174e-02 +Nonblocking Multileader, 10 leaders: 3.437814e-02 +Nonblocking Locality Aware, 10 leaders: 9.977474e-03 +Nonblocking Multileader Locality, 10 leaders: 4.092614e-02 +Pairwise Multileader, 20 leaders: 2.998883e-02 +Pairwise Locality Aware, 20 leaders: 1.498471e-02 +Pairwise Multileader Locality, 20 leaders: 2.277271e-02 +Nonblocking Multileader, 20 leaders: 1.753231e-02 +Nonblocking Locality Aware, 20 leaders: 1.075345e-02 +Nonblocking Multileader Locality, 20 leaders: 1.990797e-02 +Size 2048 +PMPI: 2.855197e-02 +Pairwise: 2.442043e-02 +NonBlocking: 4.403103e-02 +Pairwise Hierarchical: 4.865541e-01 +Pairwise Node Aware: 2.827899e-02 +Nonblocking Hierarchical: 4.910747e-01 +Nonblocking Node Aware: 2.107164e-02 +Pairwise Multileader, 4 leaders: 1.560988e-01 +Pairwise Locality Aware, 4 leaders: 2.685077e-02 +Pairwise Multileader Locality, 4 leaders: 1.713678e-01 +Nonblocking Multileader, 4 leaders: 1.264140e-01 +Nonblocking Locality Aware, 4 leaders: 1.760178e-02 +Nonblocking Multileader Locality, 4 leaders: 1.528312e-01 +Pairwise Multileader, 10 leaders: 9.218105e-02 +Pairwise Locality Aware, 10 leaders: 2.569885e-02 +Pairwise Multileader Locality, 10 leaders: 9.697067e-02 +Nonblocking Multileader, 10 leaders: 6.899536e-02 +Nonblocking Locality Aware, 10 leaders: 1.739902e-02 +Nonblocking Multileader Locality, 10 leaders: 8.501536e-02 +Pairwise Multileader, 20 leaders: 5.468033e-02 +Pairwise Locality Aware, 20 leaders: 2.865201e-02 +Pairwise Multileader Locality, 20 leaders: 4.841904e-02 +Nonblocking Multileader, 20 leaders: 3.552496e-02 +Nonblocking Locality Aware, 20 leaders: 2.226663e-02 +Nonblocking Multileader Locality, 20 leaders: 4.176542e-02 +Size 4096 +PMPI: 5.452059e-02 +Pairwise: 4.681675e-02 +NonBlocking: 7.854784e-02 +Pairwise Hierarchical: 1.059395e+00 +Pairwise Node Aware: 5.045677e-02 +Nonblocking Hierarchical: 1.074835e+00 +Nonblocking Node Aware: 3.956202e-02 +Pairwise Multileader, 4 leaders: 2.957911e-01 +Pairwise Locality Aware, 4 leaders: 5.242875e-02 +Pairwise Multileader Locality, 4 leaders: 3.248867e-01 +Nonblocking Multileader, 4 leaders: 2.397033e-01 +Nonblocking Locality Aware, 4 leaders: 3.647875e-02 +Nonblocking Multileader Locality, 4 leaders: 2.931212e-01 +Pairwise Multileader, 10 leaders: 1.721049e-01 +Pairwise Locality Aware, 10 leaders: 5.327645e-02 +Pairwise Multileader Locality, 10 leaders: 1.847058e-01 +Nonblocking Multileader, 10 leaders: 1.322328e-01 +Nonblocking Locality Aware, 10 leaders: 3.659760e-02 +Nonblocking Multileader Locality, 10 leaders: 1.676109e-01 +Pairwise Multileader, 20 leaders: 1.119118e-01 +Pairwise Locality Aware, 20 leaders: 5.274509e-02 +Pairwise Multileader Locality, 20 leaders: 1.112258e-01 +Nonblocking Multileader, 20 leaders: 7.841417e-02 +Nonblocking Locality Aware, 20 leaders: 3.838966e-02 +Nonblocking Multileader Locality, 20 leaders: 9.633188e-02 +Size 8192 +PMPI: 8.462412e-02 +Pairwise: 9.915763e-02 +NonBlocking: 7.635351e-02 +Pairwise Hierarchical: 2.201142e+00 +Pairwise Node Aware: 9.480702e-02 +Nonblocking Hierarchical: 2.101992e+00 +Nonblocking Node Aware: 8.005940e-02 +Pairwise Multileader, 4 leaders: 5.754466e-01 +Pairwise Locality Aware, 4 leaders: 1.046198e-01 +Pairwise Multileader Locality, 4 leaders: 7.021391e-01 +Nonblocking Multileader, 4 leaders: 5.398901e-01 +Nonblocking Locality Aware, 4 leaders: 7.331527e-02 +Nonblocking Multileader Locality, 4 leaders: 6.470958e-01 +Pairwise Multileader, 10 leaders: 3.510066e-01 +Pairwise Locality Aware, 10 leaders: 1.042407e-01 +Pairwise Multileader Locality, 10 leaders: 4.044046e-01 +Nonblocking Multileader, 10 leaders: 2.657943e-01 +Nonblocking Locality Aware, 10 leaders: 7.713408e-02 +Nonblocking Multileader Locality, 10 leaders: 3.331838e-01 +Pairwise Multileader, 20 leaders: 1.972144e-01 +Pairwise Locality Aware, 20 leaders: 1.110599e-01 +Pairwise Multileader Locality, 20 leaders: 2.183719e-01 +Nonblocking Multileader, 20 leaders: 1.493967e-01 +Nonblocking Locality Aware, 20 leaders: 7.884174e-02 +Nonblocking Multileader Locality, 20 leaders: 1.865343e-01 +Size 16384 +PMPI: 1.525945e-01 +Pairwise: 1.626592e-01 +NonBlocking: 1.179548e-01 +Test 2 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 8.494959e-05 +Pairwise: 1.128560e-03 +NonBlocking: 3.977076e-04 +Pairwise Hierarchical: 3.245683e-04 +Pairwise Node Aware: 2.106898e-04 +Nonblocking Hierarchical: 3.101973e-04 +Nonblocking Node Aware: 1.936688e-04 +Pairwise Multileader, 4 leaders: 1.786569e-04 +Pairwise Locality Aware, 4 leaders: 1.223461e-04 +Pairwise Multileader Locality, 4 leaders: 1.219653e-04 +Nonblocking Multileader, 4 leaders: 9.155434e-05 +Nonblocking Locality Aware, 4 leaders: 7.362585e-05 +Nonblocking Multileader Locality, 4 leaders: 1.026035e-04 +Pairwise Multileader, 10 leaders: 2.867547e-04 +Pairwise Locality Aware, 10 leaders: 2.070061e-04 +Pairwise Multileader Locality, 10 leaders: 8.096243e-05 +Nonblocking Multileader, 10 leaders: 7.147993e-05 +Nonblocking Locality Aware, 10 leaders: 9.962626e-05 +Nonblocking Multileader Locality, 10 leaders: 6.588991e-05 +Pairwise Multileader, 20 leaders: 5.052014e-04 +Pairwise Locality Aware, 20 leaders: 4.511963e-04 +Pairwise Multileader Locality, 20 leaders: 9.040032e-05 +Nonblocking Multileader, 20 leaders: 1.111707e-04 +Nonblocking Locality Aware, 20 leaders: 1.921770e-04 +Nonblocking Multileader Locality, 20 leaders: 6.042988e-05 +Size 2 +PMPI: 1.052139e-04 +Pairwise: 1.171324e-03 +NonBlocking: 2.880394e-04 +Pairwise Hierarchical: 6.243564e-04 +Pairwise Node Aware: 2.531418e-04 +Nonblocking Hierarchical: 5.532391e-04 +Nonblocking Node Aware: 2.015524e-04 +Pairwise Multileader, 4 leaders: 1.803420e-04 +Pairwise Locality Aware, 4 leaders: 1.633583e-04 +Pairwise Multileader Locality, 4 leaders: 1.621761e-04 +Nonblocking Multileader, 4 leaders: 1.187008e-04 +Nonblocking Locality Aware, 4 leaders: 1.051760e-04 +Nonblocking Multileader Locality, 4 leaders: 1.342435e-04 +Pairwise Multileader, 10 leaders: 2.836247e-04 +Pairwise Locality Aware, 10 leaders: 2.723048e-04 +Pairwise Multileader Locality, 10 leaders: 9.696898e-05 +Nonblocking Multileader, 10 leaders: 8.268865e-05 +Nonblocking Locality Aware, 10 leaders: 1.610235e-04 +Nonblocking Multileader Locality, 10 leaders: 7.807517e-05 +Pairwise Multileader, 20 leaders: 4.508000e-04 +Pairwise Locality Aware, 20 leaders: 5.910001e-04 +Pairwise Multileader Locality, 20 leaders: 1.034632e-04 +Nonblocking Multileader, 20 leaders: 1.126363e-04 +Nonblocking Locality Aware, 20 leaders: 1.469007e-04 +Nonblocking Multileader Locality, 20 leaders: 7.603297e-05 +Size 4 +PMPI: 1.651407e-04 +Pairwise: 2.208161e-03 +NonBlocking: 3.748446e-02 +Pairwise Hierarchical: 9.496039e-04 +Pairwise Node Aware: 3.299698e-04 +Nonblocking Hierarchical: 8.521514e-04 +Nonblocking Node Aware: 2.007802e-04 +Pairwise Multileader, 4 leaders: 3.666936e-04 +Pairwise Locality Aware, 4 leaders: 1.520142e-04 +Pairwise Multileader Locality, 4 leaders: 3.258725e-04 +Nonblocking Multileader, 4 leaders: 2.215499e-04 +Nonblocking Locality Aware, 4 leaders: 1.227576e-04 +Nonblocking Multileader Locality, 4 leaders: 2.757953e-04 +Pairwise Multileader, 10 leaders: 3.091668e-04 +Pairwise Locality Aware, 10 leaders: 2.498949e-04 +Pairwise Multileader Locality, 10 leaders: 1.421724e-04 +Nonblocking Multileader, 10 leaders: 1.062570e-04 +Nonblocking Locality Aware, 10 leaders: 1.627899e-04 +Nonblocking Multileader Locality, 10 leaders: 1.100496e-04 +Pairwise Multileader, 20 leaders: 5.451047e-04 +Pairwise Locality Aware, 20 leaders: 4.500300e-04 +Pairwise Multileader Locality, 20 leaders: 1.233019e-04 +Nonblocking Multileader, 20 leaders: 1.420529e-04 +Nonblocking Locality Aware, 20 leaders: 3.441150e-04 +Nonblocking Multileader Locality, 20 leaders: 8.850616e-05 +Size 8 +PMPI: 2.612635e-04 +Pairwise: 2.630672e-03 +NonBlocking: 5.760757e-02 +Pairwise Hierarchical: 1.810559e-03 +Pairwise Node Aware: 3.703680e-04 +Nonblocking Hierarchical: 1.638596e-03 +Nonblocking Node Aware: 1.969021e-04 +Pairwise Multileader, 4 leaders: 5.726102e-04 +Pairwise Locality Aware, 4 leaders: 1.649821e-04 +Pairwise Multileader Locality, 4 leaders: 7.103411e-04 +Nonblocking Multileader, 4 leaders: 3.889815e-04 +Nonblocking Locality Aware, 4 leaders: 1.873733e-04 +Nonblocking Multileader Locality, 4 leaders: 5.565757e-04 +Pairwise Multileader, 10 leaders: 4.364729e-04 +Pairwise Locality Aware, 10 leaders: 2.792005e-04 +Pairwise Multileader Locality, 10 leaders: 2.560847e-04 +Nonblocking Multileader, 10 leaders: 1.824392e-04 +Nonblocking Locality Aware, 10 leaders: 1.520472e-04 +Nonblocking Multileader Locality, 10 leaders: 2.068628e-04 +Pairwise Multileader, 20 leaders: 5.285936e-04 +Pairwise Locality Aware, 20 leaders: 4.757390e-04 +Pairwise Multileader Locality, 20 leaders: 1.880796e-04 +Nonblocking Multileader, 20 leaders: 1.443087e-04 +Nonblocking Locality Aware, 20 leaders: 2.487081e-04 +Nonblocking Multileader Locality, 20 leaders: 1.289952e-04 +Size 16 +PMPI: 4.534827e-04 +Pairwise: 2.711888e-03 +NonBlocking: 6.675467e-02 +Pairwise Hierarchical: 3.368437e-03 +Pairwise Node Aware: 4.611964e-04 +Nonblocking Hierarchical: 3.126461e-03 +Nonblocking Node Aware: 2.770235e-04 +Pairwise Multileader, 4 leaders: 9.860675e-04 +Pairwise Locality Aware, 4 leaders: 2.618704e-04 +Pairwise Multileader Locality, 4 leaders: 1.273162e-03 +Nonblocking Multileader, 4 leaders: 7.652501e-04 +Nonblocking Locality Aware, 4 leaders: 1.952867e-04 +Nonblocking Multileader Locality, 4 leaders: 1.041204e-03 +Pairwise Multileader, 10 leaders: 7.207294e-04 +Pairwise Locality Aware, 10 leaders: 3.092657e-04 +Pairwise Multileader Locality, 10 leaders: 4.658506e-04 +Nonblocking Multileader, 10 leaders: 3.318291e-04 +Nonblocking Locality Aware, 10 leaders: 2.227917e-04 +Nonblocking Multileader Locality, 10 leaders: 4.118721e-04 +Pairwise Multileader, 20 leaders: 7.338628e-04 +Pairwise Locality Aware, 20 leaders: 4.960127e-04 +Pairwise Multileader Locality, 20 leaders: 3.152158e-04 +Nonblocking Multileader, 20 leaders: 2.432474e-04 +Nonblocking Locality Aware, 20 leaders: 2.783601e-04 +Nonblocking Multileader Locality, 20 leaders: 2.065706e-04 +Size 32 +PMPI: 8.985770e-04 +Pairwise: 2.047255e-03 +NonBlocking: 7.439932e-02 +Pairwise Hierarchical: 7.579397e-03 +Pairwise Node Aware: 6.505589e-04 +Nonblocking Hierarchical: 6.967940e-03 +Nonblocking Node Aware: 4.815199e-04 +Pairwise Multileader, 4 leaders: 2.954913e-03 +Pairwise Locality Aware, 4 leaders: 4.348642e-04 +Pairwise Multileader Locality, 4 leaders: 2.653439e-03 +Nonblocking Multileader, 4 leaders: 2.247870e-03 +Nonblocking Locality Aware, 4 leaders: 2.869654e-04 +Nonblocking Multileader Locality, 4 leaders: 2.372747e-03 +Pairwise Multileader, 10 leaders: 1.216751e-03 +Pairwise Locality Aware, 10 leaders: 4.435568e-04 +Pairwise Multileader Locality, 10 leaders: 1.148724e-03 +Nonblocking Multileader, 10 leaders: 7.300897e-04 +Nonblocking Locality Aware, 10 leaders: 3.817165e-04 +Nonblocking Multileader Locality, 10 leaders: 9.688149e-04 +Pairwise Multileader, 20 leaders: 8.952556e-04 +Pairwise Locality Aware, 20 leaders: 7.058055e-04 +Pairwise Multileader Locality, 20 leaders: 5.516283e-04 +Nonblocking Multileader, 20 leaders: 4.427494e-04 +Nonblocking Locality Aware, 20 leaders: 4.688724e-04 +Nonblocking Multileader Locality, 20 leaders: 4.482005e-04 +Size 64 +PMPI: 1.707512e-03 +Pairwise: 2.139976e-03 +NonBlocking: 6.938543e-02 +Pairwise Hierarchical: 1.727902e-02 +Pairwise Node Aware: 1.081874e-03 +Nonblocking Hierarchical: 1.515325e-02 +Nonblocking Node Aware: 6.559874e-04 +Pairwise Multileader, 4 leaders: 5.206959e-03 +Pairwise Locality Aware, 4 leaders: 8.224822e-04 +Pairwise Multileader Locality, 4 leaders: 5.158664e-03 +Nonblocking Multileader, 4 leaders: 4.050063e-03 +Nonblocking Locality Aware, 4 leaders: 7.298621e-04 +Nonblocking Multileader Locality, 4 leaders: 4.482338e-03 +Pairwise Multileader, 10 leaders: 2.286937e-03 +Pairwise Locality Aware, 10 leaders: 9.355198e-04 +Pairwise Multileader Locality, 10 leaders: 2.168113e-03 +Nonblocking Multileader, 10 leaders: 1.605587e-03 +Nonblocking Locality Aware, 10 leaders: 7.529603e-04 +Nonblocking Multileader Locality, 10 leaders: 1.889853e-03 +Pairwise Multileader, 20 leaders: 1.630200e-03 +Pairwise Locality Aware, 20 leaders: 1.029236e-03 +Pairwise Multileader Locality, 20 leaders: 1.270743e-03 +Nonblocking Multileader, 20 leaders: 7.741899e-04 +Nonblocking Locality Aware, 20 leaders: 9.248722e-04 +Nonblocking Multileader Locality, 20 leaders: 1.067735e-03 +Size 128 +PMPI: 2.974335e-03 +Pairwise: 3.077578e-03 +NonBlocking: 8.736033e-02 +Pairwise Hierarchical: 3.629376e-02 +Pairwise Node Aware: 2.036771e-03 +Nonblocking Hierarchical: 3.295812e-02 +Nonblocking Node Aware: 1.241153e-03 +Pairwise Multileader, 4 leaders: 9.661497e-03 +Pairwise Locality Aware, 4 leaders: 1.636289e-03 +Pairwise Multileader Locality, 4 leaders: 9.809184e-03 +Nonblocking Multileader, 4 leaders: 7.176396e-03 +Nonblocking Locality Aware, 4 leaders: 1.601798e-03 +Nonblocking Multileader Locality, 4 leaders: 8.141541e-03 +Pairwise Multileader, 10 leaders: 4.486287e-03 +Pairwise Locality Aware, 10 leaders: 1.652910e-03 +Pairwise Multileader Locality, 10 leaders: 4.198032e-03 +Nonblocking Multileader, 10 leaders: 3.218814e-03 +Nonblocking Locality Aware, 10 leaders: 1.435122e-03 +Nonblocking Multileader Locality, 10 leaders: 3.695000e-03 +Pairwise Multileader, 20 leaders: 2.924202e-03 +Pairwise Locality Aware, 20 leaders: 1.428040e-03 +Pairwise Multileader Locality, 20 leaders: 2.576854e-03 +Nonblocking Multileader, 20 leaders: 1.575731e-03 +Nonblocking Locality Aware, 20 leaders: 1.133968e-03 +Nonblocking Multileader Locality, 20 leaders: 2.147614e-03 +Size 256 +PMPI: 5.495343e-03 +Pairwise: 4.533308e-03 +NonBlocking: 9.374377e-02 +Pairwise Hierarchical: 6.805219e-02 +Pairwise Node Aware: 2.797055e-03 +Nonblocking Hierarchical: 6.407635e-02 +Nonblocking Node Aware: 1.860630e-03 +Pairwise Multileader, 4 leaders: 2.142128e-02 +Pairwise Locality Aware, 4 leaders: 2.689225e-03 +Pairwise Multileader Locality, 4 leaders: 1.994915e-02 +Nonblocking Multileader, 4 leaders: 1.478215e-02 +Nonblocking Locality Aware, 4 leaders: 2.084324e-03 +Nonblocking Multileader Locality, 4 leaders: 1.671944e-02 +Pairwise Multileader, 10 leaders: 1.204381e-02 +Pairwise Locality Aware, 10 leaders: 3.469127e-03 +Pairwise Multileader Locality, 10 leaders: 8.998473e-03 +Nonblocking Multileader, 10 leaders: 7.652632e-03 +Nonblocking Locality Aware, 10 leaders: 3.309156e-03 +Nonblocking Multileader Locality, 10 leaders: 7.950546e-03 +Pairwise Multileader, 20 leaders: 5.474321e-03 +Pairwise Locality Aware, 20 leaders: 2.641858e-03 +Pairwise Multileader Locality, 20 leaders: 4.741220e-03 +Nonblocking Multileader, 20 leaders: 3.229928e-03 +Nonblocking Locality Aware, 20 leaders: 2.205841e-03 +Nonblocking Multileader Locality, 20 leaders: 4.046504e-03 +Size 512 +PMPI: 6.903764e-03 +Pairwise: 7.641342e-03 +NonBlocking: 9.656501e-02 +Pairwise Hierarchical: 1.270377e-01 +Pairwise Node Aware: 6.069195e-03 +Nonblocking Hierarchical: 1.271048e-01 +Nonblocking Node Aware: 3.705993e-03 +Pairwise Multileader, 4 leaders: 4.671617e-02 +Pairwise Locality Aware, 4 leaders: 5.427060e-03 +Pairwise Multileader Locality, 4 leaders: 4.849376e-02 +Nonblocking Multileader, 4 leaders: 3.550035e-02 +Nonblocking Locality Aware, 4 leaders: 3.948567e-03 +Nonblocking Multileader Locality, 4 leaders: 4.056016e-02 +Pairwise Multileader, 10 leaders: 2.462602e-02 +Pairwise Locality Aware, 10 leaders: 6.059999e-03 +Pairwise Multileader Locality, 10 leaders: 2.175794e-02 +Nonblocking Multileader, 10 leaders: 1.668108e-02 +Nonblocking Locality Aware, 10 leaders: 5.249593e-03 +Nonblocking Multileader Locality, 10 leaders: 1.885449e-02 +Pairwise Multileader, 20 leaders: 1.081160e-02 +Pairwise Locality Aware, 20 leaders: 6.004783e-03 +Pairwise Multileader Locality, 20 leaders: 1.026955e-02 +Nonblocking Multileader, 20 leaders: 7.235294e-03 +Nonblocking Locality Aware, 20 leaders: 4.636132e-03 +Nonblocking Multileader Locality, 20 leaders: 8.734501e-03 +Size 1024 +PMPI: 1.451215e-02 +Pairwise: 1.241389e-02 +NonBlocking: 5.383004e-02 +Pairwise Hierarchical: 2.376469e-01 +Pairwise Node Aware: 1.425522e-02 +Nonblocking Hierarchical: 2.379684e-01 +Nonblocking Node Aware: 7.972149e-03 +Pairwise Multileader, 4 leaders: 8.354147e-02 +Pairwise Locality Aware, 4 leaders: 1.085985e-02 +Pairwise Multileader Locality, 4 leaders: 8.948052e-02 +Nonblocking Multileader, 4 leaders: 6.541488e-02 +Nonblocking Locality Aware, 4 leaders: 7.998718e-03 +Nonblocking Multileader Locality, 4 leaders: 7.860323e-02 +Pairwise Multileader, 10 leaders: 4.775728e-02 +Pairwise Locality Aware, 10 leaders: 1.250981e-02 +Pairwise Multileader Locality, 10 leaders: 4.709947e-02 +Nonblocking Multileader, 10 leaders: 3.409164e-02 +Nonblocking Locality Aware, 10 leaders: 1.074096e-02 +Nonblocking Multileader Locality, 10 leaders: 4.062646e-02 +Pairwise Multileader, 20 leaders: 3.088177e-02 +Pairwise Locality Aware, 20 leaders: 1.372168e-02 +Pairwise Multileader Locality, 20 leaders: 2.270122e-02 +Nonblocking Multileader, 20 leaders: 1.769057e-02 +Nonblocking Locality Aware, 20 leaders: 1.019211e-02 +Nonblocking Multileader Locality, 20 leaders: 2.004356e-02 +Size 2048 +PMPI: 2.406252e-02 +Pairwise: 2.727157e-02 +NonBlocking: 4.603093e-02 +Pairwise Hierarchical: 4.802740e-01 +Pairwise Node Aware: 2.778353e-02 +Nonblocking Hierarchical: 4.978441e-01 +Nonblocking Node Aware: 2.112542e-02 +Pairwise Multileader, 4 leaders: 1.563108e-01 +Pairwise Locality Aware, 4 leaders: 2.565887e-02 +Pairwise Multileader Locality, 4 leaders: 1.701512e-01 +Nonblocking Multileader, 4 leaders: 1.253726e-01 +Nonblocking Locality Aware, 4 leaders: 1.793259e-02 +Nonblocking Multileader Locality, 4 leaders: 1.546261e-01 +Pairwise Multileader, 10 leaders: 9.200720e-02 +Pairwise Locality Aware, 10 leaders: 2.648309e-02 +Pairwise Multileader Locality, 10 leaders: 9.532460e-02 +Nonblocking Multileader, 10 leaders: 6.902795e-02 +Nonblocking Locality Aware, 10 leaders: 1.730110e-02 +Nonblocking Multileader Locality, 10 leaders: 8.496577e-02 +Pairwise Multileader, 20 leaders: 5.484598e-02 +Pairwise Locality Aware, 20 leaders: 2.800515e-02 +Pairwise Multileader Locality, 20 leaders: 4.854325e-02 +Nonblocking Multileader, 20 leaders: 3.546573e-02 +Nonblocking Locality Aware, 20 leaders: 2.101094e-02 +Nonblocking Multileader Locality, 20 leaders: 4.255791e-02 +Size 4096 +PMPI: 5.380233e-02 +Pairwise: 5.049260e-02 +NonBlocking: 7.480955e-02 +Pairwise Hierarchical: 1.057999e+00 +Pairwise Node Aware: 4.997675e-02 +Nonblocking Hierarchical: 1.039995e+00 +Nonblocking Node Aware: 3.904464e-02 +Pairwise Multileader, 4 leaders: 2.965709e-01 +Pairwise Locality Aware, 4 leaders: 4.892519e-02 +Pairwise Multileader Locality, 4 leaders: 3.268304e-01 +Nonblocking Multileader, 4 leaders: 2.415008e-01 +Nonblocking Locality Aware, 4 leaders: 3.758158e-02 +Nonblocking Multileader Locality, 4 leaders: 2.941340e-01 +Pairwise Multileader, 10 leaders: 1.723810e-01 +Pairwise Locality Aware, 10 leaders: 5.305371e-02 +Pairwise Multileader Locality, 10 leaders: 1.889240e-01 +Nonblocking Multileader, 10 leaders: 1.319824e-01 +Nonblocking Locality Aware, 10 leaders: 3.711622e-02 +Nonblocking Multileader Locality, 10 leaders: 1.670446e-01 +Pairwise Multileader, 20 leaders: 1.099744e-01 +Pairwise Locality Aware, 20 leaders: 5.691983e-02 +Pairwise Multileader Locality, 20 leaders: 1.096570e-01 +Nonblocking Multileader, 20 leaders: 7.797964e-02 +Nonblocking Locality Aware, 20 leaders: 3.746337e-02 +Nonblocking Multileader Locality, 20 leaders: 9.751081e-02 +Size 8192 +PMPI: 9.898921e-02 +Pairwise: 8.551134e-02 +NonBlocking: 8.031036e-02 +Pairwise Hierarchical: 2.211365e+00 +Pairwise Node Aware: 9.724662e-02 +Nonblocking Hierarchical: 2.231126e+00 +Nonblocking Node Aware: 7.857937e-02 +Pairwise Multileader, 4 leaders: 5.753694e-01 +Pairwise Locality Aware, 4 leaders: 9.837456e-02 +Pairwise Multileader Locality, 4 leaders: 7.005879e-01 +Nonblocking Multileader, 4 leaders: 5.426732e-01 +Nonblocking Locality Aware, 4 leaders: 7.797137e-02 +Nonblocking Multileader Locality, 4 leaders: 6.601148e-01 +Pairwise Multileader, 10 leaders: 3.490606e-01 +Pairwise Locality Aware, 10 leaders: 1.068634e-01 +Pairwise Multileader Locality, 10 leaders: 4.026395e-01 +Nonblocking Multileader, 10 leaders: 2.646758e-01 +Nonblocking Locality Aware, 10 leaders: 7.832779e-02 +Nonblocking Multileader Locality, 10 leaders: 3.330566e-01 +Pairwise Multileader, 20 leaders: 1.945484e-01 +Pairwise Locality Aware, 20 leaders: 1.118592e-01 +Pairwise Multileader Locality, 20 leaders: 2.169448e-01 +Nonblocking Multileader, 20 leaders: 1.494363e-01 +Nonblocking Locality Aware, 20 leaders: 7.747509e-02 +Nonblocking Multileader Locality, 20 leaders: 1.861424e-01 +Size 16384 +PMPI: 1.539002e-01 +Pairwise: 1.611086e-01 +NonBlocking: 1.251470e-01 +Test 3 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 9.003539e-05 +Pairwise: 9.837995e-04 +NonBlocking: 4.456171e-04 +Pairwise Hierarchical: 3.236127e-04 +Pairwise Node Aware: 2.092828e-04 +Nonblocking Hierarchical: 3.084329e-04 +Nonblocking Node Aware: 1.929396e-04 +Pairwise Multileader, 4 leaders: 1.845151e-04 +Pairwise Locality Aware, 4 leaders: 1.409551e-04 +Pairwise Multileader Locality, 4 leaders: 1.268247e-04 +Nonblocking Multileader, 4 leaders: 9.029125e-05 +Nonblocking Locality Aware, 4 leaders: 7.376163e-05 +Nonblocking Multileader Locality, 4 leaders: 1.060327e-04 +Pairwise Multileader, 10 leaders: 2.653402e-04 +Pairwise Locality Aware, 10 leaders: 2.248713e-04 +Pairwise Multileader Locality, 10 leaders: 8.420000e-05 +Nonblocking Multileader, 10 leaders: 6.826789e-05 +Nonblocking Locality Aware, 10 leaders: 1.107833e-04 +Nonblocking Multileader Locality, 10 leaders: 6.503703e-05 +Pairwise Multileader, 20 leaders: 4.433502e-04 +Pairwise Locality Aware, 20 leaders: 5.337679e-04 +Pairwise Multileader Locality, 20 leaders: 8.773740e-05 +Nonblocking Multileader, 20 leaders: 1.118888e-04 +Nonblocking Locality Aware, 20 leaders: 1.344503e-04 +Nonblocking Multileader Locality, 20 leaders: 6.358968e-05 +Size 2 +PMPI: 1.048430e-04 +Pairwise: 1.103600e-03 +NonBlocking: 3.505591e-04 +Pairwise Hierarchical: 6.139773e-04 +Pairwise Node Aware: 2.560210e-04 +Nonblocking Hierarchical: 5.514643e-04 +Nonblocking Node Aware: 2.182534e-04 +Pairwise Multileader, 4 leaders: 2.182108e-04 +Pairwise Locality Aware, 4 leaders: 1.679610e-04 +Pairwise Multileader Locality, 4 leaders: 1.638677e-04 +Nonblocking Multileader, 4 leaders: 1.133716e-04 +Nonblocking Locality Aware, 4 leaders: 8.650351e-05 +Nonblocking Multileader Locality, 4 leaders: 1.400513e-04 +Pairwise Multileader, 10 leaders: 2.891274e-04 +Pairwise Locality Aware, 10 leaders: 2.386779e-04 +Pairwise Multileader Locality, 10 leaders: 9.891240e-05 +Nonblocking Multileader, 10 leaders: 8.384544e-05 +Nonblocking Locality Aware, 10 leaders: 1.093076e-04 +Nonblocking Multileader Locality, 10 leaders: 8.452583e-05 +Pairwise Multileader, 20 leaders: 4.547672e-04 +Pairwise Locality Aware, 20 leaders: 5.353258e-04 +Pairwise Multileader Locality, 20 leaders: 1.011395e-04 +Nonblocking Multileader, 20 leaders: 1.203025e-04 +Nonblocking Locality Aware, 20 leaders: 1.463760e-04 +Nonblocking Multileader Locality, 20 leaders: 7.140667e-05 +Size 4 +PMPI: 1.605615e-04 +Pairwise: 2.388687e-03 +NonBlocking: 6.204730e-02 +Pairwise Hierarchical: 9.782758e-04 +Pairwise Node Aware: 3.315311e-04 +Nonblocking Hierarchical: 8.515919e-04 +Nonblocking Node Aware: 2.036703e-04 +Pairwise Multileader, 4 leaders: 3.171425e-04 +Pairwise Locality Aware, 4 leaders: 1.740732e-04 +Pairwise Multileader Locality, 4 leaders: 3.265484e-04 +Nonblocking Multileader, 4 leaders: 2.177254e-04 +Nonblocking Locality Aware, 4 leaders: 1.083219e-04 +Nonblocking Multileader Locality, 4 leaders: 2.692455e-04 +Pairwise Multileader, 10 leaders: 3.348813e-04 +Pairwise Locality Aware, 10 leaders: 2.358333e-04 +Pairwise Multileader Locality, 10 leaders: 1.373320e-04 +Nonblocking Multileader, 10 leaders: 1.160427e-04 +Nonblocking Locality Aware, 10 leaders: 1.482963e-04 +Nonblocking Multileader Locality, 10 leaders: 1.148415e-04 +Pairwise Multileader, 20 leaders: 5.671857e-04 +Pairwise Locality Aware, 20 leaders: 4.901539e-04 +Pairwise Multileader Locality, 20 leaders: 1.420330e-04 +Nonblocking Multileader, 20 leaders: 1.240113e-04 +Nonblocking Locality Aware, 20 leaders: 4.005606e-04 +Nonblocking Multileader Locality, 20 leaders: 1.026537e-04 +Size 8 +PMPI: 2.534318e-04 +Pairwise: 2.339394e-03 +NonBlocking: 8.676914e-02 +Pairwise Hierarchical: 1.774107e-03 +Pairwise Node Aware: 3.818504e-04 +Nonblocking Hierarchical: 1.636045e-03 +Nonblocking Node Aware: 2.173112e-04 +Pairwise Multileader, 4 leaders: 5.801781e-04 +Pairwise Locality Aware, 4 leaders: 1.728502e-04 +Pairwise Multileader Locality, 4 leaders: 6.900248e-04 +Nonblocking Multileader, 4 leaders: 3.929852e-04 +Nonblocking Locality Aware, 4 leaders: 2.007319e-04 +Nonblocking Multileader Locality, 4 leaders: 5.678120e-04 +Pairwise Multileader, 10 leaders: 4.066646e-04 +Pairwise Locality Aware, 10 leaders: 3.016014e-04 +Pairwise Multileader Locality, 10 leaders: 2.588642e-04 +Nonblocking Multileader, 10 leaders: 1.820031e-04 +Nonblocking Locality Aware, 10 leaders: 1.869433e-04 +Nonblocking Multileader Locality, 10 leaders: 2.119462e-04 +Pairwise Multileader, 20 leaders: 6.546384e-04 +Pairwise Locality Aware, 20 leaders: 4.342734e-04 +Pairwise Multileader Locality, 20 leaders: 2.222865e-04 +Nonblocking Multileader, 20 leaders: 1.612682e-04 +Nonblocking Locality Aware, 20 leaders: 2.036629e-04 +Nonblocking Multileader Locality, 20 leaders: 1.646817e-04 +Size 16 +PMPI: 4.092161e-04 +Pairwise: 2.459241e-03 +NonBlocking: 6.790504e-02 +Pairwise Hierarchical: 3.460950e-03 +Pairwise Node Aware: 4.487187e-04 +Nonblocking Hierarchical: 3.133212e-03 +Nonblocking Node Aware: 2.843565e-04 +Pairwise Multileader, 4 leaders: 1.005908e-03 +Pairwise Locality Aware, 4 leaders: 2.431309e-04 +Pairwise Multileader Locality, 4 leaders: 1.264315e-03 +Nonblocking Multileader, 4 leaders: 7.626428e-04 +Nonblocking Locality Aware, 4 leaders: 1.943928e-04 +Nonblocking Multileader Locality, 4 leaders: 1.052659e-03 +Pairwise Multileader, 10 leaders: 6.290052e-04 +Pairwise Locality Aware, 10 leaders: 3.966995e-04 +Pairwise Multileader Locality, 10 leaders: 4.718753e-04 +Nonblocking Multileader, 10 leaders: 3.369493e-04 +Nonblocking Locality Aware, 10 leaders: 2.816035e-04 +Nonblocking Multileader Locality, 10 leaders: 4.130463e-04 +Pairwise Multileader, 20 leaders: 6.561726e-04 +Pairwise Locality Aware, 20 leaders: 7.247824e-04 +Pairwise Multileader Locality, 20 leaders: 2.990198e-04 +Nonblocking Multileader, 20 leaders: 2.266919e-04 +Nonblocking Locality Aware, 20 leaders: 3.535534e-04 +Nonblocking Multileader Locality, 20 leaders: 2.056921e-04 +Size 32 +PMPI: 8.055351e-04 +Pairwise: 2.745292e-03 +NonBlocking: 8.548823e-02 +Pairwise Hierarchical: 7.590386e-03 +Pairwise Node Aware: 6.685266e-04 +Nonblocking Hierarchical: 7.069178e-03 +Nonblocking Node Aware: 4.665175e-04 +Pairwise Multileader, 4 leaders: 2.928107e-03 +Pairwise Locality Aware, 4 leaders: 4.433405e-04 +Pairwise Multileader Locality, 4 leaders: 2.707608e-03 +Nonblocking Multileader, 4 leaders: 2.267662e-03 +Nonblocking Locality Aware, 4 leaders: 3.073322e-04 +Nonblocking Multileader Locality, 4 leaders: 2.363972e-03 +Pairwise Multileader, 10 leaders: 1.147677e-03 +Pairwise Locality Aware, 10 leaders: 5.876902e-04 +Pairwise Multileader Locality, 10 leaders: 1.131453e-03 +Nonblocking Multileader, 10 leaders: 7.620383e-04 +Nonblocking Locality Aware, 10 leaders: 4.719683e-04 +Nonblocking Multileader Locality, 10 leaders: 9.756492e-04 +Pairwise Multileader, 20 leaders: 1.056596e-03 +Pairwise Locality Aware, 20 leaders: 6.207859e-04 +Pairwise Multileader Locality, 20 leaders: 6.102264e-04 +Nonblocking Multileader, 20 leaders: 4.024486e-04 +Nonblocking Locality Aware, 20 leaders: 3.489562e-04 +Nonblocking Multileader Locality, 20 leaders: 4.928846e-04 +Size 64 +PMPI: 1.680368e-03 +Pairwise: 2.508637e-03 +NonBlocking: 8.709414e-02 +Pairwise Hierarchical: 1.751088e-02 +Pairwise Node Aware: 9.822771e-04 +Nonblocking Hierarchical: 1.520069e-02 +Nonblocking Node Aware: 6.525371e-04 +Pairwise Multileader, 4 leaders: 5.119916e-03 +Pairwise Locality Aware, 4 leaders: 7.720803e-04 +Pairwise Multileader Locality, 4 leaders: 5.136549e-03 +Nonblocking Multileader, 4 leaders: 4.079787e-03 +Nonblocking Locality Aware, 4 leaders: 5.599634e-04 +Nonblocking Multileader Locality, 4 leaders: 4.460227e-03 +Pairwise Multileader, 10 leaders: 2.306914e-03 +Pairwise Locality Aware, 10 leaders: 9.286559e-04 +Pairwise Multileader Locality, 10 leaders: 2.173259e-03 +Nonblocking Multileader, 10 leaders: 1.657352e-03 +Nonblocking Locality Aware, 10 leaders: 8.839543e-04 +Nonblocking Multileader Locality, 10 leaders: 1.898617e-03 +Pairwise Multileader, 20 leaders: 1.639723e-03 +Pairwise Locality Aware, 20 leaders: 9.255357e-04 +Pairwise Multileader Locality, 20 leaders: 1.273217e-03 +Nonblocking Multileader, 20 leaders: 7.302745e-04 +Nonblocking Locality Aware, 20 leaders: 6.814257e-04 +Nonblocking Multileader Locality, 20 leaders: 1.071104e-03 +Size 128 +PMPI: 2.714578e-03 +Pairwise: 3.312167e-03 +NonBlocking: 8.756806e-02 +Pairwise Hierarchical: 3.665088e-02 +Pairwise Node Aware: 1.919495e-03 +Nonblocking Hierarchical: 3.311402e-02 +Nonblocking Node Aware: 1.253405e-03 +Pairwise Multileader, 4 leaders: 9.628525e-03 +Pairwise Locality Aware, 4 leaders: 1.446169e-03 +Pairwise Multileader Locality, 4 leaders: 9.635515e-03 +Nonblocking Multileader, 4 leaders: 7.242997e-03 +Nonblocking Locality Aware, 4 leaders: 1.290109e-03 +Nonblocking Multileader Locality, 4 leaders: 8.121060e-03 +Pairwise Multileader, 10 leaders: 4.520864e-03 +Pairwise Locality Aware, 10 leaders: 1.633897e-03 +Pairwise Multileader Locality, 10 leaders: 4.218792e-03 +Nonblocking Multileader, 10 leaders: 3.171843e-03 +Nonblocking Locality Aware, 10 leaders: 1.405251e-03 +Nonblocking Multileader Locality, 10 leaders: 3.697252e-03 +Pairwise Multileader, 20 leaders: 3.002726e-03 +Pairwise Locality Aware, 20 leaders: 1.405071e-03 +Pairwise Multileader Locality, 20 leaders: 2.565910e-03 +Nonblocking Multileader, 20 leaders: 1.569284e-03 +Nonblocking Locality Aware, 20 leaders: 1.126488e-03 +Nonblocking Multileader Locality, 20 leaders: 2.153053e-03 +Size 256 +PMPI: 4.824998e-03 +Pairwise: 4.730930e-03 +NonBlocking: 1.164421e-01 +Pairwise Hierarchical: 6.810626e-02 +Pairwise Node Aware: 2.828511e-03 +Nonblocking Hierarchical: 6.352618e-02 +Nonblocking Node Aware: 1.899004e-03 +Pairwise Multileader, 4 leaders: 2.119029e-02 +Pairwise Locality Aware, 4 leaders: 2.773169e-03 +Pairwise Multileader Locality, 4 leaders: 1.948768e-02 +Nonblocking Multileader, 4 leaders: 1.507328e-02 +Nonblocking Locality Aware, 4 leaders: 2.024413e-03 +Nonblocking Multileader Locality, 4 leaders: 1.676282e-02 +Pairwise Multileader, 10 leaders: 1.197528e-02 +Pairwise Locality Aware, 10 leaders: 3.638425e-03 +Pairwise Multileader Locality, 10 leaders: 9.079099e-03 +Nonblocking Multileader, 10 leaders: 7.683385e-03 +Nonblocking Locality Aware, 10 leaders: 3.406741e-03 +Nonblocking Multileader Locality, 10 leaders: 8.056723e-03 +Pairwise Multileader, 20 leaders: 5.803189e-03 +Pairwise Locality Aware, 20 leaders: 3.426751e-03 +Pairwise Multileader Locality, 20 leaders: 4.781364e-03 +Nonblocking Multileader, 20 leaders: 3.377333e-03 +Nonblocking Locality Aware, 20 leaders: 3.097523e-03 +Nonblocking Multileader Locality, 20 leaders: 4.043391e-03 +Size 512 +PMPI: 7.152963e-03 +Pairwise: 8.043347e-03 +NonBlocking: 7.106549e-02 +Pairwise Hierarchical: 1.288605e-01 +Pairwise Node Aware: 5.596140e-03 +Nonblocking Hierarchical: 1.230825e-01 +Nonblocking Node Aware: 3.550458e-03 +Pairwise Multileader, 4 leaders: 4.676640e-02 +Pairwise Locality Aware, 4 leaders: 5.724145e-03 +Pairwise Multileader Locality, 4 leaders: 4.897406e-02 +Nonblocking Multileader, 4 leaders: 3.500201e-02 +Nonblocking Locality Aware, 4 leaders: 4.966170e-03 +Nonblocking Multileader Locality, 4 leaders: 4.068434e-02 +Pairwise Multileader, 10 leaders: 2.478362e-02 +Pairwise Locality Aware, 10 leaders: 5.707233e-03 +Pairwise Multileader Locality, 10 leaders: 2.163453e-02 +Nonblocking Multileader, 10 leaders: 1.662676e-02 +Nonblocking Locality Aware, 10 leaders: 5.181482e-03 +Nonblocking Multileader Locality, 10 leaders: 1.901174e-02 +Pairwise Multileader, 20 leaders: 1.093064e-02 +Pairwise Locality Aware, 20 leaders: 6.266782e-03 +Pairwise Multileader Locality, 20 leaders: 1.017050e-02 +Nonblocking Multileader, 20 leaders: 7.218931e-03 +Nonblocking Locality Aware, 20 leaders: 4.587966e-03 +Nonblocking Multileader Locality, 20 leaders: 8.703127e-03 +Size 1024 +PMPI: 1.426345e-02 +Pairwise: 1.205941e-02 +NonBlocking: 6.951189e-02 +Pairwise Hierarchical: 2.367250e-01 +Pairwise Node Aware: 1.234505e-02 +Nonblocking Hierarchical: 2.351009e-01 +Nonblocking Node Aware: 8.640167e-03 +Pairwise Multileader, 4 leaders: 8.442994e-02 +Pairwise Locality Aware, 4 leaders: 1.124018e-02 +Pairwise Multileader Locality, 4 leaders: 8.983660e-02 +Nonblocking Multileader, 4 leaders: 6.459297e-02 +Nonblocking Locality Aware, 4 leaders: 7.392463e-03 +Nonblocking Multileader Locality, 4 leaders: 7.843213e-02 +Pairwise Multileader, 10 leaders: 4.712573e-02 +Pairwise Locality Aware, 10 leaders: 1.351582e-02 +Pairwise Multileader Locality, 10 leaders: 4.676846e-02 +Nonblocking Multileader, 10 leaders: 3.426283e-02 +Nonblocking Locality Aware, 10 leaders: 1.010701e-02 +Nonblocking Multileader Locality, 10 leaders: 4.061707e-02 +Pairwise Multileader, 20 leaders: 3.087490e-02 +Pairwise Locality Aware, 20 leaders: 1.323540e-02 +Pairwise Multileader Locality, 20 leaders: 2.290712e-02 +Nonblocking Multileader, 20 leaders: 1.743766e-02 +Nonblocking Locality Aware, 20 leaders: 1.056490e-02 +Nonblocking Multileader Locality, 20 leaders: 1.983034e-02 +Size 2048 +PMPI: 2.829695e-02 +Pairwise: 2.453477e-02 +NonBlocking: 3.882707e-02 +Pairwise Hierarchical: 4.883312e-01 +Pairwise Node Aware: 2.957765e-02 +Nonblocking Hierarchical: 4.895530e-01 +Nonblocking Node Aware: 2.126099e-02 +Pairwise Multileader, 4 leaders: 1.587148e-01 +Pairwise Locality Aware, 4 leaders: 2.716367e-02 +Pairwise Multileader Locality, 4 leaders: 1.688766e-01 +Nonblocking Multileader, 4 leaders: 1.243425e-01 +Nonblocking Locality Aware, 4 leaders: 1.788663e-02 +Nonblocking Multileader Locality, 4 leaders: 1.533369e-01 +Pairwise Multileader, 10 leaders: 9.244563e-02 +Pairwise Locality Aware, 10 leaders: 2.507082e-02 +Pairwise Multileader Locality, 10 leaders: 9.687116e-02 +Nonblocking Multileader, 10 leaders: 6.903464e-02 +Nonblocking Locality Aware, 10 leaders: 1.806697e-02 +Nonblocking Multileader Locality, 10 leaders: 8.395501e-02 +Pairwise Multileader, 20 leaders: 5.601203e-02 +Pairwise Locality Aware, 20 leaders: 2.619792e-02 +Pairwise Multileader Locality, 20 leaders: 4.872543e-02 +Nonblocking Multileader, 20 leaders: 3.553436e-02 +Nonblocking Locality Aware, 20 leaders: 2.097358e-02 +Nonblocking Multileader Locality, 20 leaders: 4.239577e-02 +Size 4096 +PMPI: 5.709436e-02 +Pairwise: 5.096595e-02 +NonBlocking: 7.066536e-02 +Pairwise Hierarchical: 1.069341e+00 +Pairwise Node Aware: 4.686441e-02 +Nonblocking Hierarchical: 1.047922e+00 +Nonblocking Node Aware: 4.041792e-02 +Pairwise Multileader, 4 leaders: 2.983162e-01 +Pairwise Locality Aware, 4 leaders: 4.887606e-02 +Pairwise Multileader Locality, 4 leaders: 3.233656e-01 +Nonblocking Multileader, 4 leaders: 2.409396e-01 +Nonblocking Locality Aware, 4 leaders: 3.746551e-02 +Nonblocking Multileader Locality, 4 leaders: 3.034269e-01 +Pairwise Multileader, 10 leaders: 1.725978e-01 +Pairwise Locality Aware, 10 leaders: 5.322238e-02 +Pairwise Multileader Locality, 10 leaders: 1.882970e-01 +Nonblocking Multileader, 10 leaders: 1.322218e-01 +Nonblocking Locality Aware, 10 leaders: 3.740200e-02 +Nonblocking Multileader Locality, 10 leaders: 1.657388e-01 +Pairwise Multileader, 20 leaders: 1.118302e-01 +Pairwise Locality Aware, 20 leaders: 5.293383e-02 +Pairwise Multileader Locality, 20 leaders: 1.122136e-01 +Nonblocking Multileader, 20 leaders: 7.836270e-02 +Nonblocking Locality Aware, 20 leaders: 3.880985e-02 +Nonblocking Multileader Locality, 20 leaders: 9.711870e-02 +Size 8192 +PMPI: 9.131465e-02 +Pairwise: 9.313317e-02 +NonBlocking: 7.835080e-02 +Pairwise Hierarchical: 2.176738e+00 +Pairwise Node Aware: 9.282294e-02 +Nonblocking Hierarchical: 2.098595e+00 +Nonblocking Node Aware: 7.908449e-02 +Pairwise Multileader, 4 leaders: 5.803010e-01 +Pairwise Locality Aware, 4 leaders: 1.014790e-01 +Pairwise Multileader Locality, 4 leaders: 6.839794e-01 +Nonblocking Multileader, 4 leaders: 5.364208e-01 +Nonblocking Locality Aware, 4 leaders: 7.739482e-02 +Nonblocking Multileader Locality, 4 leaders: 6.366745e-01 +Pairwise Multileader, 10 leaders: 3.520704e-01 +Pairwise Locality Aware, 10 leaders: 1.023712e-01 +Pairwise Multileader Locality, 10 leaders: 3.935495e-01 +Nonblocking Multileader, 10 leaders: 2.643492e-01 +Nonblocking Locality Aware, 10 leaders: 8.249396e-02 +Nonblocking Multileader Locality, 10 leaders: 3.337388e-01 +Pairwise Multileader, 20 leaders: 2.039744e-01 +Pairwise Locality Aware, 20 leaders: 1.021264e-01 +Pairwise Multileader Locality, 20 leaders: 2.133511e-01 +Nonblocking Multileader, 20 leaders: 1.536117e-01 +Nonblocking Locality Aware, 20 leaders: 7.591939e-02 +Nonblocking Multileader Locality, 20 leaders: 1.842417e-01 +Size 16384 +PMPI: 1.749012e-01 +Pairwise: 1.665259e-01 +NonBlocking: 1.153381e-01 diff --git a/runscripts/dane_results/alltoall_N4.2484420.err b/runscripts/dane_results/alltoall_N4.2484420.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/dane_results/alltoall_N4.2484420.out b/runscripts/dane_results/alltoall_N4.2484420.out new file mode 100644 index 000000000..0ebd21ae4 --- /dev/null +++ b/runscripts/dane_results/alltoall_N4.2484420.out @@ -0,0 +1,783 @@ +Test 1 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 6.946438e-05 +Pairwise: 9.948843e-04 +NonBlocking: 3.025598e-04 +Pairwise Hierarchical: 3.118841e-04 +Pairwise Node Aware: 1.997367e-04 +Nonblocking Hierarchical: 3.046977e-04 +Nonblocking Node Aware: 1.873744e-04 +Pairwise Multileader, 4 leaders: 1.363676e-04 +Pairwise Locality Aware, 4 leaders: 1.209689e-04 +Pairwise Multileader Locality, 4 leaders: 1.182513e-04 +Nonblocking Multileader, 4 leaders: 8.811113e-05 +Nonblocking Locality Aware, 4 leaders: 7.296535e-05 +Nonblocking Multileader Locality, 4 leaders: 1.017345e-04 +Pairwise Multileader, 10 leaders: 1.904007e-04 +Pairwise Locality Aware, 10 leaders: 2.102181e-04 +Pairwise Multileader Locality, 10 leaders: 7.130789e-05 +Nonblocking Multileader, 10 leaders: 6.370118e-05 +Nonblocking Locality Aware, 10 leaders: 1.046075e-04 +Nonblocking Multileader Locality, 10 leaders: 6.951437e-05 +Pairwise Multileader, 20 leaders: 3.759574e-04 +Pairwise Locality Aware, 20 leaders: 3.551266e-04 +Pairwise Multileader Locality, 20 leaders: 8.082962e-05 +Nonblocking Multileader, 20 leaders: 9.890605e-05 +Nonblocking Locality Aware, 20 leaders: 1.196319e-04 +Nonblocking Multileader Locality, 20 leaders: 5.773648e-05 +Size 2 +PMPI: 8.603753e-05 +Pairwise: 8.987875e-04 +NonBlocking: 3.116871e-04 +Pairwise Hierarchical: 6.073445e-04 +Pairwise Node Aware: 2.469773e-04 +Nonblocking Hierarchical: 5.552905e-04 +Nonblocking Node Aware: 1.966488e-04 +Pairwise Multileader, 4 leaders: 1.631388e-04 +Pairwise Locality Aware, 4 leaders: 1.245130e-04 +Pairwise Multileader Locality, 4 leaders: 1.664689e-04 +Nonblocking Multileader, 4 leaders: 1.148694e-04 +Nonblocking Locality Aware, 4 leaders: 6.308642e-05 +Nonblocking Multileader Locality, 4 leaders: 1.318482e-04 +Pairwise Multileader, 10 leaders: 2.100659e-04 +Pairwise Locality Aware, 10 leaders: 2.203746e-04 +Pairwise Multileader Locality, 10 leaders: 8.906444e-05 +Nonblocking Multileader, 10 leaders: 7.676682e-05 +Nonblocking Locality Aware, 10 leaders: 1.291528e-04 +Nonblocking Multileader Locality, 10 leaders: 8.801911e-05 +Pairwise Multileader, 20 leaders: 3.703115e-04 +Pairwise Locality Aware, 20 leaders: 4.768778e-04 +Pairwise Multileader Locality, 20 leaders: 9.459443e-05 +Nonblocking Multileader, 20 leaders: 1.034755e-04 +Nonblocking Locality Aware, 20 leaders: 1.528625e-04 +Nonblocking Multileader Locality, 20 leaders: 6.922722e-05 +Size 4 +PMPI: 1.239149e-04 +Pairwise: 1.786888e-03 +NonBlocking: 7.430214e-02 +Pairwise Hierarchical: 9.471216e-04 +Pairwise Node Aware: 2.761664e-04 +Nonblocking Hierarchical: 8.653438e-04 +Nonblocking Node Aware: 2.081054e-04 +Pairwise Multileader, 4 leaders: 3.017414e-04 +Pairwise Locality Aware, 4 leaders: 1.330115e-04 +Pairwise Multileader Locality, 4 leaders: 4.583136e-04 +Nonblocking Multileader, 4 leaders: 2.147224e-04 +Nonblocking Locality Aware, 4 leaders: 1.556205e-04 +Nonblocking Multileader Locality, 4 leaders: 4.086605e-04 +Pairwise Multileader, 10 leaders: 2.376332e-04 +Pairwise Locality Aware, 10 leaders: 2.415229e-04 +Pairwise Multileader Locality, 10 leaders: 1.263015e-04 +Nonblocking Multileader, 10 leaders: 1.148850e-04 +Nonblocking Locality Aware, 10 leaders: 1.374037e-04 +Nonblocking Multileader Locality, 10 leaders: 1.057741e-04 +Pairwise Multileader, 20 leaders: 3.839602e-04 +Pairwise Locality Aware, 20 leaders: 3.541777e-04 +Pairwise Multileader Locality, 20 leaders: 1.399876e-04 +Nonblocking Multileader, 20 leaders: 1.584098e-04 +Nonblocking Locality Aware, 20 leaders: 2.754941e-04 +Nonblocking Multileader Locality, 20 leaders: 1.080189e-04 +Size 8 +PMPI: 2.217312e-04 +Pairwise: 1.858513e-03 +NonBlocking: 4.238749e-02 +Pairwise Hierarchical: 1.834796e-03 +Pairwise Node Aware: 3.665370e-04 +Nonblocking Hierarchical: 1.671303e-03 +Nonblocking Node Aware: 2.105117e-04 +Pairwise Multileader, 4 leaders: 5.701421e-04 +Pairwise Locality Aware, 4 leaders: 2.387705e-04 +Pairwise Multileader Locality, 4 leaders: 9.333196e-04 +Nonblocking Multileader, 4 leaders: 3.842812e-04 +Nonblocking Locality Aware, 4 leaders: 1.748732e-04 +Nonblocking Multileader Locality, 4 leaders: 8.110897e-04 +Pairwise Multileader, 10 leaders: 3.173345e-04 +Pairwise Locality Aware, 10 leaders: 2.698401e-04 +Pairwise Multileader Locality, 10 leaders: 2.531142e-04 +Nonblocking Multileader, 10 leaders: 1.596123e-04 +Nonblocking Locality Aware, 10 leaders: 1.564679e-04 +Nonblocking Multileader Locality, 10 leaders: 1.920275e-04 +Pairwise Multileader, 20 leaders: 3.925973e-04 +Pairwise Locality Aware, 20 leaders: 4.028931e-04 +Pairwise Multileader Locality, 20 leaders: 1.880262e-04 +Nonblocking Multileader, 20 leaders: 1.472788e-04 +Nonblocking Locality Aware, 20 leaders: 1.772499e-04 +Nonblocking Multileader Locality, 20 leaders: 1.493492e-04 +Size 16 +PMPI: 4.250974e-04 +Pairwise: 2.078448e-03 +NonBlocking: 6.199202e-02 +Pairwise Hierarchical: 3.487726e-03 +Pairwise Node Aware: 4.525816e-04 +Nonblocking Hierarchical: 3.194871e-03 +Nonblocking Node Aware: 2.559557e-04 +Pairwise Multileader, 4 leaders: 9.755942e-04 +Pairwise Locality Aware, 4 leaders: 2.680853e-04 +Pairwise Multileader Locality, 4 leaders: 1.205561e-03 +Nonblocking Multileader, 4 leaders: 7.437045e-04 +Nonblocking Locality Aware, 4 leaders: 2.087098e-04 +Nonblocking Multileader Locality, 4 leaders: 1.023546e-03 +Pairwise Multileader, 10 leaders: 5.513977e-04 +Pairwise Locality Aware, 10 leaders: 3.280394e-04 +Pairwise Multileader Locality, 10 leaders: 6.633482e-04 +Nonblocking Multileader, 10 leaders: 3.381610e-04 +Nonblocking Locality Aware, 10 leaders: 2.401134e-04 +Nonblocking Multileader Locality, 10 leaders: 4.424665e-04 +Pairwise Multileader, 20 leaders: 5.588116e-04 +Pairwise Locality Aware, 20 leaders: 4.249169e-04 +Pairwise Multileader Locality, 20 leaders: 3.168276e-04 +Nonblocking Multileader, 20 leaders: 3.328655e-04 +Nonblocking Locality Aware, 20 leaders: 2.443557e-04 +Nonblocking Multileader Locality, 20 leaders: 2.382974e-04 +Size 32 +PMPI: 7.000538e-04 +Pairwise: 2.528879e-03 +NonBlocking: 8.145026e-02 +Pairwise Hierarchical: 7.483949e-03 +Pairwise Node Aware: 6.062761e-04 +Nonblocking Hierarchical: 6.942538e-03 +Nonblocking Node Aware: 4.233100e-04 +Pairwise Multileader, 4 leaders: 2.865099e-03 +Pairwise Locality Aware, 4 leaders: 4.097918e-04 +Pairwise Multileader Locality, 4 leaders: 2.646943e-03 +Nonblocking Multileader, 4 leaders: 2.233687e-03 +Nonblocking Locality Aware, 4 leaders: 3.854863e-04 +Nonblocking Multileader Locality, 4 leaders: 2.384043e-03 +Pairwise Multileader, 10 leaders: 1.114283e-03 +Pairwise Locality Aware, 10 leaders: 4.192997e-04 +Pairwise Multileader Locality, 10 leaders: 1.101041e-03 +Nonblocking Multileader, 10 leaders: 7.396458e-04 +Nonblocking Locality Aware, 10 leaders: 4.541113e-04 +Nonblocking Multileader Locality, 10 leaders: 9.629481e-04 +Pairwise Multileader, 20 leaders: 7.084874e-04 +Pairwise Locality Aware, 20 leaders: 6.087696e-04 +Pairwise Multileader Locality, 20 leaders: 5.137563e-04 +Nonblocking Multileader, 20 leaders: 4.382002e-04 +Nonblocking Locality Aware, 20 leaders: 4.315775e-04 +Nonblocking Multileader Locality, 20 leaders: 4.115286e-04 +Size 64 +PMPI: 1.383623e-03 +Pairwise: 1.794521e-03 +NonBlocking: 8.699382e-02 +Pairwise Hierarchical: 1.675387e-02 +Pairwise Node Aware: 9.229457e-04 +Nonblocking Hierarchical: 1.477215e-02 +Nonblocking Node Aware: 6.361252e-04 +Pairwise Multileader, 4 leaders: 5.104107e-03 +Pairwise Locality Aware, 4 leaders: 7.496076e-04 +Pairwise Multileader Locality, 4 leaders: 5.105271e-03 +Nonblocking Multileader, 4 leaders: 4.001734e-03 +Nonblocking Locality Aware, 4 leaders: 7.171133e-04 +Nonblocking Multileader Locality, 4 leaders: 4.411158e-03 +Pairwise Multileader, 10 leaders: 2.118195e-03 +Pairwise Locality Aware, 10 leaders: 7.113603e-04 +Pairwise Multileader Locality, 10 leaders: 2.092906e-03 +Nonblocking Multileader, 10 leaders: 1.563381e-03 +Nonblocking Locality Aware, 10 leaders: 6.519859e-04 +Nonblocking Multileader Locality, 10 leaders: 1.854594e-03 +Pairwise Multileader, 20 leaders: 1.345620e-03 +Pairwise Locality Aware, 20 leaders: 8.708649e-04 +Pairwise Multileader Locality, 20 leaders: 1.246267e-03 +Nonblocking Multileader, 20 leaders: 7.735239e-04 +Nonblocking Locality Aware, 20 leaders: 7.578235e-04 +Nonblocking Multileader Locality, 20 leaders: 1.074401e-03 +Size 128 +PMPI: 2.235330e-03 +Pairwise: 2.120692e-03 +NonBlocking: 8.741049e-02 +Pairwise Hierarchical: 3.575447e-02 +Pairwise Node Aware: 1.770558e-03 +Nonblocking Hierarchical: 3.192599e-02 +Nonblocking Node Aware: 1.244303e-03 +Pairwise Multileader, 4 leaders: 9.595071e-03 +Pairwise Locality Aware, 4 leaders: 1.468120e-03 +Pairwise Multileader Locality, 4 leaders: 9.502161e-03 +Nonblocking Multileader, 4 leaders: 7.172765e-03 +Nonblocking Locality Aware, 4 leaders: 1.320554e-03 +Nonblocking Multileader Locality, 4 leaders: 8.239949e-03 +Pairwise Multileader, 10 leaders: 4.326267e-03 +Pairwise Locality Aware, 10 leaders: 1.493001e-03 +Pairwise Multileader Locality, 10 leaders: 4.105445e-03 +Nonblocking Multileader, 10 leaders: 3.173652e-03 +Nonblocking Locality Aware, 10 leaders: 1.458913e-03 +Nonblocking Multileader Locality, 10 leaders: 3.638307e-03 +Pairwise Multileader, 20 leaders: 2.640458e-03 +Pairwise Locality Aware, 20 leaders: 1.286511e-03 +Pairwise Multileader Locality, 20 leaders: 2.497973e-03 +Nonblocking Multileader, 20 leaders: 1.619594e-03 +Nonblocking Locality Aware, 20 leaders: 1.125594e-03 +Nonblocking Multileader Locality, 20 leaders: 2.082767e-03 +Size 256 +PMPI: 4.474826e-03 +Pairwise: 3.737296e-03 +NonBlocking: 8.869824e-02 +Pairwise Hierarchical: 6.754247e-02 +Pairwise Node Aware: 2.530572e-03 +Nonblocking Hierarchical: 6.353691e-02 +Nonblocking Node Aware: 1.796831e-03 +Pairwise Multileader, 4 leaders: 2.150529e-02 +Pairwise Locality Aware, 4 leaders: 2.534347e-03 +Pairwise Multileader Locality, 4 leaders: 1.934682e-02 +Nonblocking Multileader, 4 leaders: 1.507793e-02 +Nonblocking Locality Aware, 4 leaders: 1.904309e-03 +Nonblocking Multileader Locality, 4 leaders: 1.672523e-02 +Pairwise Multileader, 10 leaders: 1.181014e-02 +Pairwise Locality Aware, 10 leaders: 2.908793e-03 +Pairwise Multileader Locality, 10 leaders: 8.911338e-03 +Nonblocking Multileader, 10 leaders: 7.480159e-03 +Nonblocking Locality Aware, 10 leaders: 3.319023e-03 +Nonblocking Multileader Locality, 10 leaders: 7.974252e-03 +Pairwise Multileader, 20 leaders: 5.145576e-03 +Pairwise Locality Aware, 20 leaders: 2.593139e-03 +Pairwise Multileader Locality, 20 leaders: 5.006402e-03 +Nonblocking Multileader, 20 leaders: 3.442169e-03 +Nonblocking Locality Aware, 20 leaders: 2.231019e-03 +Nonblocking Multileader Locality, 20 leaders: 4.173455e-03 +Size 512 +PMPI: 6.342290e-03 +Pairwise: 5.911525e-03 +NonBlocking: 8.325573e-02 +Pairwise Hierarchical: 1.275826e-01 +Pairwise Node Aware: 5.270799e-03 +Nonblocking Hierarchical: 1.248087e-01 +Nonblocking Node Aware: 3.371832e-03 +Pairwise Multileader, 4 leaders: 4.562922e-02 +Pairwise Locality Aware, 4 leaders: 5.061186e-03 +Pairwise Multileader Locality, 4 leaders: 4.800735e-02 +Nonblocking Multileader, 4 leaders: 3.449020e-02 +Nonblocking Locality Aware, 4 leaders: 3.588340e-03 +Nonblocking Multileader Locality, 4 leaders: 4.073374e-02 +Pairwise Multileader, 10 leaders: 2.423562e-02 +Pairwise Locality Aware, 10 leaders: 5.919084e-03 +Pairwise Multileader Locality, 10 leaders: 2.126236e-02 +Nonblocking Multileader, 10 leaders: 1.633669e-02 +Nonblocking Locality Aware, 10 leaders: 5.493310e-03 +Nonblocking Multileader Locality, 10 leaders: 1.864578e-02 +Pairwise Multileader, 20 leaders: 1.093093e-02 +Pairwise Locality Aware, 20 leaders: 6.104921e-03 +Pairwise Multileader Locality, 20 leaders: 1.007559e-02 +Nonblocking Multileader, 20 leaders: 7.568540e-03 +Nonblocking Locality Aware, 20 leaders: 7.536536e-03 +Nonblocking Multileader Locality, 20 leaders: 8.718225e-03 +Test 2 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 7.751021e-05 +Pairwise: 8.408197e-04 +NonBlocking: 3.018726e-04 +Pairwise Hierarchical: 3.129586e-04 +Pairwise Node Aware: 1.987195e-04 +Nonblocking Hierarchical: 2.990450e-04 +Nonblocking Node Aware: 1.875753e-04 +Pairwise Multileader, 4 leaders: 1.412861e-04 +Pairwise Locality Aware, 4 leaders: 1.304816e-04 +Pairwise Multileader Locality, 4 leaders: 1.191487e-04 +Nonblocking Multileader, 4 leaders: 8.715677e-05 +Nonblocking Locality Aware, 4 leaders: 5.707913e-05 +Nonblocking Multileader Locality, 4 leaders: 1.022035e-04 +Pairwise Multileader, 10 leaders: 1.943755e-04 +Pairwise Locality Aware, 10 leaders: 1.916346e-04 +Pairwise Multileader Locality, 10 leaders: 6.960039e-05 +Nonblocking Multileader, 10 leaders: 6.655613e-05 +Nonblocking Locality Aware, 10 leaders: 9.233468e-05 +Nonblocking Multileader Locality, 10 leaders: 5.916822e-05 +Pairwise Multileader, 20 leaders: 3.524363e-04 +Pairwise Locality Aware, 20 leaders: 3.656779e-04 +Pairwise Multileader Locality, 20 leaders: 8.328611e-05 +Nonblocking Multileader, 20 leaders: 9.798720e-05 +Nonblocking Locality Aware, 20 leaders: 9.755919e-05 +Nonblocking Multileader Locality, 20 leaders: 5.553031e-05 +Size 2 +PMPI: 9.092169e-05 +Pairwise: 1.011820e-03 +NonBlocking: 2.812059e-04 +Pairwise Hierarchical: 6.073309e-04 +Pairwise Node Aware: 2.443763e-04 +Nonblocking Hierarchical: 5.379482e-04 +Nonblocking Node Aware: 1.965803e-04 +Pairwise Multileader, 4 leaders: 1.755375e-04 +Pairwise Locality Aware, 4 leaders: 1.481973e-04 +Pairwise Multileader Locality, 4 leaders: 1.662933e-04 +Nonblocking Multileader, 4 leaders: 1.113800e-04 +Nonblocking Locality Aware, 4 leaders: 7.977554e-05 +Nonblocking Multileader Locality, 4 leaders: 1.398303e-04 +Pairwise Multileader, 10 leaders: 1.935953e-04 +Pairwise Locality Aware, 10 leaders: 2.688875e-04 +Pairwise Multileader Locality, 10 leaders: 9.274700e-05 +Nonblocking Multileader, 10 leaders: 7.765241e-05 +Nonblocking Locality Aware, 10 leaders: 9.919573e-05 +Nonblocking Multileader Locality, 10 leaders: 8.000301e-05 +Pairwise Multileader, 20 leaders: 4.401860e-04 +Pairwise Locality Aware, 20 leaders: 3.652299e-04 +Pairwise Multileader Locality, 20 leaders: 1.038219e-04 +Nonblocking Multileader, 20 leaders: 1.020800e-04 +Nonblocking Locality Aware, 20 leaders: 1.141819e-04 +Nonblocking Multileader Locality, 20 leaders: 7.481780e-05 +Size 4 +PMPI: 1.273579e-04 +Pairwise: 1.682748e-03 +NonBlocking: 4.971871e-02 +Pairwise Hierarchical: 9.699660e-04 +Pairwise Node Aware: 2.640470e-04 +Nonblocking Hierarchical: 8.771828e-04 +Nonblocking Node Aware: 1.955395e-04 +Pairwise Multileader, 4 leaders: 3.089314e-04 +Pairwise Locality Aware, 4 leaders: 1.560751e-04 +Pairwise Multileader Locality, 4 leaders: 3.271189e-04 +Nonblocking Multileader, 4 leaders: 2.199365e-04 +Nonblocking Locality Aware, 4 leaders: 1.090183e-04 +Nonblocking Multileader Locality, 4 leaders: 2.738775e-04 +Pairwise Multileader, 10 leaders: 2.123673e-04 +Pairwise Locality Aware, 10 leaders: 2.727920e-04 +Pairwise Multileader Locality, 10 leaders: 1.256941e-04 +Nonblocking Multileader, 10 leaders: 1.114449e-04 +Nonblocking Locality Aware, 10 leaders: 1.544456e-04 +Nonblocking Multileader Locality, 10 leaders: 1.050684e-04 +Pairwise Multileader, 20 leaders: 3.760358e-04 +Pairwise Locality Aware, 20 leaders: 4.024503e-04 +Pairwise Multileader Locality, 20 leaders: 1.223921e-04 +Nonblocking Multileader, 20 leaders: 1.181594e-04 +Nonblocking Locality Aware, 20 leaders: 2.499234e-04 +Nonblocking Multileader Locality, 20 leaders: 9.102978e-05 +Size 8 +PMPI: 2.089952e-04 +Pairwise: 2.075054e-03 +NonBlocking: 6.197029e-02 +Pairwise Hierarchical: 1.811229e-03 +Pairwise Node Aware: 3.652807e-04 +Nonblocking Hierarchical: 1.642158e-03 +Nonblocking Node Aware: 2.093884e-04 +Pairwise Multileader, 4 leaders: 5.275196e-04 +Pairwise Locality Aware, 4 leaders: 1.751762e-04 +Pairwise Multileader Locality, 4 leaders: 6.879276e-04 +Nonblocking Multileader, 4 leaders: 3.852104e-04 +Nonblocking Locality Aware, 4 leaders: 1.677160e-04 +Nonblocking Multileader Locality, 4 leaders: 5.616632e-04 +Pairwise Multileader, 10 leaders: 3.166743e-04 +Pairwise Locality Aware, 10 leaders: 2.411912e-04 +Pairwise Multileader Locality, 10 leaders: 2.481982e-04 +Nonblocking Multileader, 10 leaders: 1.727777e-04 +Nonblocking Locality Aware, 10 leaders: 1.700344e-04 +Nonblocking Multileader Locality, 10 leaders: 2.009964e-04 +Pairwise Multileader, 20 leaders: 4.258797e-04 +Pairwise Locality Aware, 20 leaders: 4.624699e-04 +Pairwise Multileader Locality, 20 leaders: 1.830183e-04 +Nonblocking Multileader, 20 leaders: 1.638531e-04 +Nonblocking Locality Aware, 20 leaders: 1.910338e-04 +Nonblocking Multileader Locality, 20 leaders: 1.230935e-04 +Size 16 +PMPI: 3.959919e-04 +Pairwise: 1.850704e-03 +NonBlocking: 6.086110e-02 +Pairwise Hierarchical: 3.333951e-03 +Pairwise Node Aware: 5.105480e-04 +Nonblocking Hierarchical: 3.191698e-03 +Nonblocking Node Aware: 2.577914e-04 +Pairwise Multileader, 4 leaders: 9.556107e-04 +Pairwise Locality Aware, 4 leaders: 2.243242e-04 +Pairwise Multileader Locality, 4 leaders: 1.253786e-03 +Nonblocking Multileader, 4 leaders: 7.642980e-04 +Nonblocking Locality Aware, 4 leaders: 1.810681e-04 +Nonblocking Multileader Locality, 4 leaders: 1.024745e-03 +Pairwise Multileader, 10 leaders: 5.693912e-04 +Pairwise Locality Aware, 10 leaders: 3.077442e-04 +Pairwise Multileader Locality, 10 leaders: 4.545753e-04 +Nonblocking Multileader, 10 leaders: 3.511657e-04 +Nonblocking Locality Aware, 10 leaders: 3.073061e-04 +Nonblocking Multileader Locality, 10 leaders: 4.046379e-04 +Pairwise Multileader, 20 leaders: 5.915025e-04 +Pairwise Locality Aware, 20 leaders: 4.035002e-04 +Pairwise Multileader Locality, 20 leaders: 3.184744e-04 +Nonblocking Multileader, 20 leaders: 2.623705e-04 +Nonblocking Locality Aware, 20 leaders: 2.805480e-04 +Nonblocking Multileader Locality, 20 leaders: 3.055341e-04 +Size 32 +PMPI: 6.803239e-04 +Pairwise: 2.036370e-03 +NonBlocking: 7.531200e-02 +Pairwise Hierarchical: 7.510413e-03 +Pairwise Node Aware: 6.447624e-04 +Nonblocking Hierarchical: 6.933380e-03 +Nonblocking Node Aware: 4.356083e-04 +Pairwise Multileader, 4 leaders: 2.860828e-03 +Pairwise Locality Aware, 4 leaders: 4.200799e-04 +Pairwise Multileader Locality, 4 leaders: 2.683846e-03 +Nonblocking Multileader, 4 leaders: 2.249189e-03 +Nonblocking Locality Aware, 4 leaders: 4.048795e-04 +Nonblocking Multileader Locality, 4 leaders: 2.349367e-03 +Pairwise Multileader, 10 leaders: 1.108187e-03 +Pairwise Locality Aware, 10 leaders: 3.982048e-04 +Pairwise Multileader Locality, 10 leaders: 1.097138e-03 +Nonblocking Multileader, 10 leaders: 7.280926e-04 +Nonblocking Locality Aware, 10 leaders: 3.789617e-04 +Nonblocking Multileader Locality, 10 leaders: 9.547494e-04 +Pairwise Multileader, 20 leaders: 7.525123e-04 +Pairwise Locality Aware, 20 leaders: 5.596352e-04 +Pairwise Multileader Locality, 20 leaders: 5.836645e-04 +Nonblocking Multileader, 20 leaders: 3.790707e-04 +Nonblocking Locality Aware, 20 leaders: 3.216816e-04 +Nonblocking Multileader Locality, 20 leaders: 4.925250e-04 +Size 64 +PMPI: 1.382524e-03 +Pairwise: 1.949132e-03 +NonBlocking: 6.931376e-02 +Pairwise Hierarchical: 1.734531e-02 +Pairwise Node Aware: 8.948768e-04 +Nonblocking Hierarchical: 1.559835e-02 +Nonblocking Node Aware: 5.920354e-04 +Pairwise Multileader, 4 leaders: 5.009987e-03 +Pairwise Locality Aware, 4 leaders: 7.205653e-04 +Pairwise Multileader Locality, 4 leaders: 5.046850e-03 +Nonblocking Multileader, 4 leaders: 4.019816e-03 +Nonblocking Locality Aware, 4 leaders: 6.368821e-04 +Nonblocking Multileader Locality, 4 leaders: 4.378674e-03 +Pairwise Multileader, 10 leaders: 2.114770e-03 +Pairwise Locality Aware, 10 leaders: 7.641785e-04 +Pairwise Multileader Locality, 10 leaders: 2.133170e-03 +Nonblocking Multileader, 10 leaders: 1.584154e-03 +Nonblocking Locality Aware, 10 leaders: 7.836849e-04 +Nonblocking Multileader Locality, 10 leaders: 1.865008e-03 +Pairwise Multileader, 20 leaders: 1.499454e-03 +Pairwise Locality Aware, 20 leaders: 9.442584e-04 +Pairwise Multileader Locality, 20 leaders: 1.222938e-03 +Nonblocking Multileader, 20 leaders: 9.009759e-04 +Nonblocking Locality Aware, 20 leaders: 7.679591e-04 +Nonblocking Multileader Locality, 20 leaders: 1.080627e-03 +Size 128 +PMPI: 2.237430e-03 +Pairwise: 2.402313e-03 +NonBlocking: 8.720383e-02 +Pairwise Hierarchical: 3.611039e-02 +Pairwise Node Aware: 1.961983e-03 +Nonblocking Hierarchical: 3.264767e-02 +Nonblocking Node Aware: 1.280950e-03 +Pairwise Multileader, 4 leaders: 9.726286e-03 +Pairwise Locality Aware, 4 leaders: 1.551139e-03 +Pairwise Multileader Locality, 4 leaders: 9.204552e-03 +Nonblocking Multileader, 4 leaders: 7.262187e-03 +Nonblocking Locality Aware, 4 leaders: 1.448882e-03 +Nonblocking Multileader Locality, 4 leaders: 8.075534e-03 +Pairwise Multileader, 10 leaders: 4.224226e-03 +Pairwise Locality Aware, 10 leaders: 1.457883e-03 +Pairwise Multileader Locality, 10 leaders: 4.107195e-03 +Nonblocking Multileader, 10 leaders: 3.130162e-03 +Nonblocking Locality Aware, 10 leaders: 1.227308e-03 +Nonblocking Multileader Locality, 10 leaders: 3.633528e-03 +Pairwise Multileader, 20 leaders: 2.704806e-03 +Pairwise Locality Aware, 20 leaders: 1.349977e-03 +Pairwise Multileader Locality, 20 leaders: 2.508850e-03 +Nonblocking Multileader, 20 leaders: 1.607671e-03 +Nonblocking Locality Aware, 20 leaders: 1.023799e-03 +Nonblocking Multileader Locality, 20 leaders: 2.126955e-03 +Size 256 +PMPI: 4.446928e-03 +Pairwise: 3.337595e-03 +NonBlocking: 8.847094e-02 +Pairwise Hierarchical: 6.553152e-02 +Pairwise Node Aware: 2.484355e-03 +Nonblocking Hierarchical: 6.283175e-02 +Nonblocking Node Aware: 1.776417e-03 +Pairwise Multileader, 4 leaders: 2.120419e-02 +Pairwise Locality Aware, 4 leaders: 2.536342e-03 +Pairwise Multileader Locality, 4 leaders: 1.932848e-02 +Nonblocking Multileader, 4 leaders: 1.508955e-02 +Nonblocking Locality Aware, 4 leaders: 1.605776e-03 +Nonblocking Multileader Locality, 4 leaders: 1.670210e-02 +Pairwise Multileader, 10 leaders: 1.187063e-02 +Pairwise Locality Aware, 10 leaders: 2.842739e-03 +Pairwise Multileader Locality, 10 leaders: 8.997115e-03 +Nonblocking Multileader, 10 leaders: 7.502523e-03 +Nonblocking Locality Aware, 10 leaders: 3.115774e-03 +Nonblocking Multileader Locality, 10 leaders: 7.980750e-03 +Pairwise Multileader, 20 leaders: 5.398884e-03 +Pairwise Locality Aware, 20 leaders: 3.323080e-03 +Pairwise Multileader Locality, 20 leaders: 4.692891e-03 +Nonblocking Multileader, 20 leaders: 3.444381e-03 +Nonblocking Locality Aware, 20 leaders: 3.377497e-03 +Nonblocking Multileader Locality, 20 leaders: 4.041641e-03 +Size 512 +PMPI: 7.510091e-03 +Pairwise: 6.440778e-03 +NonBlocking: 8.472218e-02 +Pairwise Hierarchical: 1.248920e-01 +Pairwise Node Aware: 5.268901e-03 +Nonblocking Hierarchical: 1.258050e-01 +Nonblocking Node Aware: 4.350792e-03 +Pairwise Multileader, 4 leaders: 4.654837e-02 +Pairwise Locality Aware, 4 leaders: 6.223084e-03 +Pairwise Multileader Locality, 4 leaders: 4.819646e-02 +Nonblocking Multileader, 4 leaders: 3.476598e-02 +Nonblocking Locality Aware, 4 leaders: 5.709092e-03 +Nonblocking Multileader Locality, 4 leaders: 4.009854e-02 +Pairwise Multileader, 10 leaders: 2.422785e-02 +Pairwise Locality Aware, 10 leaders: 6.162372e-03 +Pairwise Multileader Locality, 10 leaders: 2.142769e-02 +Nonblocking Multileader, 10 leaders: 1.637677e-02 +Nonblocking Locality Aware, 10 leaders: 5.650524e-03 +Nonblocking Multileader Locality, 10 leaders: 1.875039e-02 +Pairwise Multileader, 20 leaders: 1.067247e-02 +Pairwise Locality Aware, 20 leaders: 5.046145e-03 +Pairwise Multileader Locality, 20 leaders: 1.022453e-02 +Nonblocking Multileader, 20 leaders: 7.233662e-03 +Nonblocking Locality Aware, 20 leaders: 4.736229e-03 +Nonblocking Multileader Locality, 20 leaders: 8.755011e-03 +Test 3 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 7.069666e-05 +Pairwise: 9.204621e-04 +NonBlocking: 3.076007e-04 +Pairwise Hierarchical: 3.122811e-04 +Pairwise Node Aware: 1.959077e-04 +Nonblocking Hierarchical: 3.032393e-04 +Nonblocking Node Aware: 1.870291e-04 +Pairwise Multileader, 4 leaders: 1.367858e-04 +Pairwise Locality Aware, 4 leaders: 1.215656e-04 +Pairwise Multileader Locality, 4 leaders: 1.158026e-04 +Nonblocking Multileader, 4 leaders: 8.678890e-05 +Nonblocking Locality Aware, 4 leaders: 6.949508e-05 +Nonblocking Multileader Locality, 4 leaders: 1.002382e-04 +Pairwise Multileader, 10 leaders: 1.870273e-04 +Pairwise Locality Aware, 10 leaders: 2.484976e-04 +Pairwise Multileader Locality, 10 leaders: 8.144409e-05 +Nonblocking Multileader, 10 leaders: 6.427762e-05 +Nonblocking Locality Aware, 10 leaders: 9.289089e-05 +Nonblocking Multileader Locality, 10 leaders: 6.013672e-05 +Pairwise Multileader, 20 leaders: 4.224600e-04 +Pairwise Locality Aware, 20 leaders: 3.466702e-04 +Pairwise Multileader Locality, 20 leaders: 8.103154e-05 +Nonblocking Multileader, 20 leaders: 9.729827e-05 +Nonblocking Locality Aware, 20 leaders: 1.071711e-04 +Nonblocking Multileader Locality, 20 leaders: 5.720091e-05 +Size 2 +PMPI: 8.190338e-05 +Pairwise: 8.403569e-04 +NonBlocking: 2.973875e-04 +Pairwise Hierarchical: 6.041450e-04 +Pairwise Node Aware: 2.433092e-04 +Nonblocking Hierarchical: 5.451523e-04 +Nonblocking Node Aware: 1.967260e-04 +Pairwise Multileader, 4 leaders: 1.602308e-04 +Pairwise Locality Aware, 4 leaders: 1.192302e-04 +Pairwise Multileader Locality, 4 leaders: 1.895360e-04 +Nonblocking Multileader, 4 leaders: 1.090987e-04 +Nonblocking Locality Aware, 4 leaders: 6.548940e-05 +Nonblocking Multileader Locality, 4 leaders: 1.871567e-04 +Pairwise Multileader, 10 leaders: 2.035346e-04 +Pairwise Locality Aware, 10 leaders: 2.169812e-04 +Pairwise Multileader Locality, 10 leaders: 8.643994e-05 +Nonblocking Multileader, 10 leaders: 7.436677e-05 +Nonblocking Locality Aware, 10 leaders: 9.733679e-05 +Nonblocking Multileader Locality, 10 leaders: 7.394343e-05 +Pairwise Multileader, 20 leaders: 3.411475e-04 +Pairwise Locality Aware, 20 leaders: 3.809322e-04 +Pairwise Multileader Locality, 20 leaders: 9.323847e-05 +Nonblocking Multileader, 20 leaders: 1.013008e-04 +Nonblocking Locality Aware, 20 leaders: 9.854438e-05 +Nonblocking Multileader Locality, 20 leaders: 6.897046e-05 +Size 4 +PMPI: 1.172476e-04 +Pairwise: 1.687730e-03 +NonBlocking: 2.906917e-02 +Pairwise Hierarchical: 9.635229e-04 +Pairwise Node Aware: 2.601778e-04 +Nonblocking Hierarchical: 8.608202e-04 +Nonblocking Node Aware: 1.939552e-04 +Pairwise Multileader, 4 leaders: 3.045063e-04 +Pairwise Locality Aware, 4 leaders: 1.284324e-04 +Pairwise Multileader Locality, 4 leaders: 3.083485e-04 +Nonblocking Multileader, 4 leaders: 2.127709e-04 +Nonblocking Locality Aware, 4 leaders: 1.004564e-04 +Nonblocking Multileader Locality, 4 leaders: 2.604409e-04 +Pairwise Multileader, 10 leaders: 2.130620e-04 +Pairwise Locality Aware, 10 leaders: 2.317589e-04 +Pairwise Multileader Locality, 10 leaders: 1.295155e-04 +Nonblocking Multileader, 10 leaders: 1.033109e-04 +Nonblocking Locality Aware, 10 leaders: 1.238658e-04 +Nonblocking Multileader Locality, 10 leaders: 1.112300e-04 +Pairwise Multileader, 20 leaders: 3.588820e-04 +Pairwise Locality Aware, 20 leaders: 3.467901e-04 +Pairwise Multileader Locality, 20 leaders: 1.349202e-04 +Nonblocking Multileader, 20 leaders: 1.192097e-04 +Nonblocking Locality Aware, 20 leaders: 2.087633e-04 +Nonblocking Multileader Locality, 20 leaders: 1.008617e-04 +Size 8 +PMPI: 2.147496e-04 +Pairwise: 1.733976e-03 +NonBlocking: 2.514590e-02 +Pairwise Hierarchical: 1.771169e-03 +Pairwise Node Aware: 3.675223e-04 +Nonblocking Hierarchical: 1.593914e-03 +Nonblocking Node Aware: 1.998737e-04 +Pairwise Multileader, 4 leaders: 5.129436e-04 +Pairwise Locality Aware, 4 leaders: 1.731809e-04 +Pairwise Multileader Locality, 4 leaders: 6.498164e-04 +Nonblocking Multileader, 4 leaders: 3.866111e-04 +Nonblocking Locality Aware, 4 leaders: 2.617581e-04 +Nonblocking Multileader Locality, 4 leaders: 5.436502e-04 +Pairwise Multileader, 10 leaders: 3.318857e-04 +Pairwise Locality Aware, 10 leaders: 2.305519e-04 +Pairwise Multileader Locality, 10 leaders: 2.402215e-04 +Nonblocking Multileader, 10 leaders: 1.636871e-04 +Nonblocking Locality Aware, 10 leaders: 1.362842e-04 +Nonblocking Multileader Locality, 10 leaders: 1.941116e-04 +Pairwise Multileader, 20 leaders: 4.789246e-04 +Pairwise Locality Aware, 20 leaders: 3.670801e-04 +Pairwise Multileader Locality, 20 leaders: 2.532556e-04 +Nonblocking Multileader, 20 leaders: 1.529461e-04 +Nonblocking Locality Aware, 20 leaders: 2.043580e-04 +Nonblocking Multileader Locality, 20 leaders: 1.546805e-04 +Size 16 +PMPI: 3.868102e-04 +Pairwise: 2.187049e-03 +NonBlocking: 2.515311e-02 +Pairwise Hierarchical: 3.415789e-03 +Pairwise Node Aware: 4.480564e-04 +Nonblocking Hierarchical: 3.207767e-03 +Nonblocking Node Aware: 2.690920e-04 +Pairwise Multileader, 4 leaders: 9.207079e-04 +Pairwise Locality Aware, 4 leaders: 2.361479e-04 +Pairwise Multileader Locality, 4 leaders: 1.217566e-03 +Nonblocking Multileader, 4 leaders: 7.465440e-04 +Nonblocking Locality Aware, 4 leaders: 1.655742e-04 +Nonblocking Multileader Locality, 4 leaders: 1.016324e-03 +Pairwise Multileader, 10 leaders: 6.067581e-04 +Pairwise Locality Aware, 10 leaders: 2.896867e-04 +Pairwise Multileader Locality, 10 leaders: 4.492292e-04 +Nonblocking Multileader, 10 leaders: 3.453913e-04 +Nonblocking Locality Aware, 10 leaders: 2.447543e-04 +Nonblocking Multileader Locality, 10 leaders: 4.145610e-04 +Pairwise Multileader, 20 leaders: 5.212073e-04 +Pairwise Locality Aware, 20 leaders: 4.811050e-04 +Pairwise Multileader Locality, 20 leaders: 2.924721e-04 +Nonblocking Multileader, 20 leaders: 2.514646e-04 +Nonblocking Locality Aware, 20 leaders: 2.136784e-04 +Nonblocking Multileader Locality, 20 leaders: 2.059045e-04 +Size 32 +PMPI: 6.556166e-04 +Pairwise: 1.886964e-03 +NonBlocking: 8.673015e-02 +Pairwise Hierarchical: 7.551760e-03 +Pairwise Node Aware: 6.557429e-04 +Nonblocking Hierarchical: 6.919206e-03 +Nonblocking Node Aware: 4.726057e-04 +Pairwise Multileader, 4 leaders: 2.890550e-03 +Pairwise Locality Aware, 4 leaders: 3.874080e-04 +Pairwise Multileader Locality, 4 leaders: 2.606603e-03 +Nonblocking Multileader, 4 leaders: 2.217463e-03 +Nonblocking Locality Aware, 4 leaders: 2.908189e-04 +Nonblocking Multileader Locality, 4 leaders: 2.334941e-03 +Pairwise Multileader, 10 leaders: 1.043766e-03 +Pairwise Locality Aware, 10 leaders: 4.624089e-04 +Pairwise Multileader Locality, 10 leaders: 1.133140e-03 +Nonblocking Multileader, 10 leaders: 7.189869e-04 +Nonblocking Locality Aware, 10 leaders: 3.942792e-04 +Nonblocking Multileader Locality, 10 leaders: 9.486563e-04 +Pairwise Multileader, 20 leaders: 7.564861e-04 +Pairwise Locality Aware, 20 leaders: 5.158516e-04 +Pairwise Multileader Locality, 20 leaders: 5.560186e-04 +Nonblocking Multileader, 20 leaders: 4.445817e-04 +Nonblocking Locality Aware, 20 leaders: 4.210273e-04 +Nonblocking Multileader Locality, 20 leaders: 4.462217e-04 +Size 64 +PMPI: 1.476809e-03 +Pairwise: 1.906458e-03 +NonBlocking: 6.239346e-02 +Pairwise Hierarchical: 1.728805e-02 +Pairwise Node Aware: 9.211664e-04 +Nonblocking Hierarchical: 1.532525e-02 +Nonblocking Node Aware: 6.825195e-04 +Pairwise Multileader, 4 leaders: 5.172826e-03 +Pairwise Locality Aware, 4 leaders: 8.610276e-04 +Pairwise Multileader Locality, 4 leaders: 4.975802e-03 +Nonblocking Multileader, 4 leaders: 4.132606e-03 +Nonblocking Locality Aware, 4 leaders: 8.661389e-04 +Nonblocking Multileader Locality, 4 leaders: 4.368834e-03 +Pairwise Multileader, 10 leaders: 2.221411e-03 +Pairwise Locality Aware, 10 leaders: 9.831115e-04 +Pairwise Multileader Locality, 10 leaders: 2.117397e-03 +Nonblocking Multileader, 10 leaders: 1.588562e-03 +Nonblocking Locality Aware, 10 leaders: 7.893691e-04 +Nonblocking Multileader Locality, 10 leaders: 1.882212e-03 +Pairwise Multileader, 20 leaders: 1.508764e-03 +Pairwise Locality Aware, 20 leaders: 8.477131e-04 +Pairwise Multileader Locality, 20 leaders: 1.223758e-03 +Nonblocking Multileader, 20 leaders: 7.782358e-04 +Nonblocking Locality Aware, 20 leaders: 5.323570e-04 +Nonblocking Multileader Locality, 20 leaders: 1.057174e-03 +Size 128 +PMPI: 2.460799e-03 +Pairwise: 2.425296e-03 +NonBlocking: 9.274595e-02 +Pairwise Hierarchical: 3.600253e-02 +Pairwise Node Aware: 1.723068e-03 +Nonblocking Hierarchical: 3.240142e-02 +Nonblocking Node Aware: 1.228623e-03 +Pairwise Multileader, 4 leaders: 9.553669e-03 +Pairwise Locality Aware, 4 leaders: 1.378664e-03 +Pairwise Multileader Locality, 4 leaders: 9.300723e-03 +Nonblocking Multileader, 4 leaders: 7.243433e-03 +Nonblocking Locality Aware, 4 leaders: 1.167463e-03 +Nonblocking Multileader Locality, 4 leaders: 8.213132e-03 +Pairwise Multileader, 10 leaders: 4.333383e-03 +Pairwise Locality Aware, 10 leaders: 1.448245e-03 +Pairwise Multileader Locality, 10 leaders: 4.130160e-03 +Nonblocking Multileader, 10 leaders: 3.190951e-03 +Nonblocking Locality Aware, 10 leaders: 1.421797e-03 +Nonblocking Multileader Locality, 10 leaders: 3.673453e-03 +Pairwise Multileader, 20 leaders: 2.639852e-03 +Pairwise Locality Aware, 20 leaders: 1.768436e-03 +Pairwise Multileader Locality, 20 leaders: 2.532304e-03 +Nonblocking Multileader, 20 leaders: 1.700237e-03 +Nonblocking Locality Aware, 20 leaders: 1.635233e-03 +Nonblocking Multileader Locality, 20 leaders: 2.072210e-03 +Size 256 +PMPI: 4.502485e-03 +Pairwise: 4.658673e-03 +NonBlocking: 9.359530e-02 +Pairwise Hierarchical: 6.685257e-02 +Pairwise Node Aware: 2.600691e-03 +Nonblocking Hierarchical: 6.432673e-02 +Nonblocking Node Aware: 1.825857e-03 +Pairwise Multileader, 4 leaders: 2.113568e-02 +Pairwise Locality Aware, 4 leaders: 3.012736e-03 +Pairwise Multileader Locality, 4 leaders: 1.946333e-02 +Nonblocking Multileader, 4 leaders: 1.515081e-02 +Nonblocking Locality Aware, 4 leaders: 2.441414e-03 +Nonblocking Multileader Locality, 4 leaders: 1.682175e-02 +Pairwise Multileader, 10 leaders: 1.179882e-02 +Pairwise Locality Aware, 10 leaders: 2.787618e-03 +Pairwise Multileader Locality, 10 leaders: 9.089190e-03 +Nonblocking Multileader, 10 leaders: 7.532448e-03 +Nonblocking Locality Aware, 10 leaders: 3.140672e-03 +Nonblocking Multileader Locality, 10 leaders: 7.990203e-03 +Pairwise Multileader, 20 leaders: 5.325465e-03 +Pairwise Locality Aware, 20 leaders: 2.574698e-03 +Pairwise Multileader Locality, 20 leaders: 4.707074e-03 +Nonblocking Multileader, 20 leaders: 3.318498e-03 +Nonblocking Locality Aware, 20 leaders: 2.275925e-03 +Nonblocking Multileader Locality, 20 leaders: 4.054860e-03 +Size 512 +PMPI: 6.620423e-03 +Pairwise: 6.941806e-03 +NonBlocking: 9.664037e-02 +Pairwise Hierarchical: 1.273003e-01 +Pairwise Node Aware: 5.126096e-03 +Nonblocking Hierarchical: 1.250724e-01 +Nonblocking Node Aware: 3.478797e-03 +Pairwise Multileader, 4 leaders: 4.629503e-02 +Pairwise Locality Aware, 4 leaders: 5.314882e-03 +Pairwise Multileader Locality, 4 leaders: 4.852102e-02 +Nonblocking Multileader, 4 leaders: 3.404086e-02 +Nonblocking Locality Aware, 4 leaders: 3.605030e-03 +Nonblocking Multileader Locality, 4 leaders: 3.997288e-02 +Pairwise Multileader, 10 leaders: 2.432459e-02 +Pairwise Locality Aware, 10 leaders: 5.561855e-03 +Pairwise Multileader Locality, 10 leaders: 2.139577e-02 +Nonblocking Multileader, 10 leaders: 1.645548e-02 +Nonblocking Locality Aware, 10 leaders: 5.170107e-03 +Nonblocking Multileader Locality, 10 leaders: 1.872522e-02 +Pairwise Multileader, 20 leaders: 1.124306e-02 +Pairwise Locality Aware, 20 leaders: 6.530037e-03 +Pairwise Multileader Locality, 20 leaders: 1.004885e-02 +Nonblocking Multileader, 20 leaders: 7.565200e-03 +Nonblocking Locality Aware, 20 leaders: 7.564948e-03 +Nonblocking Multileader Locality, 20 leaders: 8.835839e-03 diff --git a/runscripts/dane_results/alltoall_N4.2484421.err b/runscripts/dane_results/alltoall_N4.2484421.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/dane_results/alltoall_N4.2484421.out b/runscripts/dane_results/alltoall_N4.2484421.out new file mode 100644 index 000000000..06fedc549 --- /dev/null +++ b/runscripts/dane_results/alltoall_N4.2484421.out @@ -0,0 +1,783 @@ +Test 1 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 8.536419e-05 +Pairwise: 9.623652e-04 +NonBlocking: 3.521847e-04 +Pairwise Hierarchical: 3.853303e-04 +Pairwise Node Aware: 2.397758e-04 +Nonblocking Hierarchical: 3.239074e-04 +Nonblocking Node Aware: 1.961892e-04 +Pairwise Multileader, 4 leaders: 1.706321e-04 +Pairwise Locality Aware, 4 leaders: 1.482124e-04 +Pairwise Multileader Locality, 4 leaders: 1.256653e-04 +Nonblocking Multileader, 4 leaders: 9.253972e-05 +Nonblocking Locality Aware, 4 leaders: 7.546179e-05 +Nonblocking Multileader Locality, 4 leaders: 1.069272e-04 +Pairwise Multileader, 10 leaders: 2.225326e-04 +Pairwise Locality Aware, 10 leaders: 2.654353e-04 +Pairwise Multileader Locality, 10 leaders: 8.283383e-05 +Nonblocking Multileader, 10 leaders: 7.834482e-05 +Nonblocking Locality Aware, 10 leaders: 1.144207e-04 +Nonblocking Multileader Locality, 10 leaders: 6.963450e-05 +Pairwise Multileader, 20 leaders: 5.258139e-04 +Pairwise Locality Aware, 20 leaders: 5.272373e-04 +Pairwise Multileader Locality, 20 leaders: 8.813463e-05 +Nonblocking Multileader, 20 leaders: 1.106491e-04 +Nonblocking Locality Aware, 20 leaders: 1.343695e-04 +Nonblocking Multileader Locality, 20 leaders: 6.256288e-05 +Size 2 +PMPI: 9.735644e-05 +Pairwise: 1.151690e-03 +NonBlocking: 3.835903e-04 +Pairwise Hierarchical: 6.370240e-04 +Pairwise Node Aware: 2.752468e-04 +Nonblocking Hierarchical: 5.794755e-04 +Nonblocking Node Aware: 2.082293e-04 +Pairwise Multileader, 4 leaders: 2.082851e-04 +Pairwise Locality Aware, 4 leaders: 1.527547e-04 +Pairwise Multileader Locality, 4 leaders: 1.627297e-04 +Nonblocking Multileader, 4 leaders: 1.218906e-04 +Nonblocking Locality Aware, 4 leaders: 1.125814e-04 +Nonblocking Multileader Locality, 4 leaders: 1.395831e-04 +Pairwise Multileader, 10 leaders: 2.693545e-04 +Pairwise Locality Aware, 10 leaders: 3.403822e-04 +Pairwise Multileader Locality, 10 leaders: 1.516685e-04 +Nonblocking Multileader, 10 leaders: 9.955433e-05 +Nonblocking Locality Aware, 10 leaders: 1.896517e-04 +Nonblocking Multileader Locality, 10 leaders: 8.341905e-05 +Pairwise Multileader, 20 leaders: 4.396988e-04 +Pairwise Locality Aware, 20 leaders: 4.942885e-04 +Pairwise Multileader Locality, 20 leaders: 1.033683e-04 +Nonblocking Multileader, 20 leaders: 1.182931e-04 +Nonblocking Locality Aware, 20 leaders: 1.614223e-04 +Nonblocking Multileader Locality, 20 leaders: 7.295565e-05 +Size 4 +PMPI: 1.386981e-04 +Pairwise: 2.071858e-03 +NonBlocking: 6.406679e-02 +Pairwise Hierarchical: 9.667224e-04 +Pairwise Node Aware: 2.966703e-04 +Nonblocking Hierarchical: 8.498178e-04 +Nonblocking Node Aware: 2.010158e-04 +Pairwise Multileader, 4 leaders: 3.332884e-04 +Pairwise Locality Aware, 4 leaders: 1.654377e-04 +Pairwise Multileader Locality, 4 leaders: 3.256532e-04 +Nonblocking Multileader, 4 leaders: 2.224645e-04 +Nonblocking Locality Aware, 4 leaders: 1.080151e-04 +Nonblocking Multileader Locality, 4 leaders: 2.631360e-04 +Pairwise Multileader, 10 leaders: 2.611103e-04 +Pairwise Locality Aware, 10 leaders: 2.719554e-04 +Pairwise Multileader Locality, 10 leaders: 1.444829e-04 +Nonblocking Multileader, 10 leaders: 1.134107e-04 +Nonblocking Locality Aware, 10 leaders: 1.483565e-04 +Nonblocking Multileader Locality, 10 leaders: 1.171043e-04 +Pairwise Multileader, 20 leaders: 5.501526e-04 +Pairwise Locality Aware, 20 leaders: 4.589272e-04 +Pairwise Multileader Locality, 20 leaders: 1.446457e-04 +Nonblocking Multileader, 20 leaders: 1.283113e-04 +Nonblocking Locality Aware, 20 leaders: 1.819367e-04 +Nonblocking Multileader Locality, 20 leaders: 1.078997e-04 +Size 8 +PMPI: 2.394890e-04 +Pairwise: 2.227710e-03 +NonBlocking: 5.928629e-02 +Pairwise Hierarchical: 1.766738e-03 +Pairwise Node Aware: 3.985346e-04 +Nonblocking Hierarchical: 1.611072e-03 +Nonblocking Node Aware: 2.103205e-04 +Pairwise Multileader, 4 leaders: 5.475181e-04 +Pairwise Locality Aware, 4 leaders: 2.020839e-04 +Pairwise Multileader Locality, 4 leaders: 6.859844e-04 +Nonblocking Multileader, 4 leaders: 3.896450e-04 +Nonblocking Locality Aware, 4 leaders: 2.009215e-04 +Nonblocking Multileader Locality, 4 leaders: 5.552230e-04 +Pairwise Multileader, 10 leaders: 4.019376e-04 +Pairwise Locality Aware, 10 leaders: 2.932034e-04 +Pairwise Multileader Locality, 10 leaders: 2.524527e-04 +Nonblocking Multileader, 10 leaders: 1.660132e-04 +Nonblocking Locality Aware, 10 leaders: 2.146377e-04 +Nonblocking Multileader Locality, 10 leaders: 1.938112e-04 +Pairwise Multileader, 20 leaders: 4.781502e-04 +Pairwise Locality Aware, 20 leaders: 5.470618e-04 +Pairwise Multileader Locality, 20 leaders: 1.796707e-04 +Nonblocking Multileader, 20 leaders: 1.634587e-04 +Nonblocking Locality Aware, 20 leaders: 2.524085e-04 +Nonblocking Multileader Locality, 20 leaders: 1.315003e-04 +Size 16 +PMPI: 4.281680e-04 +Pairwise: 2.317121e-03 +NonBlocking: 7.450069e-02 +Pairwise Hierarchical: 3.250982e-03 +Pairwise Node Aware: 4.986251e-04 +Nonblocking Hierarchical: 3.118980e-03 +Nonblocking Node Aware: 2.529204e-04 +Pairwise Multileader, 4 leaders: 9.515105e-04 +Pairwise Locality Aware, 4 leaders: 2.776696e-04 +Pairwise Multileader Locality, 4 leaders: 1.214534e-03 +Nonblocking Multileader, 4 leaders: 7.540784e-04 +Nonblocking Locality Aware, 4 leaders: 2.095644e-04 +Nonblocking Multileader Locality, 4 leaders: 1.017852e-03 +Pairwise Multileader, 10 leaders: 6.315138e-04 +Pairwise Locality Aware, 10 leaders: 3.996266e-04 +Pairwise Multileader Locality, 10 leaders: 4.640824e-04 +Nonblocking Multileader, 10 leaders: 3.376459e-04 +Nonblocking Locality Aware, 10 leaders: 3.161190e-04 +Nonblocking Multileader Locality, 10 leaders: 3.999725e-04 +Pairwise Multileader, 20 leaders: 6.368773e-04 +Pairwise Locality Aware, 20 leaders: 6.060846e-04 +Pairwise Multileader Locality, 20 leaders: 3.284526e-04 +Nonblocking Multileader, 20 leaders: 2.465746e-04 +Nonblocking Locality Aware, 20 leaders: 2.350997e-04 +Nonblocking Multileader Locality, 20 leaders: 2.497121e-04 +Size 32 +PMPI: 7.281583e-04 +Pairwise: 2.322678e-03 +NonBlocking: 7.438984e-02 +Pairwise Hierarchical: 7.496708e-03 +Pairwise Node Aware: 6.346964e-04 +Nonblocking Hierarchical: 6.916483e-03 +Nonblocking Node Aware: 4.222107e-04 +Pairwise Multileader, 4 leaders: 2.981819e-03 +Pairwise Locality Aware, 4 leaders: 4.768206e-04 +Pairwise Multileader Locality, 4 leaders: 2.686296e-03 +Nonblocking Multileader, 4 leaders: 2.250034e-03 +Nonblocking Locality Aware, 4 leaders: 3.077959e-04 +Nonblocking Multileader Locality, 4 leaders: 2.370382e-03 +Pairwise Multileader, 10 leaders: 1.115774e-03 +Pairwise Locality Aware, 10 leaders: 5.187994e-04 +Pairwise Multileader Locality, 10 leaders: 1.131319e-03 +Nonblocking Multileader, 10 leaders: 7.510714e-04 +Nonblocking Locality Aware, 10 leaders: 4.639184e-04 +Nonblocking Multileader Locality, 10 leaders: 9.572828e-04 +Pairwise Multileader, 20 leaders: 9.595547e-04 +Pairwise Locality Aware, 20 leaders: 6.896876e-04 +Pairwise Multileader Locality, 20 leaders: 5.552406e-04 +Nonblocking Multileader, 20 leaders: 4.522662e-04 +Nonblocking Locality Aware, 20 leaders: 5.772606e-04 +Nonblocking Multileader Locality, 20 leaders: 4.541017e-04 +Size 64 +PMPI: 1.484305e-03 +Pairwise: 2.930261e-03 +NonBlocking: 9.929848e-02 +Pairwise Hierarchical: 1.678971e-02 +Pairwise Node Aware: 1.013709e-03 +Nonblocking Hierarchical: 1.526419e-02 +Nonblocking Node Aware: 6.409946e-04 +Pairwise Multileader, 4 leaders: 5.179957e-03 +Pairwise Locality Aware, 4 leaders: 8.048001e-04 +Pairwise Multileader Locality, 4 leaders: 5.178082e-03 +Nonblocking Multileader, 4 leaders: 4.046711e-03 +Nonblocking Locality Aware, 4 leaders: 6.975098e-04 +Nonblocking Multileader Locality, 4 leaders: 4.417177e-03 +Pairwise Multileader, 10 leaders: 2.214954e-03 +Pairwise Locality Aware, 10 leaders: 8.760271e-04 +Pairwise Multileader Locality, 10 leaders: 2.167479e-03 +Nonblocking Multileader, 10 leaders: 1.596172e-03 +Nonblocking Locality Aware, 10 leaders: 7.542466e-04 +Nonblocking Multileader Locality, 10 leaders: 1.875414e-03 +Pairwise Multileader, 20 leaders: 1.538439e-03 +Pairwise Locality Aware, 20 leaders: 1.047883e-03 +Pairwise Multileader Locality, 20 leaders: 1.262711e-03 +Nonblocking Multileader, 20 leaders: 8.402222e-04 +Nonblocking Locality Aware, 20 leaders: 8.995309e-04 +Nonblocking Multileader Locality, 20 leaders: 1.065285e-03 +Size 128 +PMPI: 2.442257e-03 +Pairwise: 2.719586e-03 +NonBlocking: 9.273137e-02 +Pairwise Hierarchical: 3.570570e-02 +Pairwise Node Aware: 1.895922e-03 +Nonblocking Hierarchical: 3.215479e-02 +Nonblocking Node Aware: 1.187034e-03 +Pairwise Multileader, 4 leaders: 9.337444e-03 +Pairwise Locality Aware, 4 leaders: 1.687328e-03 +Pairwise Multileader Locality, 4 leaders: 9.342985e-03 +Nonblocking Multileader, 4 leaders: 7.289848e-03 +Nonblocking Locality Aware, 4 leaders: 1.532264e-03 +Nonblocking Multileader Locality, 4 leaders: 8.227492e-03 +Pairwise Multileader, 10 leaders: 4.528721e-03 +Pairwise Locality Aware, 10 leaders: 1.695602e-03 +Pairwise Multileader Locality, 10 leaders: 4.159072e-03 +Nonblocking Multileader, 10 leaders: 3.188021e-03 +Nonblocking Locality Aware, 10 leaders: 1.552108e-03 +Nonblocking Multileader Locality, 10 leaders: 3.701975e-03 +Pairwise Multileader, 20 leaders: 2.778367e-03 +Pairwise Locality Aware, 20 leaders: 1.828108e-03 +Pairwise Multileader Locality, 20 leaders: 2.557812e-03 +Nonblocking Multileader, 20 leaders: 1.773117e-03 +Nonblocking Locality Aware, 20 leaders: 1.732499e-03 +Nonblocking Multileader Locality, 20 leaders: 2.135864e-03 +Size 256 +PMPI: 5.005469e-03 +Pairwise: 3.837883e-03 +NonBlocking: 8.836640e-02 +Pairwise Hierarchical: 6.607381e-02 +Pairwise Node Aware: 2.564841e-03 +Nonblocking Hierarchical: 6.315866e-02 +Nonblocking Node Aware: 1.851882e-03 +Pairwise Multileader, 4 leaders: 2.154284e-02 +Pairwise Locality Aware, 4 leaders: 3.081103e-03 +Pairwise Multileader Locality, 4 leaders: 1.949419e-02 +Nonblocking Multileader, 4 leaders: 1.517833e-02 +Nonblocking Locality Aware, 4 leaders: 2.581822e-03 +Nonblocking Multileader Locality, 4 leaders: 1.693829e-02 +Pairwise Multileader, 10 leaders: 1.187716e-02 +Pairwise Locality Aware, 10 leaders: 2.915779e-03 +Pairwise Multileader Locality, 10 leaders: 9.189114e-03 +Nonblocking Multileader, 10 leaders: 7.664231e-03 +Nonblocking Locality Aware, 10 leaders: 3.138547e-03 +Nonblocking Multileader Locality, 10 leaders: 8.033214e-03 +Pairwise Multileader, 20 leaders: 5.416156e-03 +Pairwise Locality Aware, 20 leaders: 2.759849e-03 +Pairwise Multileader Locality, 20 leaders: 4.751085e-03 +Nonblocking Multileader, 20 leaders: 3.404501e-03 +Nonblocking Locality Aware, 20 leaders: 3.108744e-03 +Nonblocking Multileader Locality, 20 leaders: 4.023869e-03 +Size 512 +PMPI: 6.316583e-03 +Pairwise: 6.568837e-03 +NonBlocking: 8.337053e-02 +Pairwise Hierarchical: 1.225266e-01 +Pairwise Node Aware: 4.924787e-03 +Nonblocking Hierarchical: 1.252913e-01 +Nonblocking Node Aware: 4.349094e-03 +Pairwise Multileader, 4 leaders: 4.620888e-02 +Pairwise Locality Aware, 4 leaders: 6.253617e-03 +Pairwise Multileader Locality, 4 leaders: 4.769887e-02 +Nonblocking Multileader, 4 leaders: 3.498914e-02 +Nonblocking Locality Aware, 4 leaders: 5.187116e-03 +Nonblocking Multileader Locality, 4 leaders: 4.001431e-02 +Pairwise Multileader, 10 leaders: 2.426082e-02 +Pairwise Locality Aware, 10 leaders: 5.778809e-03 +Pairwise Multileader Locality, 10 leaders: 2.145178e-02 +Nonblocking Multileader, 10 leaders: 1.649248e-02 +Nonblocking Locality Aware, 10 leaders: 5.044111e-03 +Nonblocking Multileader Locality, 10 leaders: 1.881768e-02 +Pairwise Multileader, 20 leaders: 1.206688e-02 +Pairwise Locality Aware, 20 leaders: 6.182847e-03 +Pairwise Multileader Locality, 20 leaders: 1.026540e-02 +Nonblocking Multileader, 20 leaders: 7.423401e-03 +Nonblocking Locality Aware, 20 leaders: 5.666016e-03 +Nonblocking Multileader Locality, 20 leaders: 8.858660e-03 +Test 2 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 8.503872e-05 +Pairwise: 1.147328e-03 +NonBlocking: 3.447987e-04 +Pairwise Hierarchical: 3.164029e-04 +Pairwise Node Aware: 2.057177e-04 +Nonblocking Hierarchical: 3.037436e-04 +Nonblocking Node Aware: 1.898930e-04 +Pairwise Multileader, 4 leaders: 1.670178e-04 +Pairwise Locality Aware, 4 leaders: 1.643356e-04 +Pairwise Multileader Locality, 4 leaders: 1.257903e-04 +Nonblocking Multileader, 4 leaders: 9.101432e-05 +Nonblocking Locality Aware, 4 leaders: 9.259873e-05 +Nonblocking Multileader Locality, 4 leaders: 1.272054e-04 +Pairwise Multileader, 10 leaders: 3.965868e-04 +Pairwise Locality Aware, 10 leaders: 4.252514e-04 +Pairwise Multileader Locality, 10 leaders: 9.447734e-05 +Nonblocking Multileader, 10 leaders: 8.450188e-05 +Nonblocking Locality Aware, 10 leaders: 1.326172e-04 +Nonblocking Multileader Locality, 10 leaders: 7.890244e-05 +Pairwise Multileader, 20 leaders: 8.020913e-04 +Pairwise Locality Aware, 20 leaders: 7.461118e-04 +Pairwise Multileader Locality, 20 leaders: 1.109396e-04 +Nonblocking Multileader, 20 leaders: 1.468210e-04 +Nonblocking Locality Aware, 20 leaders: 1.802754e-04 +Nonblocking Multileader Locality, 20 leaders: 7.566706e-05 +Size 2 +PMPI: 1.440450e-04 +Pairwise: 1.696086e-03 +NonBlocking: 3.978223e-04 +Pairwise Hierarchical: 7.779179e-04 +Pairwise Node Aware: 3.029091e-04 +Nonblocking Hierarchical: 5.762528e-04 +Nonblocking Node Aware: 2.725829e-04 +Pairwise Multileader, 4 leaders: 2.668489e-04 +Pairwise Locality Aware, 4 leaders: 2.564474e-04 +Pairwise Multileader Locality, 4 leaders: 2.044564e-04 +Nonblocking Multileader, 4 leaders: 1.473160e-04 +Nonblocking Locality Aware, 4 leaders: 9.921134e-05 +Nonblocking Multileader Locality, 4 leaders: 1.532782e-04 +Pairwise Multileader, 10 leaders: 3.144887e-04 +Pairwise Locality Aware, 10 leaders: 2.794607e-04 +Pairwise Multileader Locality, 10 leaders: 9.715137e-05 +Nonblocking Multileader, 10 leaders: 9.441750e-05 +Nonblocking Locality Aware, 10 leaders: 1.451010e-04 +Nonblocking Multileader Locality, 10 leaders: 8.489989e-05 +Pairwise Multileader, 20 leaders: 5.221341e-04 +Pairwise Locality Aware, 20 leaders: 5.376418e-04 +Pairwise Multileader Locality, 20 leaders: 1.089537e-04 +Nonblocking Multileader, 20 leaders: 1.180199e-04 +Nonblocking Locality Aware, 20 leaders: 1.482294e-04 +Nonblocking Multileader Locality, 20 leaders: 8.487180e-05 +Size 4 +PMPI: 1.525129e-04 +Pairwise: 2.171399e-03 +NonBlocking: 7.428756e-02 +Pairwise Hierarchical: 9.875639e-04 +Pairwise Node Aware: 3.155647e-04 +Nonblocking Hierarchical: 8.750009e-04 +Nonblocking Node Aware: 1.970652e-04 +Pairwise Multileader, 4 leaders: 3.319691e-04 +Pairwise Locality Aware, 4 leaders: 1.842424e-04 +Pairwise Multileader Locality, 4 leaders: 3.343711e-04 +Nonblocking Multileader, 4 leaders: 2.321325e-04 +Nonblocking Locality Aware, 4 leaders: 1.390234e-04 +Nonblocking Multileader Locality, 4 leaders: 2.787225e-04 +Pairwise Multileader, 10 leaders: 2.967980e-04 +Pairwise Locality Aware, 10 leaders: 3.556998e-04 +Pairwise Multileader Locality, 10 leaders: 1.398317e-04 +Nonblocking Multileader, 10 leaders: 1.173982e-04 +Nonblocking Locality Aware, 10 leaders: 1.416175e-04 +Nonblocking Multileader Locality, 10 leaders: 1.136832e-04 +Pairwise Multileader, 20 leaders: 5.872974e-04 +Pairwise Locality Aware, 20 leaders: 5.479951e-04 +Pairwise Multileader Locality, 20 leaders: 1.300277e-04 +Nonblocking Multileader, 20 leaders: 1.301201e-04 +Nonblocking Locality Aware, 20 leaders: 2.444947e-04 +Nonblocking Multileader Locality, 20 leaders: 1.027073e-04 +Size 8 +PMPI: 2.543556e-04 +Pairwise: 2.552002e-03 +NonBlocking: 8.665556e-02 +Pairwise Hierarchical: 1.792005e-03 +Pairwise Node Aware: 3.809217e-04 +Nonblocking Hierarchical: 1.615706e-03 +Nonblocking Node Aware: 2.243701e-04 +Pairwise Multileader, 4 leaders: 5.448971e-04 +Pairwise Locality Aware, 4 leaders: 2.131799e-04 +Pairwise Multileader Locality, 4 leaders: 7.624524e-04 +Nonblocking Multileader, 4 leaders: 4.082647e-04 +Nonblocking Locality Aware, 4 leaders: 1.547627e-04 +Nonblocking Multileader Locality, 4 leaders: 5.672260e-04 +Pairwise Multileader, 10 leaders: 4.034050e-04 +Pairwise Locality Aware, 10 leaders: 3.009041e-04 +Pairwise Multileader Locality, 10 leaders: 2.718097e-04 +Nonblocking Multileader, 10 leaders: 2.078673e-04 +Nonblocking Locality Aware, 10 leaders: 2.188947e-04 +Nonblocking Multileader Locality, 10 leaders: 1.949123e-04 +Pairwise Multileader, 20 leaders: 5.798551e-04 +Pairwise Locality Aware, 20 leaders: 7.046114e-04 +Pairwise Multileader Locality, 20 leaders: 2.366816e-04 +Nonblocking Multileader, 20 leaders: 2.263987e-04 +Nonblocking Locality Aware, 20 leaders: 3.881793e-04 +Nonblocking Multileader Locality, 20 leaders: 1.656620e-04 +Size 16 +PMPI: 4.252556e-04 +Pairwise: 2.065934e-03 +NonBlocking: 8.412546e-02 +Pairwise Hierarchical: 3.359369e-03 +Pairwise Node Aware: 4.284795e-04 +Nonblocking Hierarchical: 3.146199e-03 +Nonblocking Node Aware: 2.636088e-04 +Pairwise Multileader, 4 leaders: 9.596287e-04 +Pairwise Locality Aware, 4 leaders: 3.019160e-04 +Pairwise Multileader Locality, 4 leaders: 1.268897e-03 +Nonblocking Multileader, 4 leaders: 7.417896e-04 +Nonblocking Locality Aware, 4 leaders: 2.688435e-04 +Nonblocking Multileader Locality, 4 leaders: 1.055673e-03 +Pairwise Multileader, 10 leaders: 6.653915e-04 +Pairwise Locality Aware, 10 leaders: 3.231002e-04 +Pairwise Multileader Locality, 10 leaders: 4.570654e-04 +Nonblocking Multileader, 10 leaders: 3.420159e-04 +Nonblocking Locality Aware, 10 leaders: 2.441980e-04 +Nonblocking Multileader Locality, 10 leaders: 4.076832e-04 +Pairwise Multileader, 20 leaders: 6.473523e-04 +Pairwise Locality Aware, 20 leaders: 5.593454e-04 +Pairwise Multileader Locality, 20 leaders: 3.368069e-04 +Nonblocking Multileader, 20 leaders: 2.678911e-04 +Nonblocking Locality Aware, 20 leaders: 3.211008e-04 +Nonblocking Multileader Locality, 20 leaders: 2.669388e-04 +Size 32 +PMPI: 7.380577e-04 +Pairwise: 2.332438e-03 +NonBlocking: 8.667199e-02 +Pairwise Hierarchical: 7.586778e-03 +Pairwise Node Aware: 6.279347e-04 +Nonblocking Hierarchical: 6.973994e-03 +Nonblocking Node Aware: 3.883381e-04 +Pairwise Multileader, 4 leaders: 2.953817e-03 +Pairwise Locality Aware, 4 leaders: 4.732939e-04 +Pairwise Multileader Locality, 4 leaders: 2.682764e-03 +Nonblocking Multileader, 4 leaders: 2.246431e-03 +Nonblocking Locality Aware, 4 leaders: 4.012886e-04 +Nonblocking Multileader Locality, 4 leaders: 2.342059e-03 +Pairwise Multileader, 10 leaders: 1.139253e-03 +Pairwise Locality Aware, 10 leaders: 5.279653e-04 +Pairwise Multileader Locality, 10 leaders: 1.120471e-03 +Nonblocking Multileader, 10 leaders: 7.373799e-04 +Nonblocking Locality Aware, 10 leaders: 4.282235e-04 +Nonblocking Multileader Locality, 10 leaders: 9.578859e-04 +Pairwise Multileader, 20 leaders: 8.934132e-04 +Pairwise Locality Aware, 20 leaders: 6.884096e-04 +Pairwise Multileader Locality, 20 leaders: 5.045908e-04 +Nonblocking Multileader, 20 leaders: 3.639560e-04 +Nonblocking Locality Aware, 20 leaders: 3.597273e-04 +Nonblocking Multileader Locality, 20 leaders: 4.498044e-04 +Size 64 +PMPI: 1.513596e-03 +Pairwise: 2.577955e-03 +NonBlocking: 8.687678e-02 +Pairwise Hierarchical: 1.714496e-02 +Pairwise Node Aware: 1.041416e-03 +Nonblocking Hierarchical: 1.534374e-02 +Nonblocking Node Aware: 6.723209e-04 +Pairwise Multileader, 4 leaders: 5.206848e-03 +Pairwise Locality Aware, 4 leaders: 7.038882e-04 +Pairwise Multileader Locality, 4 leaders: 5.057481e-03 +Nonblocking Multileader, 4 leaders: 4.048510e-03 +Nonblocking Locality Aware, 4 leaders: 5.162056e-04 +Nonblocking Multileader Locality, 4 leaders: 4.469792e-03 +Pairwise Multileader, 10 leaders: 2.244122e-03 +Pairwise Locality Aware, 10 leaders: 1.035060e-03 +Pairwise Multileader Locality, 10 leaders: 2.161126e-03 +Nonblocking Multileader, 10 leaders: 1.590444e-03 +Nonblocking Locality Aware, 10 leaders: 8.163976e-04 +Nonblocking Multileader Locality, 10 leaders: 1.883395e-03 +Pairwise Multileader, 20 leaders: 1.513863e-03 +Pairwise Locality Aware, 20 leaders: 9.314801e-04 +Pairwise Multileader Locality, 20 leaders: 1.260137e-03 +Nonblocking Multileader, 20 leaders: 7.863468e-04 +Nonblocking Locality Aware, 20 leaders: 5.962122e-04 +Nonblocking Multileader Locality, 20 leaders: 1.044856e-03 +Size 128 +PMPI: 2.503398e-03 +Pairwise: 2.992225e-03 +NonBlocking: 1.153107e-01 +Pairwise Hierarchical: 3.511593e-02 +Pairwise Node Aware: 2.079859e-03 +Nonblocking Hierarchical: 3.223228e-02 +Nonblocking Node Aware: 1.289319e-03 +Pairwise Multileader, 4 leaders: 9.976199e-03 +Pairwise Locality Aware, 4 leaders: 1.383952e-03 +Pairwise Multileader Locality, 4 leaders: 9.441967e-03 +Nonblocking Multileader, 4 leaders: 7.420107e-03 +Nonblocking Locality Aware, 4 leaders: 1.061108e-03 +Nonblocking Multileader Locality, 4 leaders: 8.250059e-03 +Pairwise Multileader, 10 leaders: 4.330419e-03 +Pairwise Locality Aware, 10 leaders: 1.609336e-03 +Pairwise Multileader Locality, 10 leaders: 4.113153e-03 +Nonblocking Multileader, 10 leaders: 3.147640e-03 +Nonblocking Locality Aware, 10 leaders: 1.402367e-03 +Nonblocking Multileader Locality, 10 leaders: 3.645248e-03 +Pairwise Multileader, 20 leaders: 2.783240e-03 +Pairwise Locality Aware, 20 leaders: 1.562159e-03 +Pairwise Multileader Locality, 20 leaders: 2.513526e-03 +Nonblocking Multileader, 20 leaders: 1.661522e-03 +Nonblocking Locality Aware, 20 leaders: 1.336675e-03 +Nonblocking Multileader Locality, 20 leaders: 2.116458e-03 +Size 256 +PMPI: 4.778965e-03 +Pairwise: 4.131146e-03 +NonBlocking: 1.159799e-01 +Pairwise Hierarchical: 6.658492e-02 +Pairwise Node Aware: 2.637469e-03 +Nonblocking Hierarchical: 6.493608e-02 +Nonblocking Node Aware: 1.769324e-03 +Pairwise Multileader, 4 leaders: 2.149274e-02 +Pairwise Locality Aware, 4 leaders: 2.874304e-03 +Pairwise Multileader Locality, 4 leaders: 2.028505e-02 +Nonblocking Multileader, 4 leaders: 1.481903e-02 +Nonblocking Locality Aware, 4 leaders: 2.319860e-03 +Nonblocking Multileader Locality, 4 leaders: 1.746520e-02 +Pairwise Multileader, 10 leaders: 1.199875e-02 +Pairwise Locality Aware, 10 leaders: 2.970612e-03 +Pairwise Multileader Locality, 10 leaders: 9.003040e-03 +Nonblocking Multileader, 10 leaders: 7.571161e-03 +Nonblocking Locality Aware, 10 leaders: 2.986022e-03 +Nonblocking Multileader Locality, 10 leaders: 8.029052e-03 +Pairwise Multileader, 20 leaders: 5.704556e-03 +Pairwise Locality Aware, 20 leaders: 3.087881e-03 +Pairwise Multileader Locality, 20 leaders: 5.083781e-03 +Nonblocking Multileader, 20 leaders: 3.408210e-03 +Nonblocking Locality Aware, 20 leaders: 2.983196e-03 +Nonblocking Multileader Locality, 20 leaders: 4.027800e-03 +Size 512 +PMPI: 6.301292e-03 +Pairwise: 6.578791e-03 +NonBlocking: 9.424077e-02 +Pairwise Hierarchical: 1.250610e-01 +Pairwise Node Aware: 5.080147e-03 +Nonblocking Hierarchical: 1.260840e-01 +Nonblocking Node Aware: 3.462787e-03 +Pairwise Multileader, 4 leaders: 4.601277e-02 +Pairwise Locality Aware, 4 leaders: 5.822949e-03 +Pairwise Multileader Locality, 4 leaders: 4.769676e-02 +Nonblocking Multileader, 4 leaders: 3.504825e-02 +Nonblocking Locality Aware, 4 leaders: 4.523787e-03 +Nonblocking Multileader Locality, 4 leaders: 4.067766e-02 +Pairwise Multileader, 10 leaders: 2.398237e-02 +Pairwise Locality Aware, 10 leaders: 5.721293e-03 +Pairwise Multileader Locality, 10 leaders: 2.145000e-02 +Nonblocking Multileader, 10 leaders: 1.634917e-02 +Nonblocking Locality Aware, 10 leaders: 5.265444e-03 +Nonblocking Multileader Locality, 10 leaders: 1.872206e-02 +Pairwise Multileader, 20 leaders: 1.125266e-02 +Pairwise Locality Aware, 20 leaders: 6.089763e-03 +Pairwise Multileader Locality, 20 leaders: 1.014712e-02 +Nonblocking Multileader, 20 leaders: 7.519066e-03 +Nonblocking Locality Aware, 20 leaders: 6.245117e-03 +Nonblocking Multileader Locality, 20 leaders: 8.819144e-03 +Test 3 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 8.160425e-05 +Pairwise: 1.059951e-03 +NonBlocking: 3.299666e-04 +Pairwise Hierarchical: 6.691913e-04 +Pairwise Node Aware: 2.115173e-04 +Nonblocking Hierarchical: 3.070221e-04 +Nonblocking Node Aware: 1.930389e-04 +Pairwise Multileader, 4 leaders: 1.614867e-04 +Pairwise Locality Aware, 4 leaders: 1.501370e-04 +Pairwise Multileader Locality, 4 leaders: 1.235407e-04 +Nonblocking Multileader, 4 leaders: 9.001299e-05 +Nonblocking Locality Aware, 4 leaders: 7.590228e-05 +Nonblocking Multileader Locality, 4 leaders: 1.047795e-04 +Pairwise Multileader, 10 leaders: 2.258053e-04 +Pairwise Locality Aware, 10 leaders: 2.564098e-04 +Pairwise Multileader Locality, 10 leaders: 7.760243e-05 +Nonblocking Multileader, 10 leaders: 7.028652e-05 +Nonblocking Locality Aware, 10 leaders: 1.056585e-04 +Nonblocking Multileader Locality, 10 leaders: 6.515994e-05 +Pairwise Multileader, 20 leaders: 4.504461e-04 +Pairwise Locality Aware, 20 leaders: 4.659882e-04 +Pairwise Multileader Locality, 20 leaders: 9.069284e-05 +Nonblocking Multileader, 20 leaders: 1.115179e-04 +Nonblocking Locality Aware, 20 leaders: 1.379441e-04 +Nonblocking Multileader Locality, 20 leaders: 6.062203e-05 +Size 2 +PMPI: 1.022397e-04 +Pairwise: 1.074684e-03 +NonBlocking: 2.991657e-04 +Pairwise Hierarchical: 6.277252e-04 +Pairwise Node Aware: 2.567484e-04 +Nonblocking Hierarchical: 5.648385e-04 +Nonblocking Node Aware: 2.048006e-04 +Pairwise Multileader, 4 leaders: 1.849324e-04 +Pairwise Locality Aware, 4 leaders: 1.676040e-04 +Pairwise Multileader Locality, 4 leaders: 1.614267e-04 +Nonblocking Multileader, 4 leaders: 1.181166e-04 +Nonblocking Locality Aware, 4 leaders: 1.012088e-04 +Nonblocking Multileader Locality, 4 leaders: 1.331734e-04 +Pairwise Multileader, 10 leaders: 2.383712e-04 +Pairwise Locality Aware, 10 leaders: 3.143924e-04 +Pairwise Multileader Locality, 10 leaders: 9.450926e-05 +Nonblocking Multileader, 10 leaders: 8.274559e-05 +Nonblocking Locality Aware, 10 leaders: 1.285407e-04 +Nonblocking Multileader Locality, 10 leaders: 7.963962e-05 +Pairwise Multileader, 20 leaders: 4.284175e-04 +Pairwise Locality Aware, 20 leaders: 5.206094e-04 +Pairwise Multileader Locality, 20 leaders: 1.017414e-04 +Nonblocking Multileader, 20 leaders: 1.212474e-04 +Nonblocking Locality Aware, 20 leaders: 1.784239e-04 +Nonblocking Multileader Locality, 20 leaders: 7.414924e-05 +Size 4 +PMPI: 1.353080e-04 +Pairwise: 2.390051e-03 +NonBlocking: 2.513909e-02 +Pairwise Hierarchical: 9.623351e-04 +Pairwise Node Aware: 2.866608e-04 +Nonblocking Hierarchical: 8.648708e-04 +Nonblocking Node Aware: 1.994908e-04 +Pairwise Multileader, 4 leaders: 3.292700e-04 +Pairwise Locality Aware, 4 leaders: 1.697356e-04 +Pairwise Multileader Locality, 4 leaders: 3.277876e-04 +Nonblocking Multileader, 4 leaders: 2.220115e-04 +Nonblocking Locality Aware, 4 leaders: 1.285451e-04 +Nonblocking Multileader Locality, 4 leaders: 2.761111e-04 +Pairwise Multileader, 10 leaders: 2.725210e-04 +Pairwise Locality Aware, 10 leaders: 2.812213e-04 +Pairwise Multileader Locality, 10 leaders: 1.419304e-04 +Nonblocking Multileader, 10 leaders: 1.060863e-04 +Nonblocking Locality Aware, 10 leaders: 1.351330e-04 +Nonblocking Multileader Locality, 10 leaders: 1.164907e-04 +Pairwise Multileader, 20 leaders: 4.561576e-04 +Pairwise Locality Aware, 20 leaders: 5.035543e-04 +Pairwise Multileader Locality, 20 leaders: 1.431338e-04 +Nonblocking Multileader, 20 leaders: 1.396965e-04 +Nonblocking Locality Aware, 20 leaders: 2.009915e-04 +Nonblocking Multileader Locality, 20 leaders: 1.060871e-04 +Size 8 +PMPI: 2.323792e-04 +Pairwise: 2.148919e-03 +NonBlocking: 3.744771e-02 +Pairwise Hierarchical: 1.781874e-03 +Pairwise Node Aware: 3.627679e-04 +Nonblocking Hierarchical: 1.616453e-03 +Nonblocking Node Aware: 2.093806e-04 +Pairwise Multileader, 4 leaders: 5.482580e-04 +Pairwise Locality Aware, 4 leaders: 2.010825e-04 +Pairwise Multileader Locality, 4 leaders: 6.966980e-04 +Nonblocking Multileader, 4 leaders: 3.943012e-04 +Nonblocking Locality Aware, 4 leaders: 1.796779e-04 +Nonblocking Multileader Locality, 4 leaders: 5.583468e-04 +Pairwise Multileader, 10 leaders: 3.842727e-04 +Pairwise Locality Aware, 10 leaders: 3.183921e-04 +Pairwise Multileader Locality, 10 leaders: 3.866585e-04 +Nonblocking Multileader, 10 leaders: 1.699607e-04 +Nonblocking Locality Aware, 10 leaders: 1.688090e-04 +Nonblocking Multileader Locality, 10 leaders: 2.709031e-04 +Pairwise Multileader, 20 leaders: 5.208399e-04 +Pairwise Locality Aware, 20 leaders: 4.492799e-04 +Pairwise Multileader Locality, 20 leaders: 2.042945e-04 +Nonblocking Multileader, 20 leaders: 1.576662e-04 +Nonblocking Locality Aware, 20 leaders: 2.033167e-04 +Nonblocking Multileader Locality, 20 leaders: 1.577144e-04 +Size 16 +PMPI: 4.299114e-04 +Pairwise: 2.028508e-03 +NonBlocking: 6.208112e-02 +Pairwise Hierarchical: 3.426231e-03 +Pairwise Node Aware: 4.362934e-04 +Nonblocking Hierarchical: 3.161230e-03 +Nonblocking Node Aware: 2.639943e-04 +Pairwise Multileader, 4 leaders: 9.669630e-04 +Pairwise Locality Aware, 4 leaders: 2.484167e-04 +Pairwise Multileader Locality, 4 leaders: 1.573520e-03 +Nonblocking Multileader, 4 leaders: 7.566808e-04 +Nonblocking Locality Aware, 4 leaders: 1.849520e-04 +Nonblocking Multileader Locality, 4 leaders: 1.308051e-03 +Pairwise Multileader, 10 leaders: 6.872063e-04 +Pairwise Locality Aware, 10 leaders: 3.882338e-04 +Pairwise Multileader Locality, 10 leaders: 4.731467e-04 +Nonblocking Multileader, 10 leaders: 3.382441e-04 +Nonblocking Locality Aware, 10 leaders: 2.849526e-04 +Nonblocking Multileader Locality, 10 leaders: 3.732331e-04 +Pairwise Multileader, 20 leaders: 6.547761e-04 +Pairwise Locality Aware, 20 leaders: 5.547814e-04 +Pairwise Multileader Locality, 20 leaders: 5.211253e-04 +Nonblocking Multileader, 20 leaders: 3.154539e-04 +Nonblocking Locality Aware, 20 leaders: 2.966421e-04 +Nonblocking Multileader Locality, 20 leaders: 3.239389e-04 +Size 32 +PMPI: 7.821414e-04 +Pairwise: 3.149487e-03 +NonBlocking: 6.211283e-02 +Pairwise Hierarchical: 7.533360e-03 +Pairwise Node Aware: 7.050657e-04 +Nonblocking Hierarchical: 6.919556e-03 +Nonblocking Node Aware: 4.461024e-04 +Pairwise Multileader, 4 leaders: 3.142784e-03 +Pairwise Locality Aware, 4 leaders: 4.181528e-04 +Pairwise Multileader Locality, 4 leaders: 2.994883e-03 +Nonblocking Multileader, 4 leaders: 2.254028e-03 +Nonblocking Locality Aware, 4 leaders: 3.255928e-04 +Nonblocking Multileader Locality, 4 leaders: 2.575763e-03 +Pairwise Multileader, 10 leaders: 1.153586e-03 +Pairwise Locality Aware, 10 leaders: 4.734344e-04 +Pairwise Multileader Locality, 10 leaders: 1.142984e-03 +Nonblocking Multileader, 10 leaders: 7.537922e-04 +Nonblocking Locality Aware, 10 leaders: 3.939736e-04 +Nonblocking Multileader Locality, 10 leaders: 9.665523e-04 +Pairwise Multileader, 20 leaders: 9.207546e-04 +Pairwise Locality Aware, 20 leaders: 6.438669e-04 +Pairwise Multileader Locality, 20 leaders: 7.809054e-04 +Nonblocking Multileader, 20 leaders: 3.622393e-04 +Nonblocking Locality Aware, 20 leaders: 3.683352e-04 +Nonblocking Multileader Locality, 20 leaders: 5.182173e-04 +Size 64 +PMPI: 1.477805e-03 +Pairwise: 2.520031e-03 +NonBlocking: 7.459694e-02 +Pairwise Hierarchical: 1.729241e-02 +Pairwise Node Aware: 1.088477e-03 +Nonblocking Hierarchical: 1.534172e-02 +Nonblocking Node Aware: 6.873836e-04 +Pairwise Multileader, 4 leaders: 5.367269e-03 +Pairwise Locality Aware, 4 leaders: 7.455375e-04 +Pairwise Multileader Locality, 4 leaders: 5.274970e-03 +Nonblocking Multileader, 4 leaders: 4.044455e-03 +Nonblocking Locality Aware, 4 leaders: 6.281986e-04 +Nonblocking Multileader Locality, 4 leaders: 4.938802e-03 +Pairwise Multileader, 10 leaders: 2.355858e-03 +Pairwise Locality Aware, 10 leaders: 8.556073e-04 +Pairwise Multileader Locality, 10 leaders: 2.179106e-03 +Nonblocking Multileader, 10 leaders: 1.612965e-03 +Nonblocking Locality Aware, 10 leaders: 6.652883e-04 +Nonblocking Multileader Locality, 10 leaders: 1.877660e-03 +Pairwise Multileader, 20 leaders: 1.574124e-03 +Pairwise Locality Aware, 20 leaders: 9.711777e-04 +Pairwise Multileader Locality, 20 leaders: 1.598368e-03 +Nonblocking Multileader, 20 leaders: 7.489191e-04 +Nonblocking Locality Aware, 20 leaders: 6.355830e-04 +Nonblocking Multileader Locality, 20 leaders: 1.285594e-03 +Size 128 +PMPI: 2.443652e-03 +Pairwise: 2.963848e-03 +NonBlocking: 8.720009e-02 +Pairwise Hierarchical: 3.536366e-02 +Pairwise Node Aware: 2.147465e-03 +Nonblocking Hierarchical: 3.255915e-02 +Nonblocking Node Aware: 1.296337e-03 +Pairwise Multileader, 4 leaders: 9.796390e-03 +Pairwise Locality Aware, 4 leaders: 1.529353e-03 +Pairwise Multileader Locality, 4 leaders: 9.542159e-03 +Nonblocking Multileader, 4 leaders: 7.235498e-03 +Nonblocking Locality Aware, 4 leaders: 1.214882e-03 +Nonblocking Multileader Locality, 4 leaders: 8.216949e-03 +Pairwise Multileader, 10 leaders: 4.353505e-03 +Pairwise Locality Aware, 10 leaders: 1.637865e-03 +Pairwise Multileader Locality, 10 leaders: 4.137277e-03 +Nonblocking Multileader, 10 leaders: 3.182319e-03 +Nonblocking Locality Aware, 10 leaders: 1.418826e-03 +Nonblocking Multileader Locality, 10 leaders: 3.650863e-03 +Pairwise Multileader, 20 leaders: 2.779098e-03 +Pairwise Locality Aware, 20 leaders: 1.660206e-03 +Pairwise Multileader Locality, 20 leaders: 3.057611e-03 +Nonblocking Multileader, 20 leaders: 1.732876e-03 +Nonblocking Locality Aware, 20 leaders: 1.522408e-03 +Nonblocking Multileader Locality, 20 leaders: 2.230248e-03 +Size 256 +PMPI: 4.757780e-03 +Pairwise: 3.960353e-03 +NonBlocking: 9.374619e-02 +Pairwise Hierarchical: 6.612757e-02 +Pairwise Node Aware: 2.675937e-03 +Nonblocking Hierarchical: 6.385499e-02 +Nonblocking Node Aware: 1.741943e-03 +Pairwise Multileader, 4 leaders: 2.131690e-02 +Pairwise Locality Aware, 4 leaders: 2.934999e-03 +Pairwise Multileader Locality, 4 leaders: 1.954665e-02 +Nonblocking Multileader, 4 leaders: 1.505629e-02 +Nonblocking Locality Aware, 4 leaders: 2.305864e-03 +Nonblocking Multileader Locality, 4 leaders: 1.736268e-02 +Pairwise Multileader, 10 leaders: 1.216783e-02 +Pairwise Locality Aware, 10 leaders: 3.092426e-03 +Pairwise Multileader Locality, 10 leaders: 8.987292e-03 +Nonblocking Multileader, 10 leaders: 7.564239e-03 +Nonblocking Locality Aware, 10 leaders: 3.228154e-03 +Nonblocking Multileader Locality, 10 leaders: 7.987688e-03 +Pairwise Multileader, 20 leaders: 5.400729e-03 +Pairwise Locality Aware, 20 leaders: 2.714080e-03 +Pairwise Multileader Locality, 20 leaders: 4.733432e-03 +Nonblocking Multileader, 20 leaders: 3.520349e-03 +Nonblocking Locality Aware, 20 leaders: 2.492493e-03 +Nonblocking Multileader Locality, 20 leaders: 4.039316e-03 +Size 512 +PMPI: 7.190840e-03 +Pairwise: 7.106278e-03 +NonBlocking: 6.664823e-02 +Pairwise Hierarchical: 1.237513e-01 +Pairwise Node Aware: 5.511044e-03 +Nonblocking Hierarchical: 1.226265e-01 +Nonblocking Node Aware: 4.684876e-03 +Pairwise Multileader, 4 leaders: 4.554884e-02 +Pairwise Locality Aware, 4 leaders: 5.507487e-03 +Pairwise Multileader Locality, 4 leaders: 4.891368e-02 +Nonblocking Multileader, 4 leaders: 3.526300e-02 +Nonblocking Locality Aware, 4 leaders: 4.041416e-03 +Nonblocking Multileader Locality, 4 leaders: 4.036321e-02 +Pairwise Multileader, 10 leaders: 2.448000e-02 +Pairwise Locality Aware, 10 leaders: 5.742487e-03 +Pairwise Multileader Locality, 10 leaders: 2.139839e-02 +Nonblocking Multileader, 10 leaders: 1.648364e-02 +Nonblocking Locality Aware, 10 leaders: 4.998290e-03 +Nonblocking Multileader Locality, 10 leaders: 1.877412e-02 +Pairwise Multileader, 20 leaders: 1.195466e-02 +Pairwise Locality Aware, 20 leaders: 6.938230e-03 +Pairwise Multileader Locality, 20 leaders: 1.023432e-02 +Nonblocking Multileader, 20 leaders: 7.561302e-03 +Nonblocking Locality Aware, 20 leaders: 7.325736e-03 +Nonblocking Multileader Locality, 20 leaders: 8.916453e-03 diff --git a/runscripts/dane_results/alltoall_N64.2381606.err b/runscripts/dane_results/alltoall_N64.2381606.err new file mode 100644 index 000000000..1a698caf9 --- /dev/null +++ b/runscripts/dane_results/alltoall_N64.2381606.err @@ -0,0 +1,3 @@ +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +slurmstepd: error: *** STEP 2381606.2 ON dane95 CANCELLED AT 2025-04-30T17:31:11 DUE TO TIME LIMIT *** +slurmstepd: error: *** JOB 2381606 ON dane95 CANCELLED AT 2025-04-30T17:31:11 DUE TO TIME LIMIT *** diff --git a/runscripts/dane_results/alltoall_N64.2381606.out b/runscripts/dane_results/alltoall_N64.2381606.out new file mode 100644 index 000000000..f446befd7 --- /dev/null +++ b/runscripts/dane_results/alltoall_N64.2381606.out @@ -0,0 +1,759 @@ +Test 1 Running Alltoall Benchmark on 64 nodes and 6400 tasks +Size 1 +PMPI: 1.853250e-03 +Pairwise: 5.229431e-02 +NonBlocking: 4.909642e-01 +Pairwise Hierarchical: 5.511840e-03 +Pairwise Node Aware: 1.343061e-03 +Nonblocking Hierarchical: 4.374256e-03 +Nonblocking Node Aware: 6.913206e-04 +Pairwise Multileader, 4 leaders: 4.413421e-03 +Pairwise Locality Aware, 4 leaders: 4.052943e-03 +Pairwise Multileader Locality, 4 leaders: 2.208484e-03 +Nonblocking Multileader, 4 leaders: 1.438868e-03 +Nonblocking Locality Aware, 4 leaders: 1.156946e-01 +Nonblocking Multileader Locality, 4 leaders: 1.334195e-03 +Pairwise Multileader, 10 leaders: 8.212322e-03 +Pairwise Locality Aware, 10 leaders: 8.841473e-03 +Pairwise Multileader Locality, 10 leaders: 1.428593e-03 +Nonblocking Multileader, 10 leaders: 3.698618e-02 +Nonblocking Locality Aware, 10 leaders: 1.383892e-01 +Nonblocking Multileader Locality, 10 leaders: 7.457503e-04 +Pairwise Multileader, 20 leaders: 1.503757e-02 +Pairwise Locality Aware, 20 leaders: 1.674775e-02 +Pairwise Multileader Locality, 20 leaders: 1.141586e-03 +Nonblocking Multileader, 20 leaders: 5.062844e-02 +Nonblocking Locality Aware, 20 leaders: 2.262428e-01 +Nonblocking Multileader Locality, 20 leaders: 4.954338e-04 +Size 2 +PMPI: 2.325307e-03 +Pairwise: 5.490045e-02 +NonBlocking: 6.485142e-01 +Pairwise Hierarchical: 1.340694e-02 +Pairwise Node Aware: 1.447033e-03 +Nonblocking Hierarchical: 1.048308e-02 +Nonblocking Node Aware: 9.073084e-04 +Pairwise Multileader, 4 leaders: 5.220621e-03 +Pairwise Locality Aware, 4 leaders: 3.770378e-03 +Pairwise Multileader Locality, 4 leaders: 4.010033e-03 +Nonblocking Multileader, 4 leaders: 2.731769e-03 +Nonblocking Locality Aware, 4 leaders: 1.116058e-01 +Nonblocking Multileader Locality, 4 leaders: 2.842595e-03 +Pairwise Multileader, 10 leaders: 8.478415e-03 +Pairwise Locality Aware, 10 leaders: 8.881337e-03 +Pairwise Multileader Locality, 10 leaders: 1.730275e-03 +Nonblocking Multileader, 10 leaders: 6.609934e-02 +Nonblocking Locality Aware, 10 leaders: 1.382320e-01 +Nonblocking Multileader Locality, 10 leaders: 1.082976e-03 +Pairwise Multileader, 20 leaders: 1.639842e-02 +Pairwise Locality Aware, 20 leaders: 1.743608e-02 +Pairwise Multileader Locality, 20 leaders: 1.346984e-03 +Nonblocking Multileader, 20 leaders: 7.007585e-02 +Nonblocking Locality Aware, 20 leaders: 1.396507e-01 +Nonblocking Multileader Locality, 20 leaders: 7.041593e-04 +Size 4 +PMPI: 4.711732e-03 +Pairwise: 8.118875e-02 +NonBlocking: 4.904277e-01 +Pairwise Hierarchical: 2.423709e-02 +Pairwise Node Aware: 1.801560e-03 +Nonblocking Hierarchical: 1.882779e-02 +Nonblocking Node Aware: 1.467578e-03 +Pairwise Multileader, 4 leaders: 7.734259e-03 +Pairwise Locality Aware, 4 leaders: 4.298208e-03 +Pairwise Multileader Locality, 4 leaders: 6.403251e-03 +Nonblocking Multileader, 4 leaders: 5.248053e-03 +Nonblocking Locality Aware, 4 leaders: 1.156411e-01 +Nonblocking Multileader Locality, 4 leaders: 5.017547e-03 +Pairwise Multileader, 10 leaders: 9.898564e-03 +Pairwise Locality Aware, 10 leaders: 9.300625e-03 +Pairwise Multileader Locality, 10 leaders: 3.224400e-03 +Nonblocking Multileader, 10 leaders: 5.661166e-02 +Nonblocking Locality Aware, 10 leaders: 1.388172e-01 +Nonblocking Multileader Locality, 10 leaders: 2.116326e-03 +Pairwise Multileader, 20 leaders: 1.726780e-02 +Pairwise Locality Aware, 20 leaders: 1.699958e-02 +Pairwise Multileader Locality, 20 leaders: 1.822374e-03 +Nonblocking Multileader, 20 leaders: 1.000439e-01 +Nonblocking Locality Aware, 20 leaders: 1.934487e-01 +Nonblocking Multileader Locality, 20 leaders: 4.196455e-02 +Size 8 +PMPI: 7.181479e-03 +Pairwise: 8.077737e-02 +NonBlocking: 8.094280e-01 +Pairwise Hierarchical: 4.917601e-02 +Pairwise Node Aware: 2.711936e-03 +Nonblocking Hierarchical: 3.717256e-02 +Nonblocking Node Aware: 2.830719e-03 +Pairwise Multileader, 4 leaders: 1.203050e-02 +Pairwise Locality Aware, 4 leaders: 4.841770e-03 +Pairwise Multileader Locality, 4 leaders: 1.359750e-02 +Nonblocking Multileader, 4 leaders: 7.581782e-03 +Nonblocking Locality Aware, 4 leaders: 1.384433e-01 +Nonblocking Multileader Locality, 4 leaders: 9.782755e-03 +Pairwise Multileader, 10 leaders: 1.207797e-02 +Pairwise Locality Aware, 10 leaders: 9.807894e-03 +Pairwise Multileader Locality, 10 leaders: 2.235879e-02 +Nonblocking Multileader, 10 leaders: 5.638882e-02 +Nonblocking Locality Aware, 10 leaders: 1.385773e-01 +Nonblocking Multileader Locality, 10 leaders: 3.837043e-03 +Pairwise Multileader, 20 leaders: 1.787261e-02 +Pairwise Locality Aware, 20 leaders: 1.760922e-02 +Pairwise Multileader Locality, 20 leaders: 3.540040e-03 +Nonblocking Multileader, 20 leaders: 6.672029e-02 +Nonblocking Locality Aware, 20 leaders: 2.266052e-01 +Nonblocking Multileader Locality, 20 leaders: 2.745862e-03 +Size 16 +PMPI: 1.375799e-02 +Pairwise: 8.244090e-02 +NonBlocking: 5.462968e-01 +Pairwise Hierarchical: 9.242773e-02 +Pairwise Node Aware: 5.145794e-03 +Nonblocking Hierarchical: 7.010829e-02 +Nonblocking Node Aware: 5.970805e-03 +Pairwise Multileader, 4 leaders: 2.016040e-02 +Pairwise Locality Aware, 4 leaders: 6.076860e-03 +Pairwise Multileader Locality, 4 leaders: 2.705350e-02 +Nonblocking Multileader, 4 leaders: 1.417426e-02 +Nonblocking Locality Aware, 4 leaders: 1.170033e-01 +Nonblocking Multileader Locality, 4 leaders: 1.827618e-02 +Pairwise Multileader, 10 leaders: 1.861255e-02 +Pairwise Locality Aware, 10 leaders: 1.187412e-02 +Pairwise Multileader Locality, 10 leaders: 1.072981e-02 +Nonblocking Multileader, 10 leaders: 9.340128e-03 +Nonblocking Locality Aware, 10 leaders: 1.406266e-01 +Nonblocking Multileader Locality, 10 leaders: 8.487434e-03 +Pairwise Multileader, 20 leaders: 2.146021e-02 +Pairwise Locality Aware, 20 leaders: 1.960645e-02 +Pairwise Multileader Locality, 20 leaders: 6.017942e-03 +Nonblocking Multileader, 20 leaders: 9.801890e-02 +Nonblocking Locality Aware, 20 leaders: 1.633452e-01 +Nonblocking Multileader Locality, 20 leaders: 4.114777e-03 +Size 32 +PMPI: 2.361477e-02 +Pairwise: 8.166834e-02 +NonBlocking: 5.966452e-01 +Pairwise Hierarchical: 1.742164e-01 +Pairwise Node Aware: 9.779019e-03 +Nonblocking Hierarchical: 1.332237e-01 +Nonblocking Node Aware: 1.325962e-02 +Pairwise Multileader, 4 leaders: 5.935182e-02 +Pairwise Locality Aware, 4 leaders: 9.657279e-03 +Pairwise Multileader Locality, 4 leaders: 6.099573e-02 +Nonblocking Multileader, 4 leaders: 4.079961e-02 +Nonblocking Locality Aware, 4 leaders: 8.669811e-02 +Nonblocking Multileader Locality, 4 leaders: 4.336562e-02 +Pairwise Multileader, 10 leaders: 3.205107e-02 +Pairwise Locality Aware, 10 leaders: 1.535298e-02 +Pairwise Multileader Locality, 10 leaders: 2.737857e-02 +Nonblocking Multileader, 10 leaders: 8.121552e-02 +Nonblocking Locality Aware, 10 leaders: 1.411113e-01 +Nonblocking Multileader Locality, 10 leaders: 2.062384e-02 +Pairwise Multileader, 20 leaders: 2.846993e-02 +Pairwise Locality Aware, 20 leaders: 2.312786e-02 +Pairwise Multileader Locality, 20 leaders: 1.187825e-02 +Nonblocking Multileader, 20 leaders: 8.788931e-02 +Nonblocking Locality Aware, 20 leaders: 1.639900e-01 +Nonblocking Multileader Locality, 20 leaders: 1.000725e-02 +Size 64 +PMPI: 4.906214e-02 +Pairwise: 8.952443e-02 +NonBlocking: 3.391413e-01 +Pairwise Hierarchical: 3.495456e-01 +Pairwise Node Aware: 3.366488e-02 +Nonblocking Hierarchical: 1.613672e+00 +Nonblocking Node Aware: 1.818482e-02 +Pairwise Multileader, 4 leaders: 1.070577e-01 +Pairwise Locality Aware, 4 leaders: 1.690163e-02 +Pairwise Multileader Locality, 4 leaders: 1.142580e-01 +Nonblocking Multileader, 4 leaders: 7.644849e-02 +Nonblocking Locality Aware, 4 leaders: 2.841776e-02 +Nonblocking Multileader Locality, 4 leaders: 8.414960e-02 +Pairwise Multileader, 10 leaders: 6.107207e-02 +Pairwise Locality Aware, 10 leaders: 2.420799e-02 +Pairwise Multileader Locality, 10 leaders: 5.745309e-02 +Nonblocking Multileader, 10 leaders: 4.611117e-01 +Nonblocking Locality Aware, 10 leaders: 1.464202e-01 +Nonblocking Multileader Locality, 10 leaders: 4.458645e-02 +Pairwise Multileader, 20 leaders: 4.358550e-02 +Pairwise Locality Aware, 20 leaders: 3.064012e-02 +Pairwise Multileader Locality, 20 leaders: 3.011418e-02 +Nonblocking Multileader, 20 leaders: 4.829025e-02 +Nonblocking Locality Aware, 20 leaders: 1.709314e-01 +Nonblocking Multileader Locality, 20 leaders: 2.197115e-02 +Size 128 +PMPI: 9.911509e-02 +Pairwise: 1.059748e-01 +NonBlocking: 5.040859e-01 +Pairwise Hierarchical: 7.124823e-01 +Pairwise Node Aware: 4.104285e-02 +Nonblocking Hierarchical: 5.522180e-01 +Nonblocking Node Aware: 3.997490e-02 +Pairwise Multileader, 4 leaders: 1.983019e-01 +Pairwise Locality Aware, 4 leaders: 3.707885e-02 +Pairwise Multileader Locality, 4 leaders: 2.223157e-01 +Nonblocking Multileader, 4 leaders: 1.364530e-01 +Nonblocking Locality Aware, 4 leaders: 7.077471e-02 +Nonblocking Multileader Locality, 4 leaders: 1.655670e-01 +Pairwise Multileader, 10 leaders: 1.171447e-01 +Pairwise Locality Aware, 10 leaders: 4.468636e-02 +Pairwise Multileader Locality, 10 leaders: 1.135550e-01 +Nonblocking Multileader, 10 leaders: 3.336201e-01 +Nonblocking Locality Aware, 10 leaders: 1.336622e-01 +Nonblocking Multileader Locality, 10 leaders: 9.065121e-02 +Pairwise Multileader, 20 leaders: 8.412252e-02 +Pairwise Locality Aware, 20 leaders: 5.031126e-02 +Pairwise Multileader Locality, 20 leaders: 6.098787e-02 +Nonblocking Multileader, 20 leaders: 2.467695e-01 +Nonblocking Locality Aware, 20 leaders: 2.307257e-01 +Nonblocking Multileader Locality, 20 leaders: 4.453689e-02 +Size 256 +PMPI: 2.083709e-01 +Pairwise: 1.492603e-01 +NonBlocking: 3.113666e-01 +Pairwise Hierarchical: 1.408455e+00 +Pairwise Node Aware: 7.001621e-02 +Nonblocking Hierarchical: 1.152403e+00 +Nonblocking Node Aware: 5.590255e-02 +Pairwise Multileader, 4 leaders: 3.691181e-01 +Pairwise Locality Aware, 4 leaders: 7.842702e-02 +Pairwise Multileader Locality, 4 leaders: 4.133302e-01 +Nonblocking Multileader, 4 leaders: 2.671303e-01 +Nonblocking Locality Aware, 4 leaders: 1.013537e-01 +Nonblocking Multileader Locality, 4 leaders: 3.132961e-01 +Pairwise Multileader, 10 leaders: 2.414708e-01 +Pairwise Locality Aware, 10 leaders: 8.262851e-02 +Pairwise Multileader Locality, 10 leaders: 2.331951e-01 +Nonblocking Multileader, 10 leaders: 1.530092e-01 +Nonblocking Locality Aware, 10 leaders: 1.312466e-01 +Nonblocking Multileader Locality, 10 leaders: 1.822544e-01 +Pairwise Multileader, 20 leaders: 1.685198e-01 +Pairwise Locality Aware, 20 leaders: 8.780537e-02 +Pairwise Multileader Locality, 20 leaders: 1.357095e-01 +Nonblocking Multileader, 20 leaders: 4.202409e-01 +Nonblocking Locality Aware, 20 leaders: 1.914014e-01 +Nonblocking Multileader Locality, 20 leaders: 1.132714e-01 +Size 512 +PMPI: 6.989250e-01 +Pairwise: 2.283056e-01 +NonBlocking: 6.752088e-01 +Pairwise Hierarchical: 2.763771e+00 +Pairwise Node Aware: 1.383046e-01 +Nonblocking Hierarchical: 2.354167e+00 +Nonblocking Node Aware: 1.088581e-01 +Pairwise Multileader, 4 leaders: 7.792310e-01 +Pairwise Locality Aware, 4 leaders: 1.617648e-01 +Pairwise Multileader Locality, 4 leaders: 8.710527e-01 +Nonblocking Multileader, 4 leaders: 6.100203e-01 +Nonblocking Locality Aware, 4 leaders: 1.378207e-01 +Nonblocking Multileader Locality, 4 leaders: 7.379856e-01 +Pairwise Multileader, 10 leaders: 4.623550e-01 +Pairwise Locality Aware, 10 leaders: 1.844608e-01 +Pairwise Multileader Locality, 10 leaders: 4.659889e-01 +Nonblocking Multileader, 10 leaders: 3.088946e-01 +Nonblocking Locality Aware, 10 leaders: 1.751933e-01 +Nonblocking Multileader Locality, 10 leaders: 3.700855e-01 +Pairwise Multileader, 20 leaders: 3.415160e-01 +Pairwise Locality Aware, 20 leaders: 2.051929e-01 +Pairwise Multileader Locality, 20 leaders: 2.657863e-01 +Nonblocking Multileader, 20 leaders: 1.276064e+00 +Nonblocking Locality Aware, 20 leaders: 2.911612e-01 +Nonblocking Multileader Locality, 20 leaders: 2.251196e-01 +Test 2 Running Alltoall Benchmark on 64 nodes and 6400 tasks +Size 1 +PMPI: 1.851493e-03 +Pairwise: 5.235001e-02 +NonBlocking: 8.059248e-01 +Pairwise Hierarchical: 5.464492e-03 +Pairwise Node Aware: 1.306297e-03 +Nonblocking Hierarchical: 4.096370e-03 +Nonblocking Node Aware: 7.577382e-04 +Pairwise Multileader, 4 leaders: 4.306581e-03 +Pairwise Locality Aware, 4 leaders: 3.814747e-03 +Pairwise Multileader Locality, 4 leaders: 2.189717e-03 +Nonblocking Multileader, 4 leaders: 1.630621e-03 +Nonblocking Locality Aware, 4 leaders: 9.218737e-02 +Nonblocking Multileader Locality, 4 leaders: 1.329712e-03 +Pairwise Multileader, 10 leaders: 8.248220e-03 +Pairwise Locality Aware, 10 leaders: 8.760938e-03 +Pairwise Multileader Locality, 10 leaders: 1.316474e-03 +Nonblocking Multileader, 10 leaders: 3.869900e-02 +Nonblocking Locality Aware, 10 leaders: 1.384080e-01 +Nonblocking Multileader Locality, 10 leaders: 6.936062e-04 +Pairwise Multileader, 20 leaders: 1.563730e-02 +Pairwise Locality Aware, 20 leaders: 1.710000e-02 +Pairwise Multileader Locality, 20 leaders: 1.122645e-03 +Nonblocking Multileader, 20 leaders: 3.836690e-02 +Nonblocking Locality Aware, 20 leaders: 2.074161e-01 +Nonblocking Multileader Locality, 20 leaders: 4.681823e-04 +Size 2 +PMPI: 2.333698e-03 +Pairwise: 5.610690e-02 +NonBlocking: 8.078728e-01 +Pairwise Hierarchical: 1.345484e-02 +Pairwise Node Aware: 1.425581e-03 +Nonblocking Hierarchical: 1.056588e-02 +Nonblocking Node Aware: 8.749962e-04 +Pairwise Multileader, 4 leaders: 5.359840e-03 +Pairwise Locality Aware, 4 leaders: 3.674636e-03 +Pairwise Multileader Locality, 4 leaders: 3.986806e-03 +Nonblocking Multileader, 4 leaders: 2.397645e-03 +Nonblocking Locality Aware, 4 leaders: 9.230954e-02 +Nonblocking Multileader Locality, 4 leaders: 2.876615e-03 +Pairwise Multileader, 10 leaders: 8.597154e-03 +Pairwise Locality Aware, 10 leaders: 9.240350e-03 +Pairwise Multileader Locality, 10 leaders: 1.685798e-03 +Nonblocking Multileader, 10 leaders: 6.258763e-02 +Nonblocking Locality Aware, 10 leaders: 1.384699e-01 +Nonblocking Multileader Locality, 10 leaders: 1.040905e-03 +Pairwise Multileader, 20 leaders: 1.638757e-02 +Pairwise Locality Aware, 20 leaders: 1.760887e-02 +Pairwise Multileader Locality, 20 leaders: 1.379739e-03 +Nonblocking Multileader, 20 leaders: 8.732994e-02 +Nonblocking Locality Aware, 20 leaders: 2.258162e-01 +Nonblocking Multileader Locality, 20 leaders: 7.830618e-04 +Size 4 +PMPI: 4.778748e-03 +Pairwise: 8.423844e-02 +NonBlocking: 4.906235e-01 +Pairwise Hierarchical: 2.399708e-02 +Pairwise Node Aware: 1.755534e-03 +Nonblocking Hierarchical: 1.797242e-02 +Nonblocking Node Aware: 1.427412e-03 +Pairwise Multileader, 4 leaders: 7.776812e-03 +Pairwise Locality Aware, 4 leaders: 4.052832e-03 +Pairwise Multileader Locality, 4 leaders: 6.477391e-03 +Nonblocking Multileader, 4 leaders: 4.410559e-03 +Nonblocking Locality Aware, 4 leaders: 1.154515e-01 +Nonblocking Multileader Locality, 4 leaders: 5.077670e-03 +Pairwise Multileader, 10 leaders: 9.612206e-03 +Pairwise Locality Aware, 10 leaders: 9.535034e-03 +Pairwise Multileader Locality, 10 leaders: 3.143633e-03 +Nonblocking Multileader, 10 leaders: 2.159962e-03 +Nonblocking Locality Aware, 10 leaders: 1.386187e-01 +Nonblocking Multileader Locality, 10 leaders: 2.302316e-03 +Pairwise Multileader, 20 leaders: 1.644004e-02 +Pairwise Locality Aware, 20 leaders: 1.749672e-02 +Pairwise Multileader Locality, 20 leaders: 1.992714e-03 +Nonblocking Multileader, 20 leaders: 9.977381e-02 +Nonblocking Locality Aware, 20 leaders: 1.623647e-01 +Nonblocking Multileader Locality, 20 leaders: 1.579994e-03 +Size 8 +PMPI: 7.055813e-03 +Pairwise: 8.112340e-02 +NonBlocking: 4.923280e-01 +Pairwise Hierarchical: 4.816998e-02 +Pairwise Node Aware: 2.601390e-03 +Nonblocking Hierarchical: 3.769166e-02 +Nonblocking Node Aware: 3.205902e-03 +Pairwise Multileader, 4 leaders: 1.232744e-02 +Pairwise Locality Aware, 4 leaders: 5.318039e-03 +Pairwise Multileader Locality, 4 leaders: 1.435621e-02 +Nonblocking Multileader, 4 leaders: 7.645211e-03 +Nonblocking Locality Aware, 4 leaders: 1.163106e-01 +Nonblocking Multileader Locality, 4 leaders: 9.840109e-03 +Pairwise Multileader, 10 leaders: 1.170880e-02 +Pairwise Locality Aware, 10 leaders: 1.035963e-02 +Pairwise Multileader Locality, 10 leaders: 5.308143e-03 +Nonblocking Multileader, 10 leaders: 5.970279e-02 +Nonblocking Locality Aware, 10 leaders: 1.617378e-01 +Nonblocking Multileader Locality, 10 leaders: 3.459384e-03 +Pairwise Multileader, 20 leaders: 1.949540e-02 +Pairwise Locality Aware, 20 leaders: 1.886419e-02 +Pairwise Multileader Locality, 20 leaders: 3.576414e-03 +Nonblocking Multileader, 20 leaders: 8.830839e-02 +Nonblocking Locality Aware, 20 leaders: 1.402847e-01 +Nonblocking Multileader Locality, 20 leaders: 2.790663e-03 +Size 16 +PMPI: 1.385707e-02 +Pairwise: 8.456514e-02 +NonBlocking: 4.908888e-01 +Pairwise Hierarchical: 9.433122e-02 +Pairwise Node Aware: 5.132059e-03 +Nonblocking Hierarchical: 7.053766e-02 +Nonblocking Node Aware: 6.077073e-03 +Pairwise Multileader, 4 leaders: 2.040216e-02 +Pairwise Locality Aware, 4 leaders: 7.100864e-03 +Pairwise Multileader Locality, 4 leaders: 2.555985e-02 +Nonblocking Multileader, 4 leaders: 1.452660e-02 +Nonblocking Locality Aware, 4 leaders: 1.413063e-01 +Nonblocking Multileader Locality, 4 leaders: 1.844244e-02 +Pairwise Multileader, 10 leaders: 1.941217e-02 +Pairwise Locality Aware, 10 leaders: 1.180592e-02 +Pairwise Multileader Locality, 10 leaders: 1.080711e-02 +Nonblocking Multileader, 10 leaders: 1.736097e-02 +Nonblocking Locality Aware, 10 leaders: 1.403167e-01 +Nonblocking Multileader Locality, 10 leaders: 8.331841e-03 +Pairwise Multileader, 20 leaders: 2.107302e-02 +Pairwise Locality Aware, 20 leaders: 1.873744e-02 +Pairwise Multileader Locality, 20 leaders: 6.348731e-03 +Nonblocking Multileader, 20 leaders: 8.423873e-02 +Nonblocking Locality Aware, 20 leaders: 1.405912e-01 +Nonblocking Multileader Locality, 20 leaders: 4.457675e-03 +Size 32 +PMPI: 2.416094e-02 +Pairwise: 8.695843e-02 +NonBlocking: 4.910589e-01 +Pairwise Hierarchical: 1.742421e-01 +Pairwise Node Aware: 9.859046e-03 +Nonblocking Hierarchical: 1.313992e-01 +Nonblocking Node Aware: 1.484489e-02 +Pairwise Multileader, 4 leaders: 5.862632e-02 +Pairwise Locality Aware, 4 leaders: 9.747648e-03 +Pairwise Multileader Locality, 4 leaders: 5.890930e-02 +Nonblocking Multileader, 4 leaders: 4.070113e-02 +Nonblocking Locality Aware, 4 leaders: 8.700265e-02 +Nonblocking Multileader Locality, 4 leaders: 4.298428e-02 +Pairwise Multileader, 10 leaders: 3.200947e-02 +Pairwise Locality Aware, 10 leaders: 1.592597e-02 +Pairwise Multileader Locality, 10 leaders: 2.712268e-02 +Nonblocking Multileader, 10 leaders: 1.008472e-01 +Nonblocking Locality Aware, 10 leaders: 1.668443e-01 +Nonblocking Multileader Locality, 10 leaders: 2.003482e-02 +Pairwise Multileader, 20 leaders: 2.960717e-02 +Pairwise Locality Aware, 20 leaders: 2.606234e-02 +Pairwise Multileader Locality, 20 leaders: 1.200014e-02 +Nonblocking Multileader, 20 leaders: 1.031404e-01 +Nonblocking Locality Aware, 20 leaders: 1.644022e-01 +Nonblocking Multileader Locality, 20 leaders: 1.051269e-02 +Size 64 +PMPI: 4.820498e-02 +Pairwise: 8.695722e-02 +NonBlocking: 5.496594e-01 +Pairwise Hierarchical: 3.403228e-01 +Pairwise Node Aware: 2.039348e-02 +Nonblocking Hierarchical: 2.629425e-01 +Nonblocking Node Aware: 1.836962e-02 +Pairwise Multileader, 4 leaders: 1.075032e-01 +Pairwise Locality Aware, 4 leaders: 1.812896e-02 +Pairwise Multileader Locality, 4 leaders: 1.163284e-01 +Nonblocking Multileader, 4 leaders: 7.571621e-02 +Nonblocking Locality Aware, 4 leaders: 3.314715e-02 +Nonblocking Multileader Locality, 4 leaders: 8.313390e-02 +Pairwise Multileader, 10 leaders: 6.304936e-02 +Pairwise Locality Aware, 10 leaders: 2.395870e-02 +Pairwise Multileader Locality, 10 leaders: 5.784860e-02 +Nonblocking Multileader, 10 leaders: 4.586987e-01 +Nonblocking Locality Aware, 10 leaders: 1.724876e-01 +Nonblocking Multileader Locality, 10 leaders: 4.404113e-02 +Pairwise Multileader, 20 leaders: 4.711148e-02 +Pairwise Locality Aware, 20 leaders: 3.189150e-02 +Pairwise Multileader Locality, 20 leaders: 2.959738e-02 +Nonblocking Multileader, 20 leaders: 8.958739e-02 +Nonblocking Locality Aware, 20 leaders: 1.770389e-01 +Nonblocking Multileader Locality, 20 leaders: 2.188624e-02 +Size 128 +PMPI: 1.006937e-01 +Pairwise: 1.032609e-01 +NonBlocking: 3.504852e-01 +Pairwise Hierarchical: 7.145275e-01 +Pairwise Node Aware: 4.099076e-02 +Nonblocking Hierarchical: 5.471699e-01 +Nonblocking Node Aware: 3.936699e-02 +Pairwise Multileader, 4 leaders: 1.997150e-01 +Pairwise Locality Aware, 4 leaders: 4.390645e-02 +Pairwise Multileader Locality, 4 leaders: 2.184377e-01 +Nonblocking Multileader, 4 leaders: 1.370781e-01 +Nonblocking Locality Aware, 4 leaders: 8.699820e-02 +Nonblocking Multileader Locality, 4 leaders: 1.634784e-01 +Pairwise Multileader, 10 leaders: 1.210056e-01 +Pairwise Locality Aware, 10 leaders: 4.695643e-02 +Pairwise Multileader Locality, 10 leaders: 1.143306e-01 +Nonblocking Multileader, 10 leaders: 4.417529e-01 +Nonblocking Locality Aware, 10 leaders: 1.553291e-01 +Nonblocking Multileader Locality, 10 leaders: 9.026833e-02 +Pairwise Multileader, 20 leaders: 8.225757e-02 +Pairwise Locality Aware, 20 leaders: 5.487302e-02 +Pairwise Multileader Locality, 20 leaders: 6.112441e-02 +Nonblocking Multileader, 20 leaders: 3.464533e-01 +Nonblocking Locality Aware, 20 leaders: 1.837456e-01 +Nonblocking Multileader Locality, 20 leaders: 4.512907e-02 +Size 256 +PMPI: 2.091428e-01 +Pairwise: 1.535031e-01 +NonBlocking: 3.001920e-01 +Pairwise Hierarchical: 1.442236e+00 +Pairwise Node Aware: 6.892858e-02 +Nonblocking Hierarchical: 1.151875e+00 +Nonblocking Node Aware: 5.520062e-02 +Pairwise Multileader, 4 leaders: 3.605513e-01 +Pairwise Locality Aware, 4 leaders: 8.025547e-02 +Pairwise Multileader Locality, 4 leaders: 4.120270e-01 +Nonblocking Multileader, 4 leaders: 2.709090e-01 +Nonblocking Locality Aware, 4 leaders: 6.709414e-02 +Nonblocking Multileader Locality, 4 leaders: 3.121524e-01 +Pairwise Multileader, 10 leaders: 2.553795e-01 +Pairwise Locality Aware, 10 leaders: 9.048265e-02 +Pairwise Multileader Locality, 10 leaders: 2.235234e-01 +Nonblocking Multileader, 10 leaders: 1.528169e-01 +Nonblocking Locality Aware, 10 leaders: 1.341493e-01 +Nonblocking Multileader Locality, 10 leaders: 1.815736e-01 +Pairwise Multileader, 20 leaders: 1.671037e-01 +Pairwise Locality Aware, 20 leaders: 8.383914e-02 +Pairwise Multileader Locality, 20 leaders: 1.358214e-01 +Nonblocking Multileader, 20 leaders: 5.072989e-01 +Nonblocking Locality Aware, 20 leaders: 1.997863e-01 +Nonblocking Multileader Locality, 20 leaders: 1.116469e-01 +Size 512 +PMPI: 1.019008e+00 +Pairwise: 2.334424e-01 +NonBlocking: 6.757430e-01 +Pairwise Hierarchical: 2.782784e+00 +Pairwise Node Aware: 1.343192e-01 +Nonblocking Hierarchical: 2.413392e+00 +Nonblocking Node Aware: 1.083205e-01 +Pairwise Multileader, 4 leaders: 7.568302e-01 +Pairwise Locality Aware, 4 leaders: 1.526351e-01 +Pairwise Multileader Locality, 4 leaders: 8.829581e-01 +Nonblocking Multileader, 4 leaders: 5.960709e-01 +Nonblocking Locality Aware, 4 leaders: 2.253518e-01 +Nonblocking Multileader Locality, 4 leaders: 7.044989e-01 +Pairwise Multileader, 10 leaders: 5.856366e-01 +Pairwise Locality Aware, 10 leaders: 1.893934e-01 +Pairwise Multileader Locality, 10 leaders: 4.660838e-01 +Nonblocking Multileader, 10 leaders: 3.043075e-01 +Nonblocking Locality Aware, 10 leaders: 1.774393e-01 +Nonblocking Multileader Locality, 10 leaders: 3.590061e-01 +Pairwise Multileader, 20 leaders: 3.354487e-01 +Pairwise Locality Aware, 20 leaders: 2.042969e-01 +Pairwise Multileader Locality, 20 leaders: 2.665898e-01 +Nonblocking Multileader, 20 leaders: 5.088857e+00 +Nonblocking Locality Aware, 20 leaders: 2.485815e-01 +Nonblocking Multileader Locality, 20 leaders: 2.263458e-01 +Test 3 Running Alltoall Benchmark on 64 nodes and 6400 tasks +Size 1 +PMPI: 1.945550e-03 +Pairwise: 4.997327e-02 +NonBlocking: 4.973290e-01 +Pairwise Hierarchical: 5.430144e-03 +Pairwise Node Aware: 1.296892e-03 +Nonblocking Hierarchical: 4.164505e-03 +Nonblocking Node Aware: 7.234483e-04 +Pairwise Multileader, 4 leaders: 4.043859e-03 +Pairwise Locality Aware, 4 leaders: 3.515898e-03 +Pairwise Multileader Locality, 4 leaders: 2.204173e-03 +Nonblocking Multileader, 4 leaders: 1.444198e-03 +Nonblocking Locality Aware, 4 leaders: 1.379902e-01 +Nonblocking Multileader Locality, 4 leaders: 1.365564e-03 +Pairwise Multileader, 10 leaders: 6.985980e-03 +Pairwise Locality Aware, 10 leaders: 7.717306e-03 +Pairwise Multileader Locality, 10 leaders: 1.414671e-03 +Nonblocking Multileader, 10 leaders: 3.506082e-02 +Nonblocking Locality Aware, 10 leaders: 1.610931e-01 +Nonblocking Multileader Locality, 10 leaders: 7.423362e-04 +Pairwise Multileader, 20 leaders: 1.451761e-02 +Pairwise Locality Aware, 20 leaders: 1.493902e-02 +Pairwise Multileader Locality, 20 leaders: 1.106878e-03 +Nonblocking Multileader, 20 leaders: 1.149622e-01 +Nonblocking Locality Aware, 20 leaders: 1.947503e-01 +Nonblocking Multileader Locality, 20 leaders: 4.660729e-04 +Size 2 +PMPI: 2.305974e-03 +Pairwise: 5.444261e-02 +NonBlocking: 4.919423e-01 +Pairwise Hierarchical: 1.290034e-02 +Pairwise Node Aware: 1.384150e-03 +Nonblocking Hierarchical: 1.017617e-02 +Nonblocking Node Aware: 8.812535e-04 +Pairwise Multileader, 4 leaders: 5.180372e-03 +Pairwise Locality Aware, 4 leaders: 3.586536e-03 +Pairwise Multileader Locality, 4 leaders: 3.965031e-03 +Nonblocking Multileader, 4 leaders: 2.428396e-03 +Nonblocking Locality Aware, 4 leaders: 1.154279e-01 +Nonblocking Multileader Locality, 4 leaders: 2.844730e-03 +Pairwise Multileader, 10 leaders: 7.101842e-03 +Pairwise Locality Aware, 10 leaders: 7.975008e-03 +Pairwise Multileader Locality, 10 leaders: 1.652316e-03 +Nonblocking Multileader, 10 leaders: 6.429055e-02 +Nonblocking Locality Aware, 10 leaders: 1.612976e-01 +Nonblocking Multileader Locality, 10 leaders: 1.038493e-03 +Pairwise Multileader, 20 leaders: 1.548748e-02 +Pairwise Locality Aware, 20 leaders: 1.610808e-02 +Pairwise Multileader Locality, 20 leaders: 1.343248e-03 +Nonblocking Multileader, 20 leaders: 9.958401e-02 +Nonblocking Locality Aware, 20 leaders: 1.396049e-01 +Nonblocking Multileader Locality, 20 leaders: 7.838157e-04 +Size 4 +PMPI: 4.682045e-03 +Pairwise: 7.659069e-02 +NonBlocking: 5.959645e-01 +Pairwise Hierarchical: 2.379709e-02 +Pairwise Node Aware: 1.843328e-03 +Nonblocking Hierarchical: 1.833028e-02 +Nonblocking Node Aware: 1.539966e-03 +Pairwise Multileader, 4 leaders: 7.844073e-03 +Pairwise Locality Aware, 4 leaders: 4.382967e-03 +Pairwise Multileader Locality, 4 leaders: 6.523069e-03 +Nonblocking Multileader, 4 leaders: 4.486832e-03 +Nonblocking Locality Aware, 4 leaders: 1.158649e-01 +Nonblocking Multileader Locality, 4 leaders: 5.093438e-03 +Pairwise Multileader, 10 leaders: 9.318136e-03 +Pairwise Locality Aware, 10 leaders: 8.243985e-03 +Pairwise Multileader Locality, 10 leaders: 3.185649e-03 +Nonblocking Multileader, 10 leaders: 3.148905e-02 +Nonblocking Locality Aware, 10 leaders: 1.387344e-01 +Nonblocking Multileader Locality, 10 leaders: 2.208979e-03 +Pairwise Multileader, 20 leaders: 1.496956e-02 +Pairwise Locality Aware, 20 leaders: 1.507340e-02 +Pairwise Multileader Locality, 20 leaders: 1.849747e-03 +Nonblocking Multileader, 20 leaders: 7.549196e-02 +Nonblocking Locality Aware, 20 leaders: 1.849220e-01 +Nonblocking Multileader Locality, 20 leaders: 1.596550e-03 +Size 8 +PMPI: 7.171621e-03 +Pairwise: 7.241140e-02 +NonBlocking: 4.913266e-01 +Pairwise Hierarchical: 4.904272e-02 +Pairwise Node Aware: 2.610268e-03 +Nonblocking Hierarchical: 3.765025e-02 +Nonblocking Node Aware: 2.837165e-03 +Pairwise Multileader, 4 leaders: 1.188056e-02 +Pairwise Locality Aware, 4 leaders: 4.686218e-03 +Pairwise Multileader Locality, 4 leaders: 1.391581e-02 +Nonblocking Multileader, 4 leaders: 7.678408e-03 +Nonblocking Locality Aware, 4 leaders: 1.162603e-01 +Nonblocking Multileader Locality, 4 leaders: 9.977388e-03 +Pairwise Multileader, 10 leaders: 1.133667e-02 +Pairwise Locality Aware, 10 leaders: 9.387209e-03 +Pairwise Multileader Locality, 10 leaders: 5.256150e-03 +Nonblocking Multileader, 10 leaders: 1.463507e-02 +Nonblocking Locality Aware, 10 leaders: 1.390789e-01 +Nonblocking Multileader Locality, 10 leaders: 3.893841e-03 +Pairwise Multileader, 20 leaders: 1.608041e-02 +Pairwise Locality Aware, 20 leaders: 1.654744e-02 +Pairwise Multileader Locality, 20 leaders: 3.526749e-03 +Nonblocking Multileader, 20 leaders: 8.826061e-02 +Nonblocking Locality Aware, 20 leaders: 2.312276e-01 +Nonblocking Multileader Locality, 20 leaders: 2.729033e-03 +Size 16 +PMPI: 1.417419e-02 +Pairwise: 7.260379e-02 +NonBlocking: 7.575936e-01 +Pairwise Hierarchical: 9.477401e-02 +Pairwise Node Aware: 4.598519e-03 +Nonblocking Hierarchical: 7.256111e-02 +Nonblocking Node Aware: 5.409148e-03 +Pairwise Multileader, 4 leaders: 2.123695e-02 +Pairwise Locality Aware, 4 leaders: 6.153875e-03 +Pairwise Multileader Locality, 4 leaders: 2.827206e-02 +Nonblocking Multileader, 4 leaders: 2.575593e-02 +Nonblocking Locality Aware, 4 leaders: 1.396366e-01 +Nonblocking Multileader Locality, 4 leaders: 1.888556e-02 +Pairwise Multileader, 10 leaders: 1.847545e-02 +Pairwise Locality Aware, 10 leaders: 1.160467e-02 +Pairwise Multileader Locality, 10 leaders: 1.083157e-02 +Nonblocking Multileader, 10 leaders: 9.034277e-03 +Nonblocking Locality Aware, 10 leaders: 1.402808e-01 +Nonblocking Multileader Locality, 10 leaders: 8.472300e-03 +Pairwise Multileader, 20 leaders: 1.954047e-02 +Pairwise Locality Aware, 20 leaders: 1.869110e-02 +Pairwise Multileader Locality, 20 leaders: 6.343623e-03 +Nonblocking Multileader, 20 leaders: 9.686023e-02 +Nonblocking Locality Aware, 20 leaders: 1.412437e-01 +Nonblocking Multileader Locality, 20 leaders: 5.510085e-03 +Size 32 +PMPI: 2.371763e-02 +Pairwise: 7.536685e-02 +NonBlocking: 4.901570e-01 +Pairwise Hierarchical: 1.784644e-01 +Pairwise Node Aware: 9.090927e-03 +Nonblocking Hierarchical: 1.338316e-01 +Nonblocking Node Aware: 1.292610e-02 +Pairwise Multileader, 4 leaders: 5.905250e-02 +Pairwise Locality Aware, 4 leaders: 1.081753e-02 +Pairwise Multileader Locality, 4 leaders: 6.059819e-02 +Nonblocking Multileader, 4 leaders: 4.116467e-02 +Nonblocking Locality Aware, 4 leaders: 1.160397e-01 +Nonblocking Multileader Locality, 4 leaders: 4.402191e-02 +Pairwise Multileader, 10 leaders: 3.092860e-02 +Pairwise Locality Aware, 10 leaders: 1.551345e-02 +Pairwise Multileader Locality, 10 leaders: 2.722979e-02 +Nonblocking Multileader, 10 leaders: 7.470007e-02 +Nonblocking Locality Aware, 10 leaders: 1.669354e-01 +Nonblocking Multileader Locality, 10 leaders: 2.024469e-02 +Pairwise Multileader, 20 leaders: 2.737507e-02 +Pairwise Locality Aware, 20 leaders: 2.185684e-02 +Pairwise Multileader Locality, 20 leaders: 1.220335e-02 +Nonblocking Multileader, 20 leaders: 1.002862e-01 +Nonblocking Locality Aware, 20 leaders: 1.385452e-01 +Nonblocking Multileader Locality, 20 leaders: 1.036667e-02 +Size 64 +PMPI: 4.738778e-02 +Pairwise: 8.178790e-02 +NonBlocking: 4.375780e-01 +Pairwise Hierarchical: 3.447070e-01 +Pairwise Node Aware: 2.062729e-02 +Nonblocking Hierarchical: 2.592447e-01 +Nonblocking Node Aware: 1.841435e-02 +Pairwise Multileader, 4 leaders: 1.083001e-01 +Pairwise Locality Aware, 4 leaders: 1.936219e-02 +Pairwise Multileader Locality, 4 leaders: 1.148065e-01 +Nonblocking Multileader, 4 leaders: 7.411383e-02 +Nonblocking Locality Aware, 4 leaders: 6.551931e-02 +Nonblocking Multileader Locality, 4 leaders: 8.297422e-02 +Pairwise Multileader, 10 leaders: 6.835252e-02 +Pairwise Locality Aware, 10 leaders: 2.201268e-02 +Pairwise Multileader Locality, 10 leaders: 5.710974e-02 +Nonblocking Multileader, 10 leaders: 4.562224e-01 +Nonblocking Locality Aware, 10 leaders: 1.723455e-01 +Nonblocking Multileader Locality, 10 leaders: 4.422683e-02 +Pairwise Multileader, 20 leaders: 4.387995e-02 +Pairwise Locality Aware, 20 leaders: 2.960016e-02 +Pairwise Multileader Locality, 20 leaders: 2.960397e-02 +Nonblocking Multileader, 20 leaders: 8.059836e-02 +Nonblocking Locality Aware, 20 leaders: 1.676726e-01 +Nonblocking Multileader Locality, 20 leaders: 2.132419e-02 +Size 128 +PMPI: 9.999311e-02 +Pairwise: 1.034839e-01 +NonBlocking: 3.452280e-01 +Pairwise Hierarchical: 7.211232e-01 +Pairwise Node Aware: 4.050688e-02 +Nonblocking Hierarchical: 5.539440e-01 +Nonblocking Node Aware: 3.955863e-02 +Pairwise Multileader, 4 leaders: 1.969136e-01 +Pairwise Locality Aware, 4 leaders: 3.693361e-02 +Pairwise Multileader Locality, 4 leaders: 2.185528e-01 +Nonblocking Multileader, 4 leaders: 1.381336e-01 +Nonblocking Locality Aware, 4 leaders: 7.169265e-02 +Nonblocking Multileader Locality, 4 leaders: 1.631716e-01 +Pairwise Multileader, 10 leaders: 1.240790e-01 +Pairwise Locality Aware, 10 leaders: 4.230511e-02 +Pairwise Multileader Locality, 10 leaders: 1.186585e-01 +Nonblocking Multileader, 10 leaders: 4.212922e-01 +Nonblocking Locality Aware, 10 leaders: 1.344109e-01 +Nonblocking Multileader Locality, 10 leaders: 9.028968e-02 +Pairwise Multileader, 20 leaders: 8.018011e-02 +Pairwise Locality Aware, 20 leaders: 4.841465e-02 +Pairwise Multileader Locality, 20 leaders: 6.221241e-02 +Nonblocking Multileader, 20 leaders: 3.452729e-01 +Nonblocking Locality Aware, 20 leaders: 1.873276e-01 +Nonblocking Multileader Locality, 20 leaders: 4.497656e-02 +Size 256 +PMPI: 2.085427e-01 +Pairwise: 1.379193e-01 +NonBlocking: 3.079389e-01 +Pairwise Hierarchical: 1.533736e+00 +Pairwise Node Aware: 6.994584e-02 +Nonblocking Hierarchical: 1.160665e+00 +Nonblocking Node Aware: 5.560811e-02 +Pairwise Multileader, 4 leaders: 3.649057e-01 +Pairwise Locality Aware, 4 leaders: 8.639721e-02 +Pairwise Multileader Locality, 4 leaders: 4.135800e-01 +Nonblocking Multileader, 4 leaders: 2.655818e-01 +Nonblocking Locality Aware, 4 leaders: 7.151663e-02 +Nonblocking Multileader Locality, 4 leaders: 3.130119e-01 +Pairwise Multileader, 10 leaders: 3.370647e-01 +Pairwise Locality Aware, 10 leaders: 8.597140e-02 +Pairwise Multileader Locality, 10 leaders: 2.342322e-01 +Nonblocking Multileader, 10 leaders: 1.530385e-01 +Nonblocking Locality Aware, 10 leaders: 1.552603e-01 +Nonblocking Multileader Locality, 10 leaders: 1.803842e-01 +Pairwise Multileader, 20 leaders: 1.906485e-01 +Pairwise Locality Aware, 20 leaders: 9.818753e-02 +Pairwise Multileader Locality, 20 leaders: 1.353524e-01 +Nonblocking Multileader, 20 leaders: 5.065689e-01 +Nonblocking Locality Aware, 20 leaders: 1.649271e-01 +Nonblocking Multileader Locality, 20 leaders: 1.131708e-01 +Size 512 +PMPI: 1.455325e+00 diff --git a/runscripts/dane_results/alltoall_N8.2373827.err b/runscripts/dane_results/alltoall_N8.2373827.err new file mode 100644 index 000000000..143407f58 --- /dev/null +++ b/runscripts/dane_results/alltoall_N8.2373827.err @@ -0,0 +1,200 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2373827.0 ON dane544 CANCELLED AT 2025-04-29T17:56:53 *** +slurmstepd: error: *** STEP 2373827.0 ON dane544 CANCELLED AT 2025-04-29T17:56:53 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1043: tasks 500-599: Killed +srun: error: dane660: tasks 100-199: Killed +srun: error: dane1390: tasks 700-799: Killed +srun: error: dane1348: tasks 600-699: Killed +srun: error: dane881: tasks 400-499: Killed +srun: error: dane784: tasks 300-399: Killed +srun: error: dane695: tasks 200-299: Killed +srun: error: dane544: tasks 0-99: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2373827.1 ON dane544 CANCELLED AT 2025-04-29T18:05:57 *** +slurmstepd: error: *** STEP 2373827.1 ON dane544 CANCELLED AT 2025-04-29T18:05:57 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1043: tasks 500-599: Killed +srun: error: dane660: tasks 100-199: Killed +srun: error: dane1390: tasks 700-799: Killed +srun: error: dane1348: tasks 600-699: Killed +srun: error: dane881: tasks 400-499: Killed +srun: error: dane784: tasks 300-399: Killed +srun: error: dane695: tasks 200-299: Killed +srun: error: dane544: tasks 0-99: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2373827.2 ON dane544 CANCELLED AT 2025-04-29T18:14:37 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1043: tasks 500-599: Killed +srun: error: dane660: tasks 100-199: Killed +srun: error: dane1390: tasks 700-799: Killed +srun: error: dane1348: tasks 600-699: Killed +srun: error: dane881: tasks 400-499: Killed +srun: error: dane784: tasks 300-399: Killed +srun: error: dane695: tasks 200-299: Killed +srun: error: dane544: tasks 0-99: Killed diff --git a/runscripts/dane_results/alltoall_N8.2373827.out b/runscripts/dane_results/alltoall_N8.2373827.out new file mode 100644 index 000000000..7f0fe3f90 --- /dev/null +++ b/runscripts/dane_results/alltoall_N8.2373827.out @@ -0,0 +1,1029 @@ +Test 1 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.375547e-04 +Pairwise: 2.161964e-03 +NonBlocking: 7.260893e-04 +Pairwise Hierarchical: 5.821792e-04 +Pairwise Node Aware: 2.775803e-04 +Nonblocking Hierarchical: 5.290669e-04 +Nonblocking Node Aware: 2.172801e-04 +Pairwise Multileader, 4 leaders: 3.041654e-04 +Pairwise Locality Aware, 4 leaders: 2.354903e-04 +Pairwise Multileader Locality, 4 leaders: 2.247317e-04 +Nonblocking Multileader, 4 leaders: 1.538970e-04 +Nonblocking Locality Aware, 4 leaders: 1.077400e-04 +Nonblocking Multileader Locality, 4 leaders: 1.788876e-04 +Pairwise Multileader, 10 leaders: 4.056476e-04 +Pairwise Locality Aware, 10 leaders: 4.893347e-04 +Pairwise Multileader Locality, 10 leaders: 1.281277e-04 +Nonblocking Multileader, 10 leaders: 1.149915e-04 +Nonblocking Locality Aware, 10 leaders: 1.941169e-04 +Nonblocking Multileader Locality, 10 leaders: 1.004777e-04 +Pairwise Multileader, 20 leaders: 8.680020e-04 +Pairwise Locality Aware, 20 leaders: 9.505851e-04 +Pairwise Multileader Locality, 20 leaders: 1.267836e-04 +Nonblocking Multileader, 20 leaders: 2.156525e-04 +Nonblocking Locality Aware, 20 leaders: 2.492938e-02 +Nonblocking Multileader Locality, 20 leaders: 9.047385e-05 +Size 2 +PMPI: 1.876481e-04 +Pairwise: 2.002784e-03 +NonBlocking: 7.307069e-04 +Pairwise Hierarchical: 1.214906e-03 +Pairwise Node Aware: 2.837756e-04 +Nonblocking Hierarchical: 1.068736e-03 +Nonblocking Node Aware: 2.163843e-04 +Pairwise Multileader, 4 leaders: 4.088811e-04 +Pairwise Locality Aware, 4 leaders: 2.635238e-04 +Pairwise Multileader Locality, 4 leaders: 4.080689e-04 +Nonblocking Multileader, 4 leaders: 2.655046e-04 +Nonblocking Locality Aware, 4 leaders: 2.019725e-04 +Nonblocking Multileader Locality, 4 leaders: 3.244474e-04 +Pairwise Multileader, 10 leaders: 4.387305e-04 +Pairwise Locality Aware, 10 leaders: 5.137763e-04 +Pairwise Multileader Locality, 10 leaders: 1.707060e-04 +Nonblocking Multileader, 10 leaders: 1.501778e-04 +Nonblocking Locality Aware, 10 leaders: 3.145841e-04 +Nonblocking Multileader Locality, 10 leaders: 1.371027e-04 +Pairwise Multileader, 20 leaders: 9.125380e-04 +Pairwise Locality Aware, 20 leaders: 8.731021e-04 +Pairwise Multileader Locality, 20 leaders: 1.537782e-04 +Nonblocking Multileader, 20 leaders: 9.332189e-04 +Nonblocking Locality Aware, 20 leaders: 3.038296e-03 +Nonblocking Multileader Locality, 20 leaders: 1.054462e-04 +Size 4 +PMPI: 3.180679e-04 +Pairwise: 4.107098e-03 +NonBlocking: 1.150980e-01 +Pairwise Hierarchical: 1.929119e-03 +Pairwise Node Aware: 3.949435e-04 +Nonblocking Hierarchical: 1.710478e-03 +Nonblocking Node Aware: 2.732098e-04 +Pairwise Multileader, 4 leaders: 6.257736e-04 +Pairwise Locality Aware, 4 leaders: 2.943736e-04 +Pairwise Multileader Locality, 4 leaders: 6.187463e-04 +Nonblocking Multileader, 4 leaders: 4.251048e-04 +Nonblocking Locality Aware, 4 leaders: 2.207974e-04 +Nonblocking Multileader Locality, 4 leaders: 5.105033e-04 +Pairwise Multileader, 10 leaders: 5.919954e-04 +Pairwise Locality Aware, 10 leaders: 5.642135e-04 +Pairwise Multileader Locality, 10 leaders: 2.874079e-04 +Nonblocking Multileader, 10 leaders: 2.796500e-04 +Nonblocking Locality Aware, 10 leaders: 6.287818e-04 +Nonblocking Multileader Locality, 10 leaders: 2.314977e-04 +Pairwise Multileader, 20 leaders: 1.030694e-03 +Pairwise Locality Aware, 20 leaders: 1.024863e-03 +Pairwise Multileader Locality, 20 leaders: 1.959993e-04 +Nonblocking Multileader, 20 leaders: 1.781090e-03 +Nonblocking Locality Aware, 20 leaders: 8.644465e-02 +Nonblocking Multileader Locality, 20 leaders: 1.470660e-04 +Size 8 +PMPI: 5.430142e-04 +Pairwise: 4.156450e-03 +NonBlocking: 1.153566e-01 +Pairwise Hierarchical: 3.666974e-03 +Pairwise Node Aware: 5.214275e-04 +Nonblocking Hierarchical: 3.258011e-03 +Nonblocking Node Aware: 4.667338e-04 +Pairwise Multileader, 4 leaders: 1.095631e-03 +Pairwise Locality Aware, 4 leaders: 3.490460e-04 +Pairwise Multileader Locality, 4 leaders: 1.432619e-03 +Nonblocking Multileader, 4 leaders: 8.025372e-04 +Nonblocking Locality Aware, 4 leaders: 3.112029e-04 +Nonblocking Multileader Locality, 4 leaders: 1.160926e-03 +Pairwise Multileader, 10 leaders: 8.651036e-04 +Pairwise Locality Aware, 10 leaders: 5.455136e-04 +Pairwise Multileader Locality, 10 leaders: 5.256556e-04 +Nonblocking Multileader, 10 leaders: 3.864221e-04 +Nonblocking Locality Aware, 10 leaders: 3.092190e-03 +Nonblocking Multileader Locality, 10 leaders: 4.502662e-04 +Pairwise Multileader, 20 leaders: 1.089016e-03 +Pairwise Locality Aware, 20 leaders: 9.865052e-04 +Pairwise Multileader Locality, 20 leaders: 3.295422e-04 +Nonblocking Multileader, 20 leaders: 1.699020e-03 +Nonblocking Locality Aware, 20 leaders: 3.722295e-02 +Nonblocking Multileader Locality, 20 leaders: 2.374015e-04 +Size 16 +PMPI: 1.001455e-03 +Pairwise: 4.357787e-03 +NonBlocking: 9.260580e-02 +Pairwise Hierarchical: 8.513041e-03 +Pairwise Node Aware: 7.017980e-04 +Nonblocking Hierarchical: 7.507446e-03 +Nonblocking Node Aware: 5.585254e-04 +Pairwise Multileader, 4 leaders: 2.310880e-03 +Pairwise Locality Aware, 4 leaders: 4.870361e-04 +Pairwise Multileader Locality, 4 leaders: 2.888447e-03 +Nonblocking Multileader, 4 leaders: 1.856430e-03 +Nonblocking Locality Aware, 4 leaders: 4.179211e-04 +Nonblocking Multileader Locality, 4 leaders: 2.398017e-03 +Pairwise Multileader, 10 leaders: 1.315612e-03 +Pairwise Locality Aware, 10 leaders: 6.683733e-04 +Pairwise Multileader Locality, 10 leaders: 1.079742e-03 +Nonblocking Multileader, 10 leaders: 7.740582e-04 +Nonblocking Locality Aware, 10 leaders: 3.129437e-03 +Nonblocking Multileader Locality, 10 leaders: 9.588717e-04 +Pairwise Multileader, 20 leaders: 1.276213e-03 +Pairwise Locality Aware, 20 leaders: 1.107833e-03 +Pairwise Multileader Locality, 20 leaders: 6.661153e-04 +Nonblocking Multileader, 20 leaders: 2.141149e-03 +Nonblocking Locality Aware, 20 leaders: 2.217817e-02 +Nonblocking Multileader Locality, 20 leaders: 5.664220e-04 +Size 32 +PMPI: 2.155550e-03 +Pairwise: 4.342498e-03 +NonBlocking: 1.155634e-01 +Pairwise Hierarchical: 1.872683e-02 +Pairwise Node Aware: 1.209662e-03 +Nonblocking Hierarchical: 1.630433e-02 +Nonblocking Node Aware: 1.081936e-03 +Pairwise Multileader, 4 leaders: 5.820216e-03 +Pairwise Locality Aware, 4 leaders: 8.187667e-04 +Pairwise Multileader Locality, 4 leaders: 5.478334e-03 +Nonblocking Multileader, 4 leaders: 4.436258e-03 +Nonblocking Locality Aware, 4 leaders: 7.721508e-04 +Nonblocking Multileader Locality, 4 leaders: 4.592588e-03 +Pairwise Multileader, 10 leaders: 2.310436e-03 +Pairwise Locality Aware, 10 leaders: 1.089272e-03 +Pairwise Multileader Locality, 10 leaders: 2.309881e-03 +Nonblocking Multileader, 10 leaders: 1.542562e-03 +Nonblocking Locality Aware, 10 leaders: 8.880447e-04 +Nonblocking Multileader Locality, 10 leaders: 1.932075e-03 +Pairwise Multileader, 20 leaders: 1.952862e-03 +Pairwise Locality Aware, 20 leaders: 1.284235e-03 +Pairwise Multileader Locality, 20 leaders: 1.221284e-03 +Nonblocking Multileader, 20 leaders: 1.774606e-03 +Nonblocking Locality Aware, 20 leaders: 8.710515e-02 +Nonblocking Multileader Locality, 20 leaders: 1.018615e-03 +Size 64 +PMPI: 3.424263e-03 +Pairwise: 4.144221e-03 +NonBlocking: 1.389390e-01 +Pairwise Hierarchical: 3.938844e-02 +Pairwise Node Aware: 2.028890e-03 +Nonblocking Hierarchical: 3.279074e-02 +Nonblocking Node Aware: 1.449450e-03 +Pairwise Multileader, 4 leaders: 1.205387e-02 +Pairwise Locality Aware, 4 leaders: 1.956623e-03 +Pairwise Multileader Locality, 4 leaders: 1.007743e-02 +Nonblocking Multileader, 4 leaders: 7.751318e-03 +Nonblocking Locality Aware, 4 leaders: 2.017532e-03 +Nonblocking Multileader Locality, 4 leaders: 8.358980e-03 +Pairwise Multileader, 10 leaders: 4.804595e-03 +Pairwise Locality Aware, 10 leaders: 1.694465e-03 +Pairwise Multileader Locality, 10 leaders: 4.453640e-03 +Nonblocking Multileader, 10 leaders: 3.328845e-03 +Nonblocking Locality Aware, 10 leaders: 1.786976e-03 +Nonblocking Multileader Locality, 10 leaders: 3.765776e-03 +Pairwise Multileader, 20 leaders: 3.128529e-03 +Pairwise Locality Aware, 20 leaders: 1.876595e-03 +Pairwise Multileader Locality, 20 leaders: 2.660735e-03 +Nonblocking Multileader, 20 leaders: 1.669609e-03 +Nonblocking Locality Aware, 20 leaders: 8.743776e-02 +Nonblocking Multileader Locality, 20 leaders: 2.184177e-03 +Size 128 +PMPI: 6.643504e-03 +Pairwise: 5.571145e-03 +NonBlocking: 1.167399e-01 +Pairwise Hierarchical: 7.575440e-02 +Pairwise Node Aware: 4.114686e-03 +Nonblocking Hierarchical: 6.351751e-02 +Nonblocking Node Aware: 3.025671e-03 +Pairwise Multileader, 4 leaders: 2.452878e-02 +Pairwise Locality Aware, 4 leaders: 3.111148e-03 +Pairwise Multileader Locality, 4 leaders: 2.310312e-02 +Nonblocking Multileader, 4 leaders: 1.519197e-02 +Nonblocking Locality Aware, 4 leaders: 3.309011e-03 +Nonblocking Multileader Locality, 4 leaders: 1.717245e-02 +Pairwise Multileader, 10 leaders: 1.014576e-02 +Pairwise Locality Aware, 10 leaders: 3.287826e-03 +Pairwise Multileader Locality, 10 leaders: 9.559164e-03 +Nonblocking Multileader, 10 leaders: 7.406706e-03 +Nonblocking Locality Aware, 10 leaders: 3.955910e-03 +Nonblocking Multileader Locality, 10 leaders: 8.344183e-03 +Pairwise Multileader, 20 leaders: 5.725656e-03 +Pairwise Locality Aware, 20 leaders: 3.135734e-03 +Pairwise Multileader Locality, 20 leaders: 5.089975e-03 +Nonblocking Multileader, 20 leaders: 3.423720e-03 +Nonblocking Locality Aware, 20 leaders: 3.825056e-02 +Nonblocking Multileader Locality, 20 leaders: 4.341241e-03 +Size 256 +PMPI: 1.228974e-02 +Pairwise: 9.146970e-03 +NonBlocking: 1.192429e-01 +Pairwise Hierarchical: 1.425480e-01 +Pairwise Node Aware: 6.144919e-03 +Nonblocking Hierarchical: 1.232033e-01 +Nonblocking Node Aware: 5.629922e-03 +Pairwise Multileader, 4 leaders: 4.920186e-02 +Pairwise Locality Aware, 4 leaders: 6.079716e-03 +Pairwise Multileader Locality, 4 leaders: 5.118241e-02 +Nonblocking Multileader, 4 leaders: 3.555731e-02 +Nonblocking Locality Aware, 4 leaders: 4.898636e-03 +Nonblocking Multileader Locality, 4 leaders: 4.013469e-02 +Pairwise Multileader, 10 leaders: 2.719532e-02 +Pairwise Locality Aware, 10 leaders: 6.392925e-03 +Pairwise Multileader Locality, 10 leaders: 2.257741e-02 +Nonblocking Multileader, 10 leaders: 1.756751e-02 +Nonblocking Locality Aware, 10 leaders: 9.005653e-03 +Nonblocking Multileader Locality, 10 leaders: 1.932430e-02 +Pairwise Multileader, 20 leaders: 1.207182e-02 +Pairwise Locality Aware, 20 leaders: 6.496883e-03 +Pairwise Multileader Locality, 20 leaders: 1.090202e-02 +Nonblocking Multileader, 20 leaders: 7.813808e-03 +Nonblocking Locality Aware, 20 leaders: 8.764549e-03 +Nonblocking Multileader Locality, 20 leaders: 9.138185e-03 +Size 512 +PMPI: 1.522098e-02 +Pairwise: 1.418722e-02 +NonBlocking: 1.178269e-01 +Pairwise Hierarchical: 2.717369e-01 +Pairwise Node Aware: 1.304950e-02 +Nonblocking Hierarchical: 2.546055e-01 +Nonblocking Node Aware: 1.143241e-02 +Pairwise Multileader, 4 leaders: 9.212347e-02 +Pairwise Locality Aware, 4 leaders: 1.357621e-02 +Pairwise Multileader Locality, 4 leaders: 1.004756e-01 +Nonblocking Multileader, 4 leaders: 6.765265e-02 +Nonblocking Locality Aware, 4 leaders: 1.277568e-02 +Nonblocking Multileader Locality, 4 leaders: 8.007690e-02 +Pairwise Multileader, 10 leaders: 5.020823e-02 +Pairwise Locality Aware, 10 leaders: 1.393692e-02 +Pairwise Multileader Locality, 10 leaders: 4.895603e-02 +Nonblocking Multileader, 10 leaders: 3.542708e-02 +Nonblocking Locality Aware, 10 leaders: 1.261089e-02 +Nonblocking Multileader Locality, 10 leaders: 4.136768e-02 +Pairwise Multileader, 20 leaders: 2.628671e-02 +Pairwise Locality Aware, 20 leaders: 1.323797e-02 +Pairwise Multileader Locality, 20 leaders: 2.420895e-02 +Nonblocking Multileader, 20 leaders: 1.649377e-02 +Nonblocking Locality Aware, 20 leaders: 2.101197e-02 +Nonblocking Multileader Locality, 20 leaders: 2.007749e-02 +Size 1024 +PMPI: 2.923484e-02 +Pairwise: 2.631370e-02 +NonBlocking: 1.099066e-01 +Pairwise Hierarchical: 5.390998e-01 +Pairwise Node Aware: 2.979435e-02 +Nonblocking Hierarchical: 5.066787e-01 +Nonblocking Node Aware: 2.521170e-02 +Pairwise Multileader, 4 leaders: 1.701844e-01 +Pairwise Locality Aware, 4 leaders: 2.685990e-02 +Pairwise Multileader Locality, 4 leaders: 1.883206e-01 +Nonblocking Multileader, 4 leaders: 1.282054e-01 +Nonblocking Locality Aware, 4 leaders: 2.083976e-02 +Nonblocking Multileader Locality, 4 leaders: 1.537469e-01 +Pairwise Multileader, 10 leaders: 9.875577e-02 +Pairwise Locality Aware, 10 leaders: 2.993699e-02 +Pairwise Multileader Locality, 10 leaders: 1.002416e-01 +Nonblocking Multileader, 10 leaders: 7.079748e-02 +Nonblocking Locality Aware, 10 leaders: 2.869075e-02 +Nonblocking Multileader Locality, 10 leaders: 8.528163e-02 +Pairwise Multileader, 20 leaders: 5.936944e-02 +Pairwise Locality Aware, 20 leaders: 3.138829e-02 +Pairwise Multileader Locality, 20 leaders: 5.078973e-02 +Nonblocking Multileader, 20 leaders: 3.701523e-02 +Nonblocking Locality Aware, 20 leaders: 2.659195e-02 +Nonblocking Multileader Locality, 20 leaders: 4.241991e-02 +Size 2048 +PMPI: 5.343701e-02 +Pairwise: 5.239310e-02 +NonBlocking: 1.101325e-01 +Pairwise Hierarchical: 1.099483e+00 +Pairwise Node Aware: 5.435726e-02 +Nonblocking Hierarchical: 1.128677e+00 +Nonblocking Node Aware: 4.443049e-02 +Pairwise Multileader, 4 leaders: 3.128048e-01 +Pairwise Locality Aware, 4 leaders: 5.405703e-02 +Pairwise Multileader Locality, 4 leaders: 3.478201e-01 +Nonblocking Multileader, 4 leaders: 2.421435e-01 +Nonblocking Locality Aware, 4 leaders: 4.141616e-02 +Nonblocking Multileader Locality, 4 leaders: 2.976295e-01 +Pairwise Multileader, 10 leaders: 1.793847e-01 +Pairwise Locality Aware, 10 leaders: 5.670287e-02 +Pairwise Multileader Locality, 10 leaders: 1.952256e-01 +Nonblocking Multileader, 10 leaders: 1.362938e-01 +Nonblocking Locality Aware, 10 leaders: 4.203086e-02 +Nonblocking Multileader Locality, 10 leaders: 1.671199e-01 +Pairwise Multileader, 20 leaders: 1.195212e-01 +Pairwise Locality Aware, 20 leaders: 5.809230e-02 +Pairwise Multileader Locality, 20 leaders: 1.145829e-01 +Nonblocking Multileader, 20 leaders: 8.174700e-02 +Nonblocking Locality Aware, 20 leaders: 4.831040e-02 +Nonblocking Multileader Locality, 20 leaders: 1.003459e-01 +Size 4096 +PMPI: 1.285362e-01 +Pairwise: 1.112194e-01 +NonBlocking: 1.725866e-01 +Pairwise Hierarchical: 2.290457e+00 +Pairwise Node Aware: 9.916759e-02 +Nonblocking Hierarchical: 2.177929e+00 +Nonblocking Node Aware: 9.002582e-02 +Pairwise Multileader, 4 leaders: 6.168731e-01 +Pairwise Locality Aware, 4 leaders: 1.069405e-01 +Pairwise Multileader Locality, 4 leaders: 7.179065e-01 +Nonblocking Multileader, 4 leaders: 5.344452e-01 +Nonblocking Locality Aware, 4 leaders: 8.708700e-02 +Nonblocking Multileader Locality, 4 leaders: 6.577190e-01 +Pairwise Multileader, 10 leaders: 3.697131e-01 +Pairwise Locality Aware, 10 leaders: 1.110821e-01 +Pairwise Multileader Locality, 10 leaders: 4.111351e-01 +Nonblocking Multileader, 10 leaders: 2.678741e-01 +Nonblocking Locality Aware, 10 leaders: 8.338982e-02 +Nonblocking Multileader Locality, 10 leaders: 3.339814e-01 +Pairwise Multileader, 20 leaders: 2.176393e-01 +Pairwise Locality Aware, 20 leaders: 1.148163e-01 +Pairwise Multileader Locality, 20 leaders: 2.211782e-01 +Nonblocking Multileader, 20 leaders: 1.579263e-01 +Nonblocking Locality Aware, 20 leaders: 8.852816e-02 +Nonblocking Multileader Locality, 20 leaders: 1.971804e-01 +Size 8192 +PMPI: 2.129191e-01 +Pairwise: 2.118655e-01 +NonBlocking: 1.886267e-01 +Test 2 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.492398e-04 +Pairwise: 2.496934e-03 +NonBlocking: 7.427949e-04 +Pairwise Hierarchical: 5.807557e-04 +Pairwise Node Aware: 2.785059e-04 +Nonblocking Hierarchical: 5.275122e-04 +Nonblocking Node Aware: 2.192806e-04 +Pairwise Multileader, 4 leaders: 3.080591e-04 +Pairwise Locality Aware, 4 leaders: 2.360101e-04 +Pairwise Multileader Locality, 4 leaders: 2.274679e-04 +Nonblocking Multileader, 4 leaders: 1.578290e-04 +Nonblocking Locality Aware, 4 leaders: 1.112757e-04 +Nonblocking Multileader Locality, 4 leaders: 1.816057e-04 +Pairwise Multileader, 10 leaders: 4.462485e-04 +Pairwise Locality Aware, 10 leaders: 4.547325e-04 +Pairwise Multileader Locality, 10 leaders: 1.312328e-04 +Nonblocking Multileader, 10 leaders: 1.176809e-04 +Nonblocking Locality Aware, 10 leaders: 2.386616e-04 +Nonblocking Multileader Locality, 10 leaders: 9.997952e-05 +Pairwise Multileader, 20 leaders: 7.989234e-04 +Pairwise Locality Aware, 20 leaders: 9.166746e-04 +Pairwise Multileader Locality, 20 leaders: 1.304480e-04 +Nonblocking Multileader, 20 leaders: 1.765165e-04 +Nonblocking Locality Aware, 20 leaders: 1.459904e-03 +Nonblocking Multileader Locality, 20 leaders: 8.157516e-05 +Size 2 +PMPI: 1.919759e-04 +Pairwise: 1.969727e-03 +NonBlocking: 7.077524e-04 +Pairwise Hierarchical: 1.225816e-03 +Pairwise Node Aware: 2.787901e-04 +Nonblocking Hierarchical: 1.065853e-03 +Nonblocking Node Aware: 2.183824e-04 +Pairwise Multileader, 4 leaders: 4.109633e-04 +Pairwise Locality Aware, 4 leaders: 2.513852e-04 +Pairwise Multileader Locality, 4 leaders: 3.970890e-04 +Nonblocking Multileader, 4 leaders: 2.628123e-04 +Nonblocking Locality Aware, 4 leaders: 1.167168e-04 +Nonblocking Multileader Locality, 4 leaders: 3.076490e-04 +Pairwise Multileader, 10 leaders: 4.886015e-04 +Pairwise Locality Aware, 10 leaders: 5.276764e-04 +Pairwise Multileader Locality, 10 leaders: 1.690112e-04 +Nonblocking Multileader, 10 leaders: 1.487905e-04 +Nonblocking Locality Aware, 10 leaders: 1.971999e-04 +Nonblocking Multileader Locality, 10 leaders: 1.395213e-04 +Pairwise Multileader, 20 leaders: 8.744362e-04 +Pairwise Locality Aware, 20 leaders: 9.038492e-04 +Pairwise Multileader Locality, 20 leaders: 1.529157e-04 +Nonblocking Multileader, 20 leaders: 1.408318e-03 +Nonblocking Locality Aware, 20 leaders: 6.757395e-03 +Nonblocking Multileader Locality, 20 leaders: 1.067338e-04 +Size 4 +PMPI: 3.122771e-04 +Pairwise: 4.353315e-03 +NonBlocking: 9.235106e-02 +Pairwise Hierarchical: 1.956134e-03 +Pairwise Node Aware: 4.090327e-04 +Nonblocking Hierarchical: 1.694139e-03 +Nonblocking Node Aware: 2.599841e-04 +Pairwise Multileader, 4 leaders: 6.266939e-04 +Pairwise Locality Aware, 4 leaders: 2.906090e-04 +Pairwise Multileader Locality, 4 leaders: 6.213225e-04 +Nonblocking Multileader, 4 leaders: 4.327028e-04 +Nonblocking Locality Aware, 4 leaders: 3.097954e-04 +Nonblocking Multileader Locality, 4 leaders: 5.137810e-04 +Pairwise Multileader, 10 leaders: 5.449424e-04 +Pairwise Locality Aware, 10 leaders: 5.153092e-04 +Pairwise Multileader Locality, 10 leaders: 2.896868e-04 +Nonblocking Multileader, 10 leaders: 2.135682e-04 +Nonblocking Locality Aware, 10 leaders: 2.749576e-04 +Nonblocking Multileader Locality, 10 leaders: 2.363953e-04 +Pairwise Multileader, 20 leaders: 1.007156e-03 +Pairwise Locality Aware, 20 leaders: 8.438905e-04 +Pairwise Multileader Locality, 20 leaders: 1.996803e-04 +Nonblocking Multileader, 20 leaders: 1.436722e-03 +Nonblocking Locality Aware, 20 leaders: 2.491148e-02 +Nonblocking Multileader Locality, 20 leaders: 1.476690e-04 +Size 8 +PMPI: 5.485782e-04 +Pairwise: 4.370843e-03 +NonBlocking: 9.243151e-02 +Pairwise Hierarchical: 3.720626e-03 +Pairwise Node Aware: 5.278771e-04 +Nonblocking Hierarchical: 3.332998e-03 +Nonblocking Node Aware: 3.997153e-04 +Pairwise Multileader, 4 leaders: 1.076402e-03 +Pairwise Locality Aware, 4 leaders: 3.312352e-04 +Pairwise Multileader Locality, 4 leaders: 1.426704e-03 +Nonblocking Multileader, 4 leaders: 8.182970e-04 +Nonblocking Locality Aware, 4 leaders: 2.752947e-04 +Nonblocking Multileader Locality, 4 leaders: 1.153109e-03 +Pairwise Multileader, 10 leaders: 7.964947e-04 +Pairwise Locality Aware, 10 leaders: 4.993251e-04 +Pairwise Multileader Locality, 10 leaders: 5.388260e-04 +Nonblocking Multileader, 10 leaders: 3.933706e-04 +Nonblocking Locality Aware, 10 leaders: 3.798766e-03 +Nonblocking Multileader Locality, 10 leaders: 4.608496e-04 +Pairwise Multileader, 20 leaders: 9.922902e-04 +Pairwise Locality Aware, 20 leaders: 9.375398e-04 +Pairwise Multileader Locality, 20 leaders: 3.313396e-04 +Nonblocking Multileader, 20 leaders: 2.471782e-03 +Nonblocking Locality Aware, 20 leaders: 7.420705e-02 +Nonblocking Multileader Locality, 20 leaders: 2.459262e-04 +Size 16 +PMPI: 1.071065e-03 +Pairwise: 4.332300e-03 +NonBlocking: 1.380275e-01 +Pairwise Hierarchical: 8.457314e-03 +Pairwise Node Aware: 7.064560e-04 +Nonblocking Hierarchical: 7.444567e-03 +Nonblocking Node Aware: 5.499228e-04 +Pairwise Multileader, 4 leaders: 2.358910e-03 +Pairwise Locality Aware, 4 leaders: 4.966302e-04 +Pairwise Multileader Locality, 4 leaders: 2.903742e-03 +Nonblocking Multileader, 4 leaders: 1.860751e-03 +Nonblocking Locality Aware, 4 leaders: 4.320913e-04 +Nonblocking Multileader Locality, 4 leaders: 2.367226e-03 +Pairwise Multileader, 10 leaders: 1.358401e-03 +Pairwise Locality Aware, 10 leaders: 6.864155e-04 +Pairwise Multileader Locality, 10 leaders: 1.074262e-03 +Nonblocking Multileader, 10 leaders: 7.761685e-04 +Nonblocking Locality Aware, 10 leaders: 5.089803e-04 +Nonblocking Multileader Locality, 10 leaders: 9.507672e-04 +Pairwise Multileader, 20 leaders: 1.292349e-03 +Pairwise Locality Aware, 20 leaders: 1.020202e-03 +Pairwise Multileader Locality, 20 leaders: 6.180847e-04 +Nonblocking Multileader, 20 leaders: 3.416951e-03 +Nonblocking Locality Aware, 20 leaders: 2.723961e-02 +Nonblocking Multileader Locality, 20 leaders: 4.905910e-04 +Size 32 +PMPI: 2.016338e-03 +Pairwise: 4.565328e-03 +NonBlocking: 1.155560e-01 +Pairwise Hierarchical: 1.874870e-02 +Pairwise Node Aware: 1.142023e-03 +Nonblocking Hierarchical: 1.613258e-02 +Nonblocking Node Aware: 9.833464e-04 +Pairwise Multileader, 4 leaders: 5.795012e-03 +Pairwise Locality Aware, 4 leaders: 7.729393e-04 +Pairwise Multileader Locality, 4 leaders: 5.503028e-03 +Nonblocking Multileader, 4 leaders: 4.446762e-03 +Nonblocking Locality Aware, 4 leaders: 6.578803e-04 +Nonblocking Multileader Locality, 4 leaders: 4.592052e-03 +Pairwise Multileader, 10 leaders: 2.369852e-03 +Pairwise Locality Aware, 10 leaders: 1.157796e-03 +Pairwise Multileader Locality, 10 leaders: 2.314762e-03 +Nonblocking Multileader, 10 leaders: 1.553616e-03 +Nonblocking Locality Aware, 10 leaders: 4.631649e-03 +Nonblocking Multileader Locality, 10 leaders: 1.922223e-03 +Pairwise Multileader, 20 leaders: 1.792910e-03 +Pairwise Locality Aware, 20 leaders: 1.368631e-03 +Pairwise Multileader Locality, 20 leaders: 1.199766e-03 +Nonblocking Multileader, 20 leaders: 9.479616e-04 +Nonblocking Locality Aware, 20 leaders: 6.203719e-02 +Nonblocking Multileader Locality, 20 leaders: 1.056836e-03 +Size 64 +PMPI: 3.369257e-03 +Pairwise: 4.185838e-03 +NonBlocking: 1.156514e-01 +Pairwise Hierarchical: 3.928918e-02 +Pairwise Node Aware: 1.934604e-03 +Nonblocking Hierarchical: 3.263681e-02 +Nonblocking Node Aware: 1.402934e-03 +Pairwise Multileader, 4 leaders: 1.198097e-02 +Pairwise Locality Aware, 4 leaders: 1.626274e-03 +Pairwise Multileader Locality, 4 leaders: 1.009827e-02 +Nonblocking Multileader, 4 leaders: 7.836669e-03 +Nonblocking Locality Aware, 4 leaders: 1.604555e-03 +Nonblocking Multileader Locality, 4 leaders: 8.418336e-03 +Pairwise Multileader, 10 leaders: 4.772433e-03 +Pairwise Locality Aware, 10 leaders: 1.736841e-03 +Pairwise Multileader Locality, 10 leaders: 4.474734e-03 +Nonblocking Multileader, 10 leaders: 3.370005e-03 +Nonblocking Locality Aware, 10 leaders: 1.944872e-03 +Nonblocking Multileader Locality, 10 leaders: 3.790057e-03 +Pairwise Multileader, 20 leaders: 3.020322e-03 +Pairwise Locality Aware, 20 leaders: 1.904468e-03 +Pairwise Multileader Locality, 20 leaders: 2.725476e-03 +Nonblocking Multileader, 20 leaders: 1.627476e-03 +Nonblocking Locality Aware, 20 leaders: 8.705865e-02 +Nonblocking Multileader Locality, 20 leaders: 2.195367e-03 +Size 128 +PMPI: 6.538744e-03 +Pairwise: 5.457260e-03 +NonBlocking: 1.168202e-01 +Pairwise Hierarchical: 7.675433e-02 +Pairwise Node Aware: 4.022558e-03 +Nonblocking Hierarchical: 6.386510e-02 +Nonblocking Node Aware: 3.068529e-03 +Pairwise Multileader, 4 leaders: 2.366100e-02 +Pairwise Locality Aware, 4 leaders: 3.107997e-03 +Pairwise Multileader Locality, 4 leaders: 2.260566e-02 +Nonblocking Multileader, 4 leaders: 1.564095e-02 +Nonblocking Locality Aware, 4 leaders: 3.465371e-03 +Nonblocking Multileader Locality, 4 leaders: 1.737768e-02 +Pairwise Multileader, 10 leaders: 1.036544e-02 +Pairwise Locality Aware, 10 leaders: 3.008568e-03 +Pairwise Multileader Locality, 10 leaders: 9.593723e-03 +Nonblocking Multileader, 10 leaders: 7.493152e-03 +Nonblocking Locality Aware, 10 leaders: 3.818021e-03 +Nonblocking Multileader Locality, 10 leaders: 8.430674e-03 +Pairwise Multileader, 20 leaders: 5.777538e-03 +Pairwise Locality Aware, 20 leaders: 3.432078e-03 +Pairwise Multileader Locality, 20 leaders: 5.163907e-03 +Nonblocking Multileader, 20 leaders: 3.645061e-03 +Nonblocking Locality Aware, 20 leaders: 7.641695e-02 +Nonblocking Multileader Locality, 20 leaders: 4.384851e-03 +Size 256 +PMPI: 1.215941e-02 +Pairwise: 9.085801e-03 +NonBlocking: 1.180513e-01 +Pairwise Hierarchical: 1.445735e-01 +Pairwise Node Aware: 6.145995e-03 +Nonblocking Hierarchical: 1.247488e-01 +Nonblocking Node Aware: 5.731894e-03 +Pairwise Multileader, 4 leaders: 4.987239e-02 +Pairwise Locality Aware, 4 leaders: 6.542574e-03 +Pairwise Multileader Locality, 4 leaders: 5.190492e-02 +Nonblocking Multileader, 4 leaders: 3.619983e-02 +Nonblocking Locality Aware, 4 leaders: 5.550339e-03 +Nonblocking Multileader Locality, 4 leaders: 4.039311e-02 +Pairwise Multileader, 10 leaders: 2.707173e-02 +Pairwise Locality Aware, 10 leaders: 6.921778e-03 +Pairwise Multileader Locality, 10 leaders: 2.283454e-02 +Nonblocking Multileader, 10 leaders: 1.757865e-02 +Nonblocking Locality Aware, 10 leaders: 9.285789e-03 +Nonblocking Multileader Locality, 10 leaders: 1.930930e-02 +Pairwise Multileader, 20 leaders: 1.299405e-02 +Pairwise Locality Aware, 20 leaders: 7.181334e-03 +Pairwise Multileader Locality, 20 leaders: 1.090062e-02 +Nonblocking Multileader, 20 leaders: 7.856869e-03 +Nonblocking Locality Aware, 20 leaders: 9.008498e-03 +Nonblocking Multileader Locality, 20 leaders: 9.171398e-03 +Size 512 +PMPI: 1.461207e-02 +Pairwise: 1.486784e-02 +NonBlocking: 1.174333e-01 +Pairwise Hierarchical: 2.726637e-01 +Pairwise Node Aware: 1.396765e-02 +Nonblocking Hierarchical: 2.478700e-01 +Nonblocking Node Aware: 1.314397e-02 +Pairwise Multileader, 4 leaders: 9.225115e-02 +Pairwise Locality Aware, 4 leaders: 1.314140e-02 +Pairwise Multileader Locality, 4 leaders: 9.996753e-02 +Nonblocking Multileader, 4 leaders: 6.804921e-02 +Nonblocking Locality Aware, 4 leaders: 1.174489e-02 +Nonblocking Multileader Locality, 4 leaders: 7.983593e-02 +Pairwise Multileader, 10 leaders: 5.043378e-02 +Pairwise Locality Aware, 10 leaders: 1.471775e-02 +Pairwise Multileader Locality, 10 leaders: 4.898639e-02 +Nonblocking Multileader, 10 leaders: 3.546601e-02 +Nonblocking Locality Aware, 10 leaders: 1.309374e-02 +Nonblocking Multileader Locality, 10 leaders: 4.165339e-02 +Pairwise Multileader, 20 leaders: 2.557379e-02 +Pairwise Locality Aware, 20 leaders: 1.258540e-02 +Pairwise Multileader Locality, 20 leaders: 2.427667e-02 +Nonblocking Multileader, 20 leaders: 1.621427e-02 +Nonblocking Locality Aware, 20 leaders: 1.412872e-02 +Nonblocking Multileader Locality, 20 leaders: 2.048110e-02 +Size 1024 +PMPI: 3.237239e-02 +Pairwise: 2.836733e-02 +NonBlocking: 1.120548e-01 +Pairwise Hierarchical: 5.256753e-01 +Pairwise Node Aware: 2.828543e-02 +Nonblocking Hierarchical: 5.060097e-01 +Nonblocking Node Aware: 2.550762e-02 +Pairwise Multileader, 4 leaders: 1.685085e-01 +Pairwise Locality Aware, 4 leaders: 2.643500e-02 +Pairwise Multileader Locality, 4 leaders: 1.883084e-01 +Nonblocking Multileader, 4 leaders: 1.260337e-01 +Nonblocking Locality Aware, 4 leaders: 2.040423e-02 +Nonblocking Multileader Locality, 4 leaders: 1.518816e-01 +Pairwise Multileader, 10 leaders: 1.019681e-01 +Pairwise Locality Aware, 10 leaders: 2.916314e-02 +Pairwise Multileader Locality, 10 leaders: 1.002229e-01 +Nonblocking Multileader, 10 leaders: 7.094857e-02 +Nonblocking Locality Aware, 10 leaders: 2.772868e-02 +Nonblocking Multileader Locality, 10 leaders: 8.573496e-02 +Pairwise Multileader, 20 leaders: 5.982535e-02 +Pairwise Locality Aware, 20 leaders: 2.894355e-02 +Pairwise Multileader Locality, 20 leaders: 5.103664e-02 +Nonblocking Multileader, 20 leaders: 3.711807e-02 +Nonblocking Locality Aware, 20 leaders: 3.045401e-02 +Nonblocking Multileader Locality, 20 leaders: 4.274571e-02 +Size 2048 +PMPI: 5.701770e-02 +Pairwise: 5.363888e-02 +NonBlocking: 1.033076e-01 +Pairwise Hierarchical: 1.134520e+00 +Pairwise Node Aware: 5.287295e-02 +Nonblocking Hierarchical: 1.040837e+00 +Nonblocking Node Aware: 4.384165e-02 +Pairwise Multileader, 4 leaders: 3.203080e-01 +Pairwise Locality Aware, 4 leaders: 5.372680e-02 +Pairwise Multileader Locality, 4 leaders: 3.478289e-01 +Nonblocking Multileader, 4 leaders: 2.429852e-01 +Nonblocking Locality Aware, 4 leaders: 4.150848e-02 +Nonblocking Multileader Locality, 4 leaders: 2.973107e-01 +Pairwise Multileader, 10 leaders: 1.805095e-01 +Pairwise Locality Aware, 10 leaders: 5.554389e-02 +Pairwise Multileader Locality, 10 leaders: 1.943748e-01 +Nonblocking Multileader, 10 leaders: 1.362096e-01 +Nonblocking Locality Aware, 10 leaders: 4.153618e-02 +Nonblocking Multileader Locality, 10 leaders: 1.674153e-01 +Pairwise Multileader, 20 leaders: 1.208826e-01 +Pairwise Locality Aware, 20 leaders: 5.549845e-02 +Pairwise Multileader Locality, 20 leaders: 1.151763e-01 +Nonblocking Multileader, 20 leaders: 8.258327e-02 +Nonblocking Locality Aware, 20 leaders: 8.210335e-02 +Nonblocking Multileader Locality, 20 leaders: 1.008880e-01 +Size 4096 +PMPI: 1.266656e-01 +Pairwise: 1.183358e-01 +NonBlocking: 1.982224e-01 +Pairwise Hierarchical: 2.268740e+00 +Pairwise Node Aware: 1.012526e-01 +Nonblocking Hierarchical: 2.247273e+00 +Nonblocking Node Aware: 8.841036e-02 +Pairwise Multileader, 4 leaders: 6.297691e-01 +Pairwise Locality Aware, 4 leaders: 1.075279e-01 +Pairwise Multileader Locality, 4 leaders: 7.191539e-01 +Nonblocking Multileader, 4 leaders: 5.398008e-01 +Nonblocking Locality Aware, 4 leaders: 8.625106e-02 +Nonblocking Multileader Locality, 4 leaders: 6.469873e-01 +Pairwise Multileader, 10 leaders: 3.702427e-01 +Pairwise Locality Aware, 10 leaders: 1.113550e-01 +Pairwise Multileader Locality, 10 leaders: 4.108912e-01 +Nonblocking Multileader, 10 leaders: 2.697719e-01 +Nonblocking Locality Aware, 10 leaders: 8.345668e-02 +Nonblocking Multileader Locality, 10 leaders: 3.380908e-01 +Pairwise Multileader, 20 leaders: 2.133646e-01 +Pairwise Locality Aware, 20 leaders: 1.149017e-01 +Pairwise Multileader Locality, 20 leaders: 2.209585e-01 +Nonblocking Multileader, 20 leaders: 1.587540e-01 +Nonblocking Locality Aware, 20 leaders: 8.781487e-02 +Nonblocking Multileader Locality, 20 leaders: 1.956479e-01 +Size 8192 +PMPI: 2.124008e-01 +Pairwise: 2.099761e-01 +NonBlocking: 1.862799e-01 +Test 3 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.477801e-04 +Pairwise: 2.194420e-03 +NonBlocking: 7.920313e-04 +Pairwise Hierarchical: 5.796816e-04 +Pairwise Node Aware: 2.762118e-04 +Nonblocking Hierarchical: 5.312088e-04 +Nonblocking Node Aware: 2.209129e-04 +Pairwise Multileader, 4 leaders: 2.927292e-04 +Pairwise Locality Aware, 4 leaders: 2.468040e-04 +Pairwise Multileader Locality, 4 leaders: 2.272670e-04 +Nonblocking Multileader, 4 leaders: 1.547999e-04 +Nonblocking Locality Aware, 4 leaders: 1.344373e-04 +Nonblocking Multileader Locality, 4 leaders: 1.809379e-04 +Pairwise Multileader, 10 leaders: 5.005820e-04 +Pairwise Locality Aware, 10 leaders: 4.054927e-04 +Pairwise Multileader Locality, 10 leaders: 1.317261e-04 +Nonblocking Multileader, 10 leaders: 1.183999e-04 +Nonblocking Locality Aware, 10 leaders: 4.265306e-04 +Nonblocking Multileader Locality, 10 leaders: 9.839831e-05 +Pairwise Multileader, 20 leaders: 8.792295e-04 +Pairwise Locality Aware, 20 leaders: 9.236341e-04 +Pairwise Multileader Locality, 20 leaders: 1.319969e-04 +Nonblocking Multileader, 20 leaders: 8.063022e-04 +Nonblocking Locality Aware, 20 leaders: 8.393318e-03 +Nonblocking Multileader Locality, 20 leaders: 8.268129e-05 +Size 2 +PMPI: 2.047297e-04 +Pairwise: 1.947590e-03 +NonBlocking: 7.184347e-04 +Pairwise Hierarchical: 1.239307e-03 +Pairwise Node Aware: 2.816645e-04 +Nonblocking Hierarchical: 1.066945e-03 +Nonblocking Node Aware: 2.196255e-04 +Pairwise Multileader, 4 leaders: 4.070547e-04 +Pairwise Locality Aware, 4 leaders: 2.628251e-04 +Pairwise Multileader Locality, 4 leaders: 4.041255e-04 +Nonblocking Multileader, 4 leaders: 2.662778e-04 +Nonblocking Locality Aware, 4 leaders: 1.483283e-04 +Nonblocking Multileader Locality, 4 leaders: 3.122518e-04 +Pairwise Multileader, 10 leaders: 5.110966e-04 +Pairwise Locality Aware, 10 leaders: 4.582182e-04 +Pairwise Multileader Locality, 10 leaders: 1.691348e-04 +Nonblocking Multileader, 10 leaders: 1.488479e-04 +Nonblocking Locality Aware, 10 leaders: 2.419907e-04 +Nonblocking Multileader Locality, 10 leaders: 1.335919e-04 +Pairwise Multileader, 20 leaders: 8.569052e-04 +Pairwise Locality Aware, 20 leaders: 1.020161e-03 +Pairwise Multileader Locality, 20 leaders: 1.496035e-04 +Nonblocking Multileader, 20 leaders: 1.002908e-03 +Nonblocking Locality Aware, 20 leaders: 9.421435e-03 +Nonblocking Multileader Locality, 20 leaders: 1.047893e-04 +Size 4 +PMPI: 3.243213e-04 +Pairwise: 3.787413e-03 +NonBlocking: 9.247596e-02 +Pairwise Hierarchical: 1.949925e-03 +Pairwise Node Aware: 4.088442e-04 +Nonblocking Hierarchical: 1.700107e-03 +Nonblocking Node Aware: 2.660587e-04 +Pairwise Multileader, 4 leaders: 6.543826e-04 +Pairwise Locality Aware, 4 leaders: 2.592227e-04 +Pairwise Multileader Locality, 4 leaders: 6.297172e-04 +Nonblocking Multileader, 4 leaders: 4.402374e-04 +Nonblocking Locality Aware, 4 leaders: 2.081470e-04 +Nonblocking Multileader Locality, 4 leaders: 5.156012e-04 +Pairwise Multileader, 10 leaders: 5.622495e-04 +Pairwise Locality Aware, 10 leaders: 5.615388e-04 +Pairwise Multileader Locality, 10 leaders: 2.846300e-04 +Nonblocking Multileader, 10 leaders: 2.162628e-04 +Nonblocking Locality Aware, 10 leaders: 1.940571e-03 +Nonblocking Multileader Locality, 10 leaders: 2.346959e-04 +Pairwise Multileader, 20 leaders: 9.540176e-04 +Pairwise Locality Aware, 20 leaders: 8.791999e-04 +Pairwise Multileader Locality, 20 leaders: 2.169033e-04 +Nonblocking Multileader, 20 leaders: 1.876236e-03 +Nonblocking Locality Aware, 20 leaders: 7.418312e-02 +Nonblocking Multileader Locality, 20 leaders: 1.680880e-04 +Size 8 +PMPI: 5.596472e-04 +Pairwise: 5.141804e-03 +NonBlocking: 9.933094e-02 +Pairwise Hierarchical: 3.667884e-03 +Pairwise Node Aware: 5.203605e-04 +Nonblocking Hierarchical: 3.264166e-03 +Nonblocking Node Aware: 4.103326e-04 +Pairwise Multileader, 4 leaders: 1.101637e-03 +Pairwise Locality Aware, 4 leaders: 3.468618e-04 +Pairwise Multileader Locality, 4 leaders: 1.425339e-03 +Nonblocking Multileader, 4 leaders: 8.220230e-04 +Nonblocking Locality Aware, 4 leaders: 3.181076e-04 +Nonblocking Multileader Locality, 4 leaders: 1.163444e-03 +Pairwise Multileader, 10 leaders: 7.963796e-04 +Pairwise Locality Aware, 10 leaders: 5.447226e-04 +Pairwise Multileader Locality, 10 leaders: 5.308783e-04 +Nonblocking Multileader, 10 leaders: 4.052424e-04 +Nonblocking Locality Aware, 10 leaders: 5.352905e-03 +Nonblocking Multileader Locality, 10 leaders: 4.689080e-04 +Pairwise Multileader, 20 leaders: 1.042847e-03 +Pairwise Locality Aware, 20 leaders: 9.625491e-04 +Pairwise Multileader Locality, 20 leaders: 3.283090e-04 +Nonblocking Multileader, 20 leaders: 2.797884e-03 +Nonblocking Locality Aware, 20 leaders: 7.421682e-02 +Nonblocking Multileader Locality, 20 leaders: 2.346514e-04 +Size 16 +PMPI: 1.024780e-03 +Pairwise: 4.347438e-03 +NonBlocking: 2.064343e-01 +Pairwise Hierarchical: 8.502394e-03 +Pairwise Node Aware: 7.126819e-04 +Nonblocking Hierarchical: 7.512588e-03 +Nonblocking Node Aware: 5.626395e-04 +Pairwise Multileader, 4 leaders: 2.324853e-03 +Pairwise Locality Aware, 4 leaders: 5.127892e-04 +Pairwise Multileader Locality, 4 leaders: 2.887188e-03 +Nonblocking Multileader, 4 leaders: 1.880462e-03 +Nonblocking Locality Aware, 4 leaders: 4.585147e-04 +Nonblocking Multileader Locality, 4 leaders: 2.370699e-03 +Pairwise Multileader, 10 leaders: 1.376635e-03 +Pairwise Locality Aware, 10 leaders: 7.266834e-04 +Pairwise Multileader Locality, 10 leaders: 1.084248e-03 +Nonblocking Multileader, 10 leaders: 7.865239e-04 +Nonblocking Locality Aware, 10 leaders: 5.360104e-04 +Nonblocking Multileader Locality, 10 leaders: 9.501798e-04 +Pairwise Multileader, 20 leaders: 1.393506e-03 +Pairwise Locality Aware, 20 leaders: 9.349970e-04 +Pairwise Multileader Locality, 20 leaders: 6.400658e-04 +Nonblocking Multileader, 20 leaders: 2.260918e-03 +Nonblocking Locality Aware, 20 leaders: 2.507127e-02 +Nonblocking Multileader Locality, 20 leaders: 4.821746e-04 +Size 32 +PMPI: 2.121056e-03 +Pairwise: 4.669107e-03 +NonBlocking: 1.155607e-01 +Pairwise Hierarchical: 1.889243e-02 +Pairwise Node Aware: 1.188675e-03 +Nonblocking Hierarchical: 1.606555e-02 +Nonblocking Node Aware: 1.056872e-03 +Pairwise Multileader, 4 leaders: 5.842089e-03 +Pairwise Locality Aware, 4 leaders: 7.653710e-04 +Pairwise Multileader Locality, 4 leaders: 5.555374e-03 +Nonblocking Multileader, 4 leaders: 4.459371e-03 +Nonblocking Locality Aware, 4 leaders: 6.792830e-04 +Nonblocking Multileader Locality, 4 leaders: 4.566882e-03 +Pairwise Multileader, 10 leaders: 2.341633e-03 +Pairwise Locality Aware, 10 leaders: 1.225468e-03 +Pairwise Multileader Locality, 10 leaders: 2.290676e-03 +Nonblocking Multileader, 10 leaders: 1.811763e-03 +Nonblocking Locality Aware, 10 leaders: 6.691185e-03 +Nonblocking Multileader Locality, 10 leaders: 1.924697e-03 +Pairwise Multileader, 20 leaders: 2.023137e-03 +Pairwise Locality Aware, 20 leaders: 1.445267e-03 +Pairwise Multileader Locality, 20 leaders: 1.241506e-03 +Nonblocking Multileader, 20 leaders: 1.342397e-03 +Nonblocking Locality Aware, 20 leaders: 8.707204e-02 +Nonblocking Multileader Locality, 20 leaders: 1.055961e-03 +Size 64 +PMPI: 3.187835e-03 +Pairwise: 5.093679e-03 +NonBlocking: 8.770372e-02 +Pairwise Hierarchical: 3.964684e-02 +Pairwise Node Aware: 2.099862e-03 +Nonblocking Hierarchical: 3.272865e-02 +Nonblocking Node Aware: 1.558563e-03 +Pairwise Multileader, 4 leaders: 1.214015e-02 +Pairwise Locality Aware, 4 leaders: 1.680084e-03 +Pairwise Multileader Locality, 4 leaders: 1.017868e-02 +Nonblocking Multileader, 4 leaders: 7.847127e-03 +Nonblocking Locality Aware, 4 leaders: 1.750401e-03 +Nonblocking Multileader Locality, 4 leaders: 8.392503e-03 +Pairwise Multileader, 10 leaders: 4.850426e-03 +Pairwise Locality Aware, 10 leaders: 1.806486e-03 +Pairwise Multileader Locality, 10 leaders: 4.465930e-03 +Nonblocking Multileader, 10 leaders: 3.347112e-03 +Nonblocking Locality Aware, 10 leaders: 1.997752e-03 +Nonblocking Multileader Locality, 10 leaders: 3.789435e-03 +Pairwise Multileader, 20 leaders: 3.350553e-03 +Pairwise Locality Aware, 20 leaders: 1.990781e-03 +Pairwise Multileader Locality, 20 leaders: 2.694670e-03 +Nonblocking Multileader, 20 leaders: 2.055874e-03 +Nonblocking Locality Aware, 20 leaders: 8.775276e-02 +Nonblocking Multileader Locality, 20 leaders: 2.168252e-03 +Size 128 +PMPI: 6.572366e-03 +Pairwise: 5.709343e-03 +NonBlocking: 1.174007e-01 +Pairwise Hierarchical: 7.605832e-02 +Pairwise Node Aware: 3.959975e-03 +Nonblocking Hierarchical: 6.373779e-02 +Nonblocking Node Aware: 3.002294e-03 +Pairwise Multileader, 4 leaders: 2.409636e-02 +Pairwise Locality Aware, 4 leaders: 3.255796e-03 +Pairwise Multileader Locality, 4 leaders: 2.238020e-02 +Nonblocking Multileader, 4 leaders: 1.542342e-02 +Nonblocking Locality Aware, 4 leaders: 3.721325e-03 +Nonblocking Multileader Locality, 4 leaders: 1.743688e-02 +Pairwise Multileader, 10 leaders: 1.034468e-02 +Pairwise Locality Aware, 10 leaders: 3.278595e-03 +Pairwise Multileader Locality, 10 leaders: 9.629539e-03 +Nonblocking Multileader, 10 leaders: 7.582757e-03 +Nonblocking Locality Aware, 10 leaders: 4.296801e-03 +Nonblocking Multileader Locality, 10 leaders: 8.368961e-03 +Pairwise Multileader, 20 leaders: 6.226603e-03 +Pairwise Locality Aware, 20 leaders: 4.076331e-03 +Pairwise Multileader Locality, 20 leaders: 5.140092e-03 +Nonblocking Multileader, 20 leaders: 4.061423e-03 +Nonblocking Locality Aware, 20 leaders: 6.947770e-02 +Nonblocking Multileader Locality, 20 leaders: 4.440517e-03 +Size 256 +PMPI: 1.217051e-02 +Pairwise: 8.258636e-03 +NonBlocking: 1.192107e-01 +Pairwise Hierarchical: 1.443543e-01 +Pairwise Node Aware: 5.980765e-03 +Nonblocking Hierarchical: 1.247764e-01 +Nonblocking Node Aware: 5.830111e-03 +Pairwise Multileader, 4 leaders: 4.943323e-02 +Pairwise Locality Aware, 4 leaders: 6.586367e-03 +Pairwise Multileader Locality, 4 leaders: 5.176054e-02 +Nonblocking Multileader, 4 leaders: 3.627339e-02 +Nonblocking Locality Aware, 4 leaders: 5.669754e-03 +Nonblocking Multileader Locality, 4 leaders: 4.169553e-02 +Pairwise Multileader, 10 leaders: 2.746748e-02 +Pairwise Locality Aware, 10 leaders: 7.174964e-03 +Pairwise Multileader Locality, 10 leaders: 2.278787e-02 +Nonblocking Multileader, 10 leaders: 1.759956e-02 +Nonblocking Locality Aware, 10 leaders: 9.485242e-03 +Nonblocking Multileader Locality, 10 leaders: 1.937620e-02 +Pairwise Multileader, 20 leaders: 1.195068e-02 +Pairwise Locality Aware, 20 leaders: 6.137245e-03 +Pairwise Multileader Locality, 20 leaders: 1.086665e-02 +Nonblocking Multileader, 20 leaders: 7.658081e-03 +Nonblocking Locality Aware, 20 leaders: 6.514931e-03 +Nonblocking Multileader Locality, 20 leaders: 9.141817e-03 +Size 512 +PMPI: 1.626067e-02 +Pairwise: 1.522162e-02 +NonBlocking: 1.409693e-01 +Pairwise Hierarchical: 2.744532e-01 +Pairwise Node Aware: 1.388373e-02 +Nonblocking Hierarchical: 2.490677e-01 +Nonblocking Node Aware: 1.311254e-02 +Pairwise Multileader, 4 leaders: 9.230078e-02 +Pairwise Locality Aware, 4 leaders: 1.321933e-02 +Pairwise Multileader Locality, 4 leaders: 9.945343e-02 +Nonblocking Multileader, 4 leaders: 6.799833e-02 +Nonblocking Locality Aware, 4 leaders: 1.150679e-02 +Nonblocking Multileader Locality, 4 leaders: 8.068939e-02 +Pairwise Multileader, 10 leaders: 4.999152e-02 +Pairwise Locality Aware, 10 leaders: 1.525930e-02 +Pairwise Multileader Locality, 10 leaders: 4.928719e-02 +Nonblocking Multileader, 10 leaders: 3.548051e-02 +Nonblocking Locality Aware, 10 leaders: 1.294853e-02 +Nonblocking Multileader Locality, 10 leaders: 4.144555e-02 +Pairwise Multileader, 20 leaders: 2.681916e-02 +Pairwise Locality Aware, 20 leaders: 1.446429e-02 +Pairwise Multileader Locality, 20 leaders: 2.428100e-02 +Nonblocking Multileader, 20 leaders: 1.795025e-02 +Nonblocking Locality Aware, 20 leaders: 2.205703e-02 +Nonblocking Multileader Locality, 20 leaders: 2.041725e-02 +Size 1024 +PMPI: 3.355808e-02 +Pairwise: 2.909846e-02 +NonBlocking: 1.278949e-01 +Pairwise Hierarchical: 5.174485e-01 +Pairwise Node Aware: 2.875355e-02 +Nonblocking Hierarchical: 5.120982e-01 +Nonblocking Node Aware: 2.486265e-02 +Pairwise Multileader, 4 leaders: 1.672548e-01 +Pairwise Locality Aware, 4 leaders: 2.644976e-02 +Pairwise Multileader Locality, 4 leaders: 1.868681e-01 +Nonblocking Multileader, 4 leaders: 1.247161e-01 +Nonblocking Locality Aware, 4 leaders: 2.041426e-02 +Nonblocking Multileader Locality, 4 leaders: 1.516575e-01 +Pairwise Multileader, 10 leaders: 9.881455e-02 +Pairwise Locality Aware, 10 leaders: 2.994245e-02 +Pairwise Multileader Locality, 10 leaders: 1.004943e-01 +Nonblocking Multileader, 10 leaders: 7.097379e-02 +Nonblocking Locality Aware, 10 leaders: 2.806164e-02 +Nonblocking Multileader Locality, 10 leaders: 8.498500e-02 +Pairwise Multileader, 20 leaders: 6.079928e-02 +Pairwise Locality Aware, 20 leaders: 2.826257e-02 +Pairwise Multileader Locality, 20 leaders: 5.082309e-02 +Nonblocking Multileader, 20 leaders: 3.704555e-02 +Nonblocking Locality Aware, 20 leaders: 2.050756e-02 +Nonblocking Multileader Locality, 20 leaders: 4.255623e-02 +Size 2048 +PMPI: 5.720381e-02 +Pairwise: 5.367289e-02 +NonBlocking: 1.180803e-01 +Pairwise Hierarchical: 1.109206e+00 +Pairwise Node Aware: 5.434365e-02 +Nonblocking Hierarchical: 1.069900e+00 +Nonblocking Node Aware: 4.483324e-02 +Pairwise Multileader, 4 leaders: 3.213961e-01 +Pairwise Locality Aware, 4 leaders: 5.333515e-02 +Pairwise Multileader Locality, 4 leaders: 3.526799e-01 +Nonblocking Multileader, 4 leaders: 2.462943e-01 +Nonblocking Locality Aware, 4 leaders: 4.137375e-02 +Nonblocking Multileader Locality, 4 leaders: 3.025101e-01 +Pairwise Multileader, 10 leaders: 1.813246e-01 +Pairwise Locality Aware, 10 leaders: 5.680679e-02 +Pairwise Multileader Locality, 10 leaders: 1.960345e-01 +Nonblocking Multileader, 10 leaders: 1.363682e-01 +Nonblocking Locality Aware, 10 leaders: 4.210551e-02 +Nonblocking Multileader Locality, 10 leaders: 1.660318e-01 +Pairwise Multileader, 20 leaders: 1.215442e-01 +Pairwise Locality Aware, 20 leaders: 5.646568e-02 +Pairwise Multileader Locality, 20 leaders: 1.136245e-01 +Nonblocking Multileader, 20 leaders: 8.225958e-02 +Nonblocking Locality Aware, 20 leaders: 4.891001e-02 +Nonblocking Multileader Locality, 20 leaders: 9.939373e-02 +Size 4096 +PMPI: 1.246555e-01 +Pairwise: 1.101897e-01 +NonBlocking: 1.930862e-01 +Pairwise Hierarchical: 2.322394e+00 +Pairwise Node Aware: 9.967997e-02 +Nonblocking Hierarchical: 2.157958e+00 +Nonblocking Node Aware: 9.009715e-02 +Pairwise Multileader, 4 leaders: 6.192988e-01 +Pairwise Locality Aware, 4 leaders: 1.068027e-01 +Pairwise Multileader Locality, 4 leaders: 7.216600e-01 +Nonblocking Multileader, 4 leaders: 5.297334e-01 +Nonblocking Locality Aware, 4 leaders: 8.845027e-02 +Nonblocking Multileader Locality, 4 leaders: 6.742049e-01 +Pairwise Multileader, 10 leaders: 3.686591e-01 +Pairwise Locality Aware, 10 leaders: 1.108920e-01 +Pairwise Multileader Locality, 10 leaders: 4.092572e-01 +Nonblocking Multileader, 10 leaders: 2.689090e-01 +Nonblocking Locality Aware, 10 leaders: 8.303398e-02 +Nonblocking Multileader Locality, 10 leaders: 3.347091e-01 +Pairwise Multileader, 20 leaders: 2.126807e-01 +Pairwise Locality Aware, 20 leaders: 1.155037e-01 +Pairwise Multileader Locality, 20 leaders: 2.192201e-01 +Nonblocking Multileader, 20 leaders: 1.602539e-01 +Nonblocking Locality Aware, 20 leaders: 8.820519e-02 +Nonblocking Multileader Locality, 20 leaders: 1.936453e-01 +Size 8192 +PMPI: 1.983414e-01 +Pairwise: 2.089195e-01 +NonBlocking: 1.842646e-01 diff --git a/runscripts/dane_results/alltoall_N8.2484418.err b/runscripts/dane_results/alltoall_N8.2484418.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/dane_results/alltoall_N8.2484418.out b/runscripts/dane_results/alltoall_N8.2484418.out new file mode 100644 index 000000000..84c6fd510 --- /dev/null +++ b/runscripts/dane_results/alltoall_N8.2484418.out @@ -0,0 +1,783 @@ +Test 1 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.595123e-04 +Pairwise: 2.341043e-03 +NonBlocking: 9.072834e-04 +Pairwise Hierarchical: 5.827138e-04 +Pairwise Node Aware: 2.787710e-04 +Nonblocking Hierarchical: 5.418186e-04 +Nonblocking Node Aware: 2.617694e-04 +Pairwise Multileader, 4 leaders: 3.465308e-04 +Pairwise Locality Aware, 4 leaders: 3.151610e-04 +Pairwise Multileader Locality, 4 leaders: 2.339392e-04 +Nonblocking Multileader, 4 leaders: 1.567388e-04 +Nonblocking Locality Aware, 4 leaders: 1.105947e-04 +Nonblocking Multileader Locality, 4 leaders: 1.810231e-04 +Pairwise Multileader, 10 leaders: 4.904207e-04 +Pairwise Locality Aware, 10 leaders: 5.085955e-04 +Pairwise Multileader Locality, 10 leaders: 1.941590e-04 +Nonblocking Multileader, 10 leaders: 1.384170e-04 +Nonblocking Locality Aware, 10 leaders: 2.164713e-04 +Nonblocking Multileader Locality, 10 leaders: 1.141591e-04 +Pairwise Multileader, 20 leaders: 1.161009e-03 +Pairwise Locality Aware, 20 leaders: 9.437381e-04 +Pairwise Multileader Locality, 20 leaders: 1.358007e-04 +Nonblocking Multileader, 20 leaders: 1.825186e-04 +Nonblocking Locality Aware, 20 leaders: 7.806981e-03 +Nonblocking Multileader Locality, 20 leaders: 8.711345e-05 +Size 2 +PMPI: 2.144424e-04 +Pairwise: 2.160829e-03 +NonBlocking: 8.762416e-04 +Pairwise Hierarchical: 1.240536e-03 +Pairwise Node Aware: 2.804588e-04 +Nonblocking Hierarchical: 1.070432e-03 +Nonblocking Node Aware: 2.203745e-04 +Pairwise Multileader, 4 leaders: 4.310766e-04 +Pairwise Locality Aware, 4 leaders: 3.142154e-04 +Pairwise Multileader Locality, 4 leaders: 4.161415e-04 +Nonblocking Multileader, 4 leaders: 2.601019e-04 +Nonblocking Locality Aware, 4 leaders: 1.134024e-04 +Nonblocking Multileader Locality, 4 leaders: 3.091440e-04 +Pairwise Multileader, 10 leaders: 5.927883e-04 +Pairwise Locality Aware, 10 leaders: 4.809025e-04 +Pairwise Multileader Locality, 10 leaders: 1.816270e-04 +Nonblocking Multileader, 10 leaders: 1.538082e-04 +Nonblocking Locality Aware, 10 leaders: 1.715791e-03 +Nonblocking Multileader Locality, 10 leaders: 1.331370e-04 +Pairwise Multileader, 20 leaders: 1.181084e-03 +Pairwise Locality Aware, 20 leaders: 8.590640e-04 +Pairwise Multileader Locality, 20 leaders: 1.555346e-04 +Nonblocking Multileader, 20 leaders: 2.483325e-04 +Nonblocking Locality Aware, 20 leaders: 1.919231e-03 +Nonblocking Multileader Locality, 20 leaders: 1.122262e-04 +Size 4 +PMPI: 3.527923e-04 +Pairwise: 4.038038e-03 +NonBlocking: 9.937362e-02 +Pairwise Hierarchical: 1.975381e-03 +Pairwise Node Aware: 4.014201e-04 +Nonblocking Hierarchical: 1.719150e-03 +Nonblocking Node Aware: 2.649465e-04 +Pairwise Multileader, 4 leaders: 7.195569e-04 +Pairwise Locality Aware, 4 leaders: 3.257071e-04 +Pairwise Multileader Locality, 4 leaders: 6.282707e-04 +Nonblocking Multileader, 4 leaders: 4.274583e-04 +Nonblocking Locality Aware, 4 leaders: 1.688112e-04 +Nonblocking Multileader Locality, 4 leaders: 5.114667e-04 +Pairwise Multileader, 10 leaders: 6.354671e-04 +Pairwise Locality Aware, 10 leaders: 5.483983e-04 +Pairwise Multileader Locality, 10 leaders: 3.103632e-04 +Nonblocking Multileader, 10 leaders: 9.310382e-04 +Nonblocking Locality Aware, 10 leaders: 7.834153e-03 +Nonblocking Multileader Locality, 10 leaders: 2.082314e-04 +Pairwise Multileader, 20 leaders: 1.100955e-03 +Pairwise Locality Aware, 20 leaders: 9.716267e-04 +Pairwise Multileader Locality, 20 leaders: 2.620125e-04 +Nonblocking Multileader, 20 leaders: 7.224559e-03 +Nonblocking Locality Aware, 20 leaders: 4.955917e-02 +Nonblocking Multileader Locality, 20 leaders: 1.834980e-04 +Size 8 +PMPI: 5.897987e-04 +Pairwise: 4.575990e-03 +NonBlocking: 1.363479e-01 +Pairwise Hierarchical: 3.724544e-03 +Pairwise Node Aware: 6.314809e-04 +Nonblocking Hierarchical: 3.316031e-03 +Nonblocking Node Aware: 4.760986e-04 +Pairwise Multileader, 4 leaders: 1.108177e-03 +Pairwise Locality Aware, 4 leaders: 4.919161e-04 +Pairwise Multileader Locality, 4 leaders: 1.581892e-03 +Nonblocking Multileader, 4 leaders: 8.632482e-04 +Nonblocking Locality Aware, 4 leaders: 3.459550e-04 +Nonblocking Multileader Locality, 4 leaders: 1.153314e-03 +Pairwise Multileader, 10 leaders: 8.673672e-04 +Pairwise Locality Aware, 10 leaders: 6.473084e-04 +Pairwise Multileader Locality, 10 leaders: 5.474518e-04 +Nonblocking Multileader, 10 leaders: 4.107296e-04 +Nonblocking Locality Aware, 10 leaders: 5.843985e-04 +Nonblocking Multileader Locality, 10 leaders: 4.244810e-04 +Pairwise Multileader, 20 leaders: 1.311864e-03 +Pairwise Locality Aware, 20 leaders: 1.100300e-03 +Pairwise Multileader Locality, 20 leaders: 3.440636e-04 +Nonblocking Multileader, 20 leaders: 3.602854e-03 +Nonblocking Locality Aware, 20 leaders: 7.450905e-02 +Nonblocking Multileader Locality, 20 leaders: 2.612727e-04 +Size 16 +PMPI: 1.003380e-03 +Pairwise: 4.161445e-03 +NonBlocking: 9.261387e-02 +Pairwise Hierarchical: 8.491808e-03 +Pairwise Node Aware: 7.326660e-04 +Nonblocking Hierarchical: 7.422624e-03 +Nonblocking Node Aware: 5.727044e-04 +Pairwise Multileader, 4 leaders: 2.438722e-03 +Pairwise Locality Aware, 4 leaders: 6.482240e-04 +Pairwise Multileader Locality, 4 leaders: 2.868449e-03 +Nonblocking Multileader, 4 leaders: 1.906322e-03 +Nonblocking Locality Aware, 4 leaders: 4.891716e-04 +Nonblocking Multileader Locality, 4 leaders: 2.402032e-03 +Pairwise Multileader, 10 leaders: 1.674367e-03 +Pairwise Locality Aware, 10 leaders: 1.170081e-03 +Pairwise Multileader Locality, 10 leaders: 1.080898e-03 +Nonblocking Multileader, 10 leaders: 8.945594e-04 +Nonblocking Locality Aware, 10 leaders: 2.465011e-03 +Nonblocking Multileader Locality, 10 leaders: 8.555762e-04 +Pairwise Multileader, 20 leaders: 1.609830e-03 +Pairwise Locality Aware, 20 leaders: 1.045984e-03 +Pairwise Multileader Locality, 20 leaders: 6.768195e-04 +Nonblocking Multileader, 20 leaders: 1.112307e-02 +Nonblocking Locality Aware, 20 leaders: 1.289931e-02 +Nonblocking Multileader Locality, 20 leaders: 5.787449e-04 +Size 32 +PMPI: 1.919315e-03 +Pairwise: 5.650338e-03 +NonBlocking: 1.289858e-01 +Pairwise Hierarchical: 1.848602e-02 +Pairwise Node Aware: 1.262376e-03 +Nonblocking Hierarchical: 1.594402e-02 +Nonblocking Node Aware: 1.265405e-03 +Pairwise Multileader, 4 leaders: 5.937700e-03 +Pairwise Locality Aware, 4 leaders: 1.102633e-03 +Pairwise Multileader Locality, 4 leaders: 5.623249e-03 +Nonblocking Multileader, 4 leaders: 4.440269e-03 +Nonblocking Locality Aware, 4 leaders: 1.042891e-03 +Nonblocking Multileader Locality, 4 leaders: 4.609465e-03 +Pairwise Multileader, 10 leaders: 2.331134e-03 +Pairwise Locality Aware, 10 leaders: 1.046553e-03 +Pairwise Multileader Locality, 10 leaders: 2.323378e-03 +Nonblocking Multileader, 10 leaders: 1.542141e-03 +Nonblocking Locality Aware, 10 leaders: 1.091109e-03 +Nonblocking Multileader Locality, 10 leaders: 1.957203e-03 +Pairwise Multileader, 20 leaders: 2.078458e-03 +Pairwise Locality Aware, 20 leaders: 1.228342e-03 +Pairwise Multileader Locality, 20 leaders: 1.208061e-03 +Nonblocking Multileader, 20 leaders: 1.820784e-03 +Nonblocking Locality Aware, 20 leaders: 7.444143e-02 +Nonblocking Multileader Locality, 20 leaders: 1.027273e-03 +Size 64 +PMPI: 3.750047e-03 +Pairwise: 5.118999e-03 +NonBlocking: 1.161016e-01 +Pairwise Hierarchical: 3.878890e-02 +Pairwise Node Aware: 2.090238e-03 +Nonblocking Hierarchical: 3.252872e-02 +Nonblocking Node Aware: 1.619737e-03 +Pairwise Multileader, 4 leaders: 1.229359e-02 +Pairwise Locality Aware, 4 leaders: 1.803904e-03 +Pairwise Multileader Locality, 4 leaders: 1.015843e-02 +Nonblocking Multileader, 4 leaders: 7.881136e-03 +Nonblocking Locality Aware, 4 leaders: 2.101848e-03 +Nonblocking Multileader Locality, 4 leaders: 8.332473e-03 +Pairwise Multileader, 10 leaders: 4.969958e-03 +Pairwise Locality Aware, 10 leaders: 1.919469e-03 +Pairwise Multileader Locality, 10 leaders: 4.691044e-03 +Nonblocking Multileader, 10 leaders: 3.453645e-03 +Nonblocking Locality Aware, 10 leaders: 1.912200e-03 +Nonblocking Multileader Locality, 10 leaders: 3.911973e-03 +Pairwise Multileader, 20 leaders: 3.113076e-03 +Pairwise Locality Aware, 20 leaders: 2.068159e-03 +Pairwise Multileader Locality, 20 leaders: 2.827202e-03 +Nonblocking Multileader, 20 leaders: 3.500831e-03 +Nonblocking Locality Aware, 20 leaders: 8.753037e-02 +Nonblocking Multileader Locality, 20 leaders: 1.961381e-03 +Size 128 +PMPI: 6.587353e-03 +Pairwise: 6.773577e-03 +NonBlocking: 1.393125e-01 +Pairwise Hierarchical: 7.641970e-02 +Pairwise Node Aware: 4.442634e-03 +Nonblocking Hierarchical: 6.341454e-02 +Nonblocking Node Aware: 2.858719e-03 +Pairwise Multileader, 4 leaders: 2.373071e-02 +Pairwise Locality Aware, 4 leaders: 3.093171e-03 +Pairwise Multileader Locality, 4 leaders: 2.220952e-02 +Nonblocking Multileader, 4 leaders: 1.548699e-02 +Nonblocking Locality Aware, 4 leaders: 3.409764e-03 +Nonblocking Multileader Locality, 4 leaders: 1.737689e-02 +Pairwise Multileader, 10 leaders: 1.080205e-02 +Pairwise Locality Aware, 10 leaders: 3.748436e-03 +Pairwise Multileader Locality, 10 leaders: 9.654820e-03 +Nonblocking Multileader, 10 leaders: 7.419077e-03 +Nonblocking Locality Aware, 10 leaders: 3.901051e-03 +Nonblocking Multileader Locality, 10 leaders: 8.352454e-03 +Pairwise Multileader, 20 leaders: 5.827092e-03 +Pairwise Locality Aware, 20 leaders: 4.301508e-03 +Pairwise Multileader Locality, 20 leaders: 5.397424e-03 +Nonblocking Multileader, 20 leaders: 3.559899e-03 +Nonblocking Locality Aware, 20 leaders: 6.421153e-02 +Nonblocking Multileader Locality, 20 leaders: 4.313694e-03 +Size 256 +PMPI: 1.189615e-02 +Pairwise: 9.083619e-03 +NonBlocking: 1.928049e-01 +Pairwise Hierarchical: 1.410454e-01 +Pairwise Node Aware: 6.055121e-03 +Nonblocking Hierarchical: 1.231184e-01 +Nonblocking Node Aware: 5.749962e-03 +Pairwise Multileader, 4 leaders: 4.994464e-02 +Pairwise Locality Aware, 4 leaders: 6.263804e-03 +Pairwise Multileader Locality, 4 leaders: 5.095470e-02 +Nonblocking Multileader, 4 leaders: 3.540888e-02 +Nonblocking Locality Aware, 4 leaders: 5.330543e-03 +Nonblocking Multileader Locality, 4 leaders: 4.018591e-02 +Pairwise Multileader, 10 leaders: 2.744159e-02 +Pairwise Locality Aware, 10 leaders: 6.930594e-03 +Pairwise Multileader Locality, 10 leaders: 2.279642e-02 +Nonblocking Multileader, 10 leaders: 1.774616e-02 +Nonblocking Locality Aware, 10 leaders: 8.265881e-03 +Nonblocking Multileader Locality, 10 leaders: 1.938714e-02 +Pairwise Multileader, 20 leaders: 1.259719e-02 +Pairwise Locality Aware, 20 leaders: 6.716046e-03 +Pairwise Multileader Locality, 20 leaders: 1.099744e-02 +Nonblocking Multileader, 20 leaders: 7.661636e-03 +Nonblocking Locality Aware, 20 leaders: 6.543500e-03 +Nonblocking Multileader Locality, 20 leaders: 9.295691e-03 +Size 512 +PMPI: 1.674484e-02 +Pairwise: 1.970157e-02 +NonBlocking: 1.695592e-01 +Pairwise Hierarchical: 2.686149e-01 +Pairwise Node Aware: 1.370127e-02 +Nonblocking Hierarchical: 2.442580e-01 +Nonblocking Node Aware: 1.592639e-02 +Pairwise Multileader, 4 leaders: 9.104226e-02 +Pairwise Locality Aware, 4 leaders: 1.498431e-02 +Pairwise Multileader Locality, 4 leaders: 9.943848e-02 +Nonblocking Multileader, 4 leaders: 6.710765e-02 +Nonblocking Locality Aware, 4 leaders: 1.185401e-02 +Nonblocking Multileader Locality, 4 leaders: 7.892737e-02 +Pairwise Multileader, 10 leaders: 5.235864e-02 +Pairwise Locality Aware, 10 leaders: 1.450518e-02 +Pairwise Multileader Locality, 10 leaders: 4.948146e-02 +Nonblocking Multileader, 10 leaders: 3.525036e-02 +Nonblocking Locality Aware, 10 leaders: 1.323553e-02 +Nonblocking Multileader Locality, 10 leaders: 4.118299e-02 +Pairwise Multileader, 20 leaders: 2.655842e-02 +Pairwise Locality Aware, 20 leaders: 1.375997e-02 +Pairwise Multileader Locality, 20 leaders: 2.440478e-02 +Nonblocking Multileader, 20 leaders: 1.755917e-02 +Nonblocking Locality Aware, 20 leaders: 1.764187e-02 +Nonblocking Multileader Locality, 20 leaders: 2.054681e-02 +Test 2 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.429111e-04 +Pairwise: 2.308493e-03 +NonBlocking: 7.722480e-04 +Pairwise Hierarchical: 6.518788e-04 +Pairwise Node Aware: 3.393320e-04 +Nonblocking Hierarchical: 5.452173e-04 +Nonblocking Node Aware: 2.452274e-04 +Pairwise Multileader, 4 leaders: 3.654759e-04 +Pairwise Locality Aware, 4 leaders: 2.854496e-04 +Pairwise Multileader Locality, 4 leaders: 2.492790e-04 +Nonblocking Multileader, 4 leaders: 2.064791e-04 +Nonblocking Locality Aware, 4 leaders: 1.398050e-04 +Nonblocking Multileader Locality, 4 leaders: 1.830481e-04 +Pairwise Multileader, 10 leaders: 8.076438e-04 +Pairwise Locality Aware, 10 leaders: 7.078896e-04 +Pairwise Multileader Locality, 10 leaders: 1.529084e-04 +Nonblocking Multileader, 10 leaders: 1.230266e-04 +Nonblocking Locality Aware, 10 leaders: 1.915785e-04 +Nonblocking Multileader Locality, 10 leaders: 1.371008e-04 +Pairwise Multileader, 20 leaders: 1.104598e-03 +Pairwise Locality Aware, 20 leaders: 1.035587e-03 +Pairwise Multileader Locality, 20 leaders: 1.326919e-04 +Nonblocking Multileader, 20 leaders: 2.252414e-03 +Nonblocking Locality Aware, 20 leaders: 2.489288e-02 +Nonblocking Multileader Locality, 20 leaders: 1.019976e-04 +Size 2 +PMPI: 2.168055e-04 +Pairwise: 3.679883e-03 +NonBlocking: 8.573435e-04 +Pairwise Hierarchical: 1.378207e-03 +Pairwise Node Aware: 3.311516e-04 +Nonblocking Hierarchical: 1.080359e-03 +Nonblocking Node Aware: 2.217825e-04 +Pairwise Multileader, 4 leaders: 4.446682e-04 +Pairwise Locality Aware, 4 leaders: 3.845879e-04 +Pairwise Multileader Locality, 4 leaders: 4.978875e-04 +Nonblocking Multileader, 4 leaders: 2.642145e-04 +Nonblocking Locality Aware, 4 leaders: 2.204905e-04 +Nonblocking Multileader Locality, 4 leaders: 2.648535e-04 +Pairwise Multileader, 10 leaders: 5.285043e-04 +Pairwise Locality Aware, 10 leaders: 5.363477e-04 +Pairwise Multileader Locality, 10 leaders: 1.735718e-04 +Nonblocking Multileader, 10 leaders: 1.487145e-04 +Nonblocking Locality Aware, 10 leaders: 1.402398e-03 +Nonblocking Multileader Locality, 10 leaders: 1.405461e-04 +Pairwise Multileader, 20 leaders: 8.640952e-04 +Pairwise Locality Aware, 20 leaders: 1.500669e-03 +Pairwise Multileader Locality, 20 leaders: 2.325759e-04 +Nonblocking Multileader, 20 leaders: 7.265396e-04 +Nonblocking Locality Aware, 20 leaders: 5.189747e-03 +Nonblocking Multileader Locality, 20 leaders: 1.295447e-04 +Size 4 +PMPI: 3.381051e-04 +Pairwise: 4.769072e-03 +NonBlocking: 1.150705e-01 +Pairwise Hierarchical: 2.117653e-03 +Pairwise Node Aware: 5.244540e-04 +Nonblocking Hierarchical: 1.862218e-03 +Nonblocking Node Aware: 3.121394e-04 +Pairwise Multileader, 4 leaders: 7.410874e-04 +Pairwise Locality Aware, 4 leaders: 3.354440e-04 +Pairwise Multileader Locality, 4 leaders: 6.388535e-04 +Nonblocking Multileader, 4 leaders: 4.315492e-04 +Nonblocking Locality Aware, 4 leaders: 3.915153e-04 +Nonblocking Multileader Locality, 4 leaders: 5.492365e-04 +Pairwise Multileader, 10 leaders: 7.735023e-04 +Pairwise Locality Aware, 10 leaders: 6.733648e-04 +Pairwise Multileader Locality, 10 leaders: 3.353074e-04 +Nonblocking Multileader, 10 leaders: 2.311376e-04 +Nonblocking Locality Aware, 10 leaders: 5.708717e-04 +Nonblocking Multileader Locality, 10 leaders: 2.684654e-04 +Pairwise Multileader, 20 leaders: 9.562198e-04 +Pairwise Locality Aware, 20 leaders: 1.313067e-03 +Pairwise Multileader Locality, 20 leaders: 2.461351e-04 +Nonblocking Multileader, 20 leaders: 1.064104e-02 +Nonblocking Locality Aware, 20 leaders: 8.656351e-02 +Nonblocking Multileader Locality, 20 leaders: 1.768366e-04 +Size 8 +PMPI: 9.193466e-04 +Pairwise: 8.948437e-03 +NonBlocking: 1.117838e-01 +Pairwise Hierarchical: 3.766014e-03 +Pairwise Node Aware: 5.508820e-04 +Nonblocking Hierarchical: 3.352782e-03 +Nonblocking Node Aware: 4.195468e-04 +Pairwise Multileader, 4 leaders: 1.226483e-03 +Pairwise Locality Aware, 4 leaders: 5.208014e-04 +Pairwise Multileader Locality, 4 leaders: 1.581078e-03 +Nonblocking Multileader, 4 leaders: 8.101535e-04 +Nonblocking Locality Aware, 4 leaders: 3.095849e-04 +Nonblocking Multileader Locality, 4 leaders: 1.203623e-03 +Pairwise Multileader, 10 leaders: 9.430170e-04 +Pairwise Locality Aware, 10 leaders: 6.842242e-04 +Pairwise Multileader Locality, 10 leaders: 6.489239e-04 +Nonblocking Multileader, 10 leaders: 3.745785e-04 +Nonblocking Locality Aware, 10 leaders: 4.951696e-04 +Nonblocking Multileader Locality, 10 leaders: 4.566195e-04 +Pairwise Multileader, 20 leaders: 1.502259e-03 +Pairwise Locality Aware, 20 leaders: 1.320626e-03 +Pairwise Multileader Locality, 20 leaders: 4.278910e-04 +Nonblocking Multileader, 20 leaders: 7.634071e-03 +Nonblocking Locality Aware, 20 leaders: 8.661323e-02 +Nonblocking Multileader Locality, 20 leaders: 4.789551e-04 +Size 16 +PMPI: 1.058340e-03 +Pairwise: 5.689095e-03 +NonBlocking: 1.295530e-01 +Pairwise Hierarchical: 8.673234e-03 +Pairwise Node Aware: 7.347109e-04 +Nonblocking Hierarchical: 7.549282e-03 +Nonblocking Node Aware: 6.799793e-04 +Pairwise Multileader, 4 leaders: 2.453245e-03 +Pairwise Locality Aware, 4 leaders: 6.599994e-04 +Pairwise Multileader Locality, 4 leaders: 2.994793e-03 +Nonblocking Multileader, 4 leaders: 1.863740e-03 +Nonblocking Locality Aware, 4 leaders: 5.870350e-04 +Nonblocking Multileader Locality, 4 leaders: 2.385284e-03 +Pairwise Multileader, 10 leaders: 1.386911e-03 +Pairwise Locality Aware, 10 leaders: 9.178883e-04 +Pairwise Multileader Locality, 10 leaders: 1.208358e-03 +Nonblocking Multileader, 10 leaders: 7.991536e-04 +Nonblocking Locality Aware, 10 leaders: 4.925123e-04 +Nonblocking Multileader Locality, 10 leaders: 9.082663e-04 +Pairwise Multileader, 20 leaders: 1.578474e-03 +Pairwise Locality Aware, 20 leaders: 1.434544e-03 +Pairwise Multileader Locality, 20 leaders: 6.563351e-04 +Nonblocking Multileader, 20 leaders: 6.240110e-03 +Nonblocking Locality Aware, 20 leaders: 4.975678e-02 +Nonblocking Multileader Locality, 20 leaders: 5.277909e-04 +Size 32 +PMPI: 2.574799e-03 +Pairwise: 5.764577e-03 +NonBlocking: 1.157644e-01 +Pairwise Hierarchical: 1.899271e-02 +Pairwise Node Aware: 1.248642e-03 +Nonblocking Hierarchical: 1.605422e-02 +Nonblocking Node Aware: 1.113965e-03 +Pairwise Multileader, 4 leaders: 6.319911e-03 +Pairwise Locality Aware, 4 leaders: 1.248580e-03 +Pairwise Multileader Locality, 4 leaders: 5.614826e-03 +Nonblocking Multileader, 4 leaders: 4.512532e-03 +Nonblocking Locality Aware, 4 leaders: 1.038778e-03 +Nonblocking Multileader Locality, 4 leaders: 4.680767e-03 +Pairwise Multileader, 10 leaders: 3.333351e-03 +Pairwise Locality Aware, 10 leaders: 1.565920e-03 +Pairwise Multileader Locality, 10 leaders: 2.359736e-03 +Nonblocking Multileader, 10 leaders: 1.557338e-03 +Nonblocking Locality Aware, 10 leaders: 4.842508e-03 +Nonblocking Multileader Locality, 10 leaders: 1.930365e-03 +Pairwise Multileader, 20 leaders: 1.861051e-03 +Pairwise Locality Aware, 20 leaders: 2.258601e-03 +Pairwise Multileader Locality, 20 leaders: 1.441766e-03 +Nonblocking Multileader, 20 leaders: 4.113329e-03 +Nonblocking Locality Aware, 20 leaders: 9.949464e-02 +Nonblocking Multileader Locality, 20 leaders: 1.002939e-03 +Size 64 +PMPI: 3.666571e-03 +Pairwise: 5.810937e-03 +NonBlocking: 1.158625e-01 +Pairwise Hierarchical: 3.951673e-02 +Pairwise Node Aware: 1.907734e-03 +Nonblocking Hierarchical: 3.261817e-02 +Nonblocking Node Aware: 1.435080e-03 +Pairwise Multileader, 4 leaders: 1.239531e-02 +Pairwise Locality Aware, 4 leaders: 2.294337e-03 +Pairwise Multileader Locality, 4 leaders: 1.005854e-02 +Nonblocking Multileader, 4 leaders: 7.850422e-03 +Nonblocking Locality Aware, 4 leaders: 2.054012e-03 +Nonblocking Multileader Locality, 4 leaders: 8.386303e-03 +Pairwise Multileader, 10 leaders: 4.845198e-03 +Pairwise Locality Aware, 10 leaders: 1.901571e-03 +Pairwise Multileader Locality, 10 leaders: 4.541169e-03 +Nonblocking Multileader, 10 leaders: 3.455051e-03 +Nonblocking Locality Aware, 10 leaders: 2.402031e-03 +Nonblocking Multileader Locality, 10 leaders: 3.849539e-03 +Pairwise Multileader, 20 leaders: 3.045419e-03 +Pairwise Locality Aware, 20 leaders: 2.209296e-03 +Pairwise Multileader Locality, 20 leaders: 2.666305e-03 +Nonblocking Multileader, 20 leaders: 2.035213e-03 +Nonblocking Locality Aware, 20 leaders: 8.772799e-02 +Nonblocking Multileader Locality, 20 leaders: 2.131122e-03 +Size 128 +PMPI: 6.111844e-03 +Pairwise: 8.289271e-03 +NonBlocking: 1.170062e-01 +Pairwise Hierarchical: 7.670202e-02 +Pairwise Node Aware: 3.980388e-03 +Nonblocking Hierarchical: 6.325288e-02 +Nonblocking Node Aware: 2.893613e-03 +Pairwise Multileader, 4 leaders: 2.405089e-02 +Pairwise Locality Aware, 4 leaders: 3.556506e-03 +Pairwise Multileader Locality, 4 leaders: 2.203973e-02 +Nonblocking Multileader, 4 leaders: 1.545603e-02 +Nonblocking Locality Aware, 4 leaders: 3.359062e-03 +Nonblocking Multileader Locality, 4 leaders: 1.766468e-02 +Pairwise Multileader, 10 leaders: 1.045460e-02 +Pairwise Locality Aware, 10 leaders: 3.275617e-03 +Pairwise Multileader Locality, 10 leaders: 9.661361e-03 +Nonblocking Multileader, 10 leaders: 7.404882e-03 +Nonblocking Locality Aware, 10 leaders: 3.919645e-03 +Nonblocking Multileader Locality, 10 leaders: 8.421777e-03 +Pairwise Multileader, 20 leaders: 5.824295e-03 +Pairwise Locality Aware, 20 leaders: 3.198101e-03 +Pairwise Multileader Locality, 20 leaders: 5.313497e-03 +Nonblocking Multileader, 20 leaders: 4.131706e-03 +Nonblocking Locality Aware, 20 leaders: 4.536316e-02 +Nonblocking Multileader Locality, 20 leaders: 4.364291e-03 +Size 256 +PMPI: 1.144388e-02 +Pairwise: 8.457740e-03 +NonBlocking: 1.653794e-01 +Pairwise Hierarchical: 1.421070e-01 +Pairwise Node Aware: 6.999060e-03 +Nonblocking Hierarchical: 1.232513e-01 +Nonblocking Node Aware: 5.630478e-03 +Pairwise Multileader, 4 leaders: 4.985034e-02 +Pairwise Locality Aware, 4 leaders: 5.814196e-03 +Pairwise Multileader Locality, 4 leaders: 5.212307e-02 +Nonblocking Multileader, 4 leaders: 3.595077e-02 +Nonblocking Locality Aware, 4 leaders: 5.343303e-03 +Nonblocking Multileader Locality, 4 leaders: 4.051068e-02 +Pairwise Multileader, 10 leaders: 2.681181e-02 +Pairwise Locality Aware, 10 leaders: 6.637976e-03 +Pairwise Multileader Locality, 10 leaders: 2.283716e-02 +Nonblocking Multileader, 10 leaders: 1.747267e-02 +Nonblocking Locality Aware, 10 leaders: 7.931501e-03 +Nonblocking Multileader Locality, 10 leaders: 1.941258e-02 +Pairwise Multileader, 20 leaders: 1.319834e-02 +Pairwise Locality Aware, 20 leaders: 6.797983e-03 +Pairwise Multileader Locality, 20 leaders: 1.097088e-02 +Nonblocking Multileader, 20 leaders: 7.928387e-03 +Nonblocking Locality Aware, 20 leaders: 1.079416e-02 +Nonblocking Multileader Locality, 20 leaders: 9.170643e-03 +Size 512 +PMPI: 1.693984e-02 +Pairwise: 1.770408e-02 +NonBlocking: 1.437298e-01 +Pairwise Hierarchical: 2.708869e-01 +Pairwise Node Aware: 1.297114e-02 +Nonblocking Hierarchical: 2.545112e-01 +Nonblocking Node Aware: 1.339886e-02 +Pairwise Multileader, 4 leaders: 9.185199e-02 +Pairwise Locality Aware, 4 leaders: 1.295573e-02 +Pairwise Multileader Locality, 4 leaders: 9.984586e-02 +Nonblocking Multileader, 4 leaders: 6.727304e-02 +Nonblocking Locality Aware, 4 leaders: 1.073322e-02 +Nonblocking Multileader Locality, 4 leaders: 7.885319e-02 +Pairwise Multileader, 10 leaders: 5.143323e-02 +Pairwise Locality Aware, 10 leaders: 1.400603e-02 +Pairwise Multileader Locality, 10 leaders: 4.925256e-02 +Nonblocking Multileader, 10 leaders: 3.527406e-02 +Nonblocking Locality Aware, 10 leaders: 1.254606e-02 +Nonblocking Multileader Locality, 10 leaders: 4.127118e-02 +Pairwise Multileader, 20 leaders: 2.534974e-02 +Pairwise Locality Aware, 20 leaders: 1.393816e-02 +Pairwise Multileader Locality, 20 leaders: 2.455653e-02 +Nonblocking Multileader, 20 leaders: 1.591834e-02 +Nonblocking Locality Aware, 20 leaders: 1.361992e-02 +Nonblocking Multileader Locality, 20 leaders: 2.016470e-02 +Test 3 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.465640e-04 +Pairwise: 2.866196e-03 +NonBlocking: 1.222988e-03 +Pairwise Hierarchical: 1.326596e-03 +Pairwise Node Aware: 3.074732e-04 +Nonblocking Hierarchical: 5.317434e-04 +Nonblocking Node Aware: 2.198674e-04 +Pairwise Multileader, 4 leaders: 3.372710e-04 +Pairwise Locality Aware, 4 leaders: 2.497725e-04 +Pairwise Multileader Locality, 4 leaders: 2.291295e-04 +Nonblocking Multileader, 4 leaders: 1.579540e-04 +Nonblocking Locality Aware, 4 leaders: 1.992363e-04 +Nonblocking Multileader Locality, 4 leaders: 1.900627e-04 +Pairwise Multileader, 10 leaders: 5.596136e-04 +Pairwise Locality Aware, 10 leaders: 1.061462e-03 +Pairwise Multileader Locality, 10 leaders: 1.771929e-04 +Nonblocking Multileader, 10 leaders: 1.209869e-04 +Nonblocking Locality Aware, 10 leaders: 7.501420e-04 +Nonblocking Multileader Locality, 10 leaders: 1.024834e-04 +Pairwise Multileader, 20 leaders: 1.130605e-03 +Pairwise Locality Aware, 20 leaders: 9.588503e-04 +Pairwise Multileader Locality, 20 leaders: 1.566817e-04 +Nonblocking Multileader, 20 leaders: 2.288140e-04 +Nonblocking Locality Aware, 20 leaders: 4.263108e-03 +Nonblocking Multileader Locality, 20 leaders: 8.544249e-05 +Size 2 +PMPI: 1.933441e-04 +Pairwise: 3.080588e-03 +NonBlocking: 8.526224e-04 +Pairwise Hierarchical: 1.247862e-03 +Pairwise Node Aware: 2.856481e-04 +Nonblocking Hierarchical: 1.070930e-03 +Nonblocking Node Aware: 2.178742e-04 +Pairwise Multileader, 4 leaders: 4.520012e-04 +Pairwise Locality Aware, 4 leaders: 3.127023e-04 +Pairwise Multileader Locality, 4 leaders: 4.205709e-04 +Nonblocking Multileader, 4 leaders: 2.671965e-04 +Nonblocking Locality Aware, 4 leaders: 1.557475e-04 +Nonblocking Multileader Locality, 4 leaders: 3.028257e-04 +Pairwise Multileader, 10 leaders: 4.886260e-04 +Pairwise Locality Aware, 10 leaders: 5.506270e-04 +Pairwise Multileader Locality, 10 leaders: 1.769506e-04 +Nonblocking Multileader, 10 leaders: 1.810555e-04 +Nonblocking Locality Aware, 10 leaders: 1.664903e-03 +Nonblocking Multileader Locality, 10 leaders: 1.428709e-04 +Pairwise Multileader, 20 leaders: 1.146950e-03 +Pairwise Locality Aware, 20 leaders: 9.376518e-04 +Pairwise Multileader Locality, 20 leaders: 1.978218e-04 +Nonblocking Multileader, 20 leaders: 8.881801e-04 +Nonblocking Locality Aware, 20 leaders: 1.541093e-02 +Nonblocking Multileader Locality, 20 leaders: 2.996887e-04 +Size 4 +PMPI: 3.873461e-04 +Pairwise: 6.016657e-03 +NonBlocking: 1.289525e-01 +Pairwise Hierarchical: 1.997911e-03 +Pairwise Node Aware: 5.690413e-04 +Nonblocking Hierarchical: 1.719211e-03 +Nonblocking Node Aware: 2.928791e-04 +Pairwise Multileader, 4 leaders: 8.716375e-04 +Pairwise Locality Aware, 4 leaders: 5.242195e-04 +Pairwise Multileader Locality, 4 leaders: 6.955590e-04 +Nonblocking Multileader, 4 leaders: 4.447693e-04 +Nonblocking Locality Aware, 4 leaders: 2.165334e-04 +Nonblocking Multileader Locality, 4 leaders: 5.492293e-04 +Pairwise Multileader, 10 leaders: 8.399890e-04 +Pairwise Locality Aware, 10 leaders: 8.726301e-04 +Pairwise Multileader Locality, 10 leaders: 3.383088e-04 +Nonblocking Multileader, 10 leaders: 3.411477e-04 +Nonblocking Locality Aware, 10 leaders: 2.515806e-03 +Nonblocking Multileader Locality, 10 leaders: 2.257545e-04 +Pairwise Multileader, 20 leaders: 1.940932e-03 +Pairwise Locality Aware, 20 leaders: 1.900148e-03 +Pairwise Multileader Locality, 20 leaders: 2.689107e-04 +Nonblocking Multileader, 20 leaders: 2.913317e-03 +Nonblocking Locality Aware, 20 leaders: 3.720491e-02 +Nonblocking Multileader Locality, 20 leaders: 2.074910e-04 +Size 8 +PMPI: 6.137353e-04 +Pairwise: 5.070091e-03 +NonBlocking: 1.839665e-01 +Pairwise Hierarchical: 3.752774e-03 +Pairwise Node Aware: 5.684407e-04 +Nonblocking Hierarchical: 3.345927e-03 +Nonblocking Node Aware: 4.276479e-04 +Pairwise Multileader, 4 leaders: 1.208718e-03 +Pairwise Locality Aware, 4 leaders: 5.108177e-04 +Pairwise Multileader Locality, 4 leaders: 1.661063e-03 +Nonblocking Multileader, 4 leaders: 8.429960e-04 +Nonblocking Locality Aware, 4 leaders: 3.367832e-04 +Nonblocking Multileader Locality, 4 leaders: 1.157349e-03 +Pairwise Multileader, 10 leaders: 7.785164e-04 +Pairwise Locality Aware, 10 leaders: 6.199939e-04 +Pairwise Multileader Locality, 10 leaders: 5.892245e-04 +Nonblocking Multileader, 10 leaders: 5.186738e-04 +Nonblocking Locality Aware, 10 leaders: 8.882251e-04 +Nonblocking Multileader Locality, 10 leaders: 4.387164e-04 +Pairwise Multileader, 20 leaders: 1.475102e-03 +Pairwise Locality Aware, 20 leaders: 1.266534e-03 +Pairwise Multileader Locality, 20 leaders: 3.667232e-04 +Nonblocking Multileader, 20 leaders: 2.547907e-03 +Nonblocking Locality Aware, 20 leaders: 4.971875e-02 +Nonblocking Multileader Locality, 20 leaders: 2.365357e-04 +Size 16 +PMPI: 1.000785e-03 +Pairwise: 8.947341e-03 +NonBlocking: 1.288819e-01 +Pairwise Hierarchical: 8.540747e-03 +Pairwise Node Aware: 7.262671e-04 +Nonblocking Hierarchical: 7.468137e-03 +Nonblocking Node Aware: 5.258089e-04 +Pairwise Multileader, 4 leaders: 2.426584e-03 +Pairwise Locality Aware, 4 leaders: 7.674931e-04 +Pairwise Multileader Locality, 4 leaders: 3.338532e-03 +Nonblocking Multileader, 4 leaders: 1.900875e-03 +Nonblocking Locality Aware, 4 leaders: 4.201122e-04 +Nonblocking Multileader Locality, 4 leaders: 2.360617e-03 +Pairwise Multileader, 10 leaders: 1.451366e-03 +Pairwise Locality Aware, 10 leaders: 8.428783e-04 +Pairwise Multileader Locality, 10 leaders: 1.172928e-03 +Nonblocking Multileader, 10 leaders: 8.669825e-04 +Nonblocking Locality Aware, 10 leaders: 1.734674e-02 +Nonblocking Multileader Locality, 10 leaders: 7.907921e-04 +Pairwise Multileader, 20 leaders: 1.509814e-03 +Pairwise Locality Aware, 20 leaders: 1.023258e-03 +Pairwise Multileader Locality, 20 leaders: 6.851750e-04 +Nonblocking Multileader, 20 leaders: 1.396942e-02 +Nonblocking Locality Aware, 20 leaders: 4.979587e-02 +Nonblocking Multileader Locality, 20 leaders: 5.669547e-04 +Size 32 +PMPI: 2.286985e-03 +Pairwise: 4.939879e-03 +NonBlocking: 1.155424e-01 +Pairwise Hierarchical: 1.864579e-02 +Pairwise Node Aware: 1.386699e-03 +Nonblocking Hierarchical: 1.633879e-02 +Nonblocking Node Aware: 9.665892e-04 +Pairwise Multileader, 4 leaders: 6.046645e-03 +Pairwise Locality Aware, 4 leaders: 1.039577e-03 +Pairwise Multileader Locality, 4 leaders: 5.604095e-03 +Nonblocking Multileader, 4 leaders: 4.512614e-03 +Nonblocking Locality Aware, 4 leaders: 8.374186e-04 +Nonblocking Multileader Locality, 4 leaders: 4.636167e-03 +Pairwise Multileader, 10 leaders: 2.347115e-03 +Pairwise Locality Aware, 10 leaders: 1.142538e-03 +Pairwise Multileader Locality, 10 leaders: 2.371481e-03 +Nonblocking Multileader, 10 leaders: 1.569612e-03 +Nonblocking Locality Aware, 10 leaders: 9.831968e-04 +Nonblocking Multileader Locality, 10 leaders: 1.941846e-03 +Pairwise Multileader, 20 leaders: 1.823599e-03 +Pairwise Locality Aware, 20 leaders: 1.328933e-03 +Pairwise Multileader Locality, 20 leaders: 1.267098e-03 +Nonblocking Multileader, 20 leaders: 1.028368e-03 +Nonblocking Locality Aware, 20 leaders: 8.677435e-02 +Nonblocking Multileader Locality, 20 leaders: 1.098692e-03 +Size 64 +PMPI: 3.067729e-03 +Pairwise: 5.056266e-03 +NonBlocking: 1.158491e-01 +Pairwise Hierarchical: 3.935327e-02 +Pairwise Node Aware: 1.962994e-03 +Nonblocking Hierarchical: 3.260232e-02 +Nonblocking Node Aware: 1.308569e-03 +Pairwise Multileader, 4 leaders: 1.220249e-02 +Pairwise Locality Aware, 4 leaders: 1.621145e-03 +Pairwise Multileader Locality, 4 leaders: 1.028880e-02 +Nonblocking Multileader, 4 leaders: 7.848507e-03 +Nonblocking Locality Aware, 4 leaders: 1.605733e-03 +Nonblocking Multileader Locality, 4 leaders: 8.433994e-03 +Pairwise Multileader, 10 leaders: 5.196455e-03 +Pairwise Locality Aware, 10 leaders: 1.970140e-03 +Pairwise Multileader Locality, 10 leaders: 4.490546e-03 +Nonblocking Multileader, 10 leaders: 3.399618e-03 +Nonblocking Locality Aware, 10 leaders: 1.976832e-03 +Nonblocking Multileader Locality, 10 leaders: 3.869483e-03 +Pairwise Multileader, 20 leaders: 3.210603e-03 +Pairwise Locality Aware, 20 leaders: 1.872318e-03 +Pairwise Multileader Locality, 20 leaders: 2.826149e-03 +Nonblocking Multileader, 20 leaders: 1.626689e-03 +Nonblocking Locality Aware, 20 leaders: 6.260395e-02 +Nonblocking Multileader Locality, 20 leaders: 2.189895e-03 +Size 128 +PMPI: 6.110418e-03 +Pairwise: 5.809613e-03 +NonBlocking: 9.448453e-02 +Pairwise Hierarchical: 7.678809e-02 +Pairwise Node Aware: 4.363079e-03 +Nonblocking Hierarchical: 6.343342e-02 +Nonblocking Node Aware: 2.819442e-03 +Pairwise Multileader, 4 leaders: 2.368876e-02 +Pairwise Locality Aware, 4 leaders: 3.107204e-03 +Pairwise Multileader Locality, 4 leaders: 2.214862e-02 +Nonblocking Multileader, 4 leaders: 1.559493e-02 +Nonblocking Locality Aware, 4 leaders: 3.571549e-03 +Nonblocking Multileader Locality, 4 leaders: 1.712899e-02 +Pairwise Multileader, 10 leaders: 9.871400e-03 +Pairwise Locality Aware, 10 leaders: 3.252498e-03 +Pairwise Multileader Locality, 10 leaders: 9.686016e-03 +Nonblocking Multileader, 10 leaders: 7.491011e-03 +Nonblocking Locality Aware, 10 leaders: 4.150141e-03 +Nonblocking Multileader Locality, 10 leaders: 8.469480e-03 +Pairwise Multileader, 20 leaders: 6.408804e-03 +Pairwise Locality Aware, 20 leaders: 3.712533e-03 +Pairwise Multileader Locality, 20 leaders: 5.119054e-03 +Nonblocking Multileader, 20 leaders: 3.369643e-03 +Nonblocking Locality Aware, 20 leaders: 1.138212e-02 +Nonblocking Multileader Locality, 20 leaders: 4.405544e-03 +Size 256 +PMPI: 1.343202e-02 +Pairwise: 1.067590e-02 +NonBlocking: 1.379329e-01 +Pairwise Hierarchical: 1.450012e-01 +Pairwise Node Aware: 5.896824e-03 +Nonblocking Hierarchical: 1.241363e-01 +Nonblocking Node Aware: 5.783089e-03 +Pairwise Multileader, 4 leaders: 4.975577e-02 +Pairwise Locality Aware, 4 leaders: 6.048134e-03 +Pairwise Multileader Locality, 4 leaders: 5.170697e-02 +Nonblocking Multileader, 4 leaders: 3.609163e-02 +Nonblocking Locality Aware, 4 leaders: 4.894851e-03 +Nonblocking Multileader Locality, 4 leaders: 4.026726e-02 +Pairwise Multileader, 10 leaders: 2.727097e-02 +Pairwise Locality Aware, 10 leaders: 8.000753e-03 +Pairwise Multileader Locality, 10 leaders: 2.285505e-02 +Nonblocking Multileader, 10 leaders: 1.753963e-02 +Nonblocking Locality Aware, 10 leaders: 8.874683e-03 +Nonblocking Multileader Locality, 10 leaders: 1.929085e-02 +Pairwise Multileader, 20 leaders: 1.213358e-02 +Pairwise Locality Aware, 20 leaders: 6.477408e-03 +Pairwise Multileader Locality, 20 leaders: 1.109015e-02 +Nonblocking Multileader, 20 leaders: 7.922589e-03 +Nonblocking Locality Aware, 20 leaders: 1.013537e-02 +Nonblocking Multileader Locality, 20 leaders: 9.204416e-03 +Size 512 +PMPI: 1.496280e-02 +Pairwise: 1.475721e-02 +NonBlocking: 1.173954e-01 +Pairwise Hierarchical: 2.712194e-01 +Pairwise Node Aware: 1.250387e-02 +Nonblocking Hierarchical: 2.479471e-01 +Nonblocking Node Aware: 1.131304e-02 +Pairwise Multileader, 4 leaders: 9.154402e-02 +Pairwise Locality Aware, 4 leaders: 1.254812e-02 +Pairwise Multileader Locality, 4 leaders: 1.009226e-01 +Nonblocking Multileader, 4 leaders: 6.776183e-02 +Nonblocking Locality Aware, 4 leaders: 1.186089e-02 +Nonblocking Multileader Locality, 4 leaders: 7.965174e-02 +Pairwise Multileader, 10 leaders: 4.977620e-02 +Pairwise Locality Aware, 10 leaders: 1.290842e-02 +Pairwise Multileader Locality, 10 leaders: 4.938837e-02 +Nonblocking Multileader, 10 leaders: 3.512841e-02 +Nonblocking Locality Aware, 10 leaders: 1.175482e-02 +Nonblocking Multileader Locality, 10 leaders: 4.105680e-02 +Pairwise Multileader, 20 leaders: 2.584741e-02 +Pairwise Locality Aware, 20 leaders: 1.251931e-02 +Pairwise Multileader Locality, 20 leaders: 2.444525e-02 +Nonblocking Multileader, 20 leaders: 1.564945e-02 +Nonblocking Locality Aware, 20 leaders: 1.350681e-02 +Nonblocking Multileader Locality, 20 leaders: 2.040874e-02 diff --git a/runscripts/dane_results/alltoall_N8.2484419.err b/runscripts/dane_results/alltoall_N8.2484419.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/dane_results/alltoall_N8.2484419.out b/runscripts/dane_results/alltoall_N8.2484419.out new file mode 100644 index 000000000..d6d43622f --- /dev/null +++ b/runscripts/dane_results/alltoall_N8.2484419.out @@ -0,0 +1,783 @@ +Test 1 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.517782e-04 +Pairwise: 2.137186e-03 +NonBlocking: 7.395765e-04 +Pairwise Hierarchical: 5.708942e-04 +Pairwise Node Aware: 2.709878e-04 +Nonblocking Hierarchical: 5.390697e-04 +Nonblocking Node Aware: 2.554375e-04 +Pairwise Multileader, 4 leaders: 2.766814e-04 +Pairwise Locality Aware, 4 leaders: 2.084785e-04 +Pairwise Multileader Locality, 4 leaders: 2.441464e-04 +Nonblocking Multileader, 4 leaders: 1.551288e-04 +Nonblocking Locality Aware, 4 leaders: 1.074139e-04 +Nonblocking Multileader Locality, 4 leaders: 1.786668e-04 +Pairwise Multileader, 10 leaders: 5.228220e-04 +Pairwise Locality Aware, 10 leaders: 5.606086e-04 +Pairwise Multileader Locality, 10 leaders: 1.365999e-04 +Nonblocking Multileader, 10 leaders: 1.201614e-04 +Nonblocking Locality Aware, 10 leaders: 2.737533e-04 +Nonblocking Multileader Locality, 10 leaders: 1.036219e-04 +Pairwise Multileader, 20 leaders: 7.844747e-04 +Pairwise Locality Aware, 20 leaders: 1.180915e-03 +Pairwise Multileader Locality, 20 leaders: 1.302687e-04 +Nonblocking Multileader, 20 leaders: 1.772287e-04 +Nonblocking Locality Aware, 20 leaders: 2.737783e-03 +Nonblocking Multileader Locality, 20 leaders: 8.815256e-05 +Size 2 +PMPI: 1.878806e-04 +Pairwise: 1.789750e-03 +NonBlocking: 6.879266e-04 +Pairwise Hierarchical: 1.280966e-03 +Pairwise Node Aware: 2.769514e-04 +Nonblocking Hierarchical: 1.081963e-03 +Nonblocking Node Aware: 2.215042e-04 +Pairwise Multileader, 4 leaders: 4.469561e-04 +Pairwise Locality Aware, 4 leaders: 2.244392e-04 +Pairwise Multileader Locality, 4 leaders: 4.019134e-04 +Nonblocking Multileader, 4 leaders: 2.676917e-04 +Nonblocking Locality Aware, 4 leaders: 1.446254e-04 +Nonblocking Multileader Locality, 4 leaders: 3.092211e-04 +Pairwise Multileader, 10 leaders: 4.981928e-04 +Pairwise Locality Aware, 10 leaders: 5.395851e-04 +Pairwise Multileader Locality, 10 leaders: 1.643634e-04 +Nonblocking Multileader, 10 leaders: 1.424475e-04 +Nonblocking Locality Aware, 10 leaders: 2.451453e-04 +Nonblocking Multileader Locality, 10 leaders: 1.437787e-04 +Pairwise Multileader, 20 leaders: 7.500208e-04 +Pairwise Locality Aware, 20 leaders: 8.043663e-04 +Pairwise Multileader Locality, 20 leaders: 1.800008e-04 +Nonblocking Multileader, 20 leaders: 3.132351e-04 +Nonblocking Locality Aware, 20 leaders: 1.949242e-03 +Nonblocking Multileader Locality, 20 leaders: 1.224455e-04 +Size 4 +PMPI: 3.064267e-04 +Pairwise: 4.768182e-03 +NonBlocking: 9.276349e-02 +Pairwise Hierarchical: 1.973075e-03 +Pairwise Node Aware: 3.983981e-04 +Nonblocking Hierarchical: 1.719525e-03 +Nonblocking Node Aware: 2.734396e-04 +Pairwise Multileader, 4 leaders: 6.706763e-04 +Pairwise Locality Aware, 4 leaders: 2.978084e-04 +Pairwise Multileader Locality, 4 leaders: 6.447913e-04 +Nonblocking Multileader, 4 leaders: 4.253058e-04 +Nonblocking Locality Aware, 4 leaders: 2.084475e-04 +Nonblocking Multileader Locality, 4 leaders: 5.411294e-04 +Pairwise Multileader, 10 leaders: 5.090484e-04 +Pairwise Locality Aware, 10 leaders: 4.237076e-04 +Pairwise Multileader Locality, 10 leaders: 2.987499e-04 +Nonblocking Multileader, 10 leaders: 2.186606e-04 +Nonblocking Locality Aware, 10 leaders: 3.375852e-04 +Nonblocking Multileader Locality, 10 leaders: 2.273480e-04 +Pairwise Multileader, 20 leaders: 1.339869e-03 +Pairwise Locality Aware, 20 leaders: 1.120297e-03 +Pairwise Multileader Locality, 20 leaders: 2.003109e-04 +Nonblocking Multileader, 20 leaders: 2.786746e-04 +Nonblocking Locality Aware, 20 leaders: 6.190841e-02 +Nonblocking Multileader Locality, 20 leaders: 1.451147e-04 +Size 8 +PMPI: 5.360841e-04 +Pairwise: 4.332961e-03 +NonBlocking: 9.964440e-02 +Pairwise Hierarchical: 4.505219e-03 +Pairwise Node Aware: 5.092267e-04 +Nonblocking Hierarchical: 3.382541e-03 +Nonblocking Node Aware: 4.298664e-04 +Pairwise Multileader, 4 leaders: 1.093637e-03 +Pairwise Locality Aware, 4 leaders: 3.157554e-04 +Pairwise Multileader Locality, 4 leaders: 1.526148e-03 +Nonblocking Multileader, 4 leaders: 8.257963e-04 +Nonblocking Locality Aware, 4 leaders: 3.183787e-04 +Nonblocking Multileader Locality, 4 leaders: 1.161113e-03 +Pairwise Multileader, 10 leaders: 8.862324e-04 +Pairwise Locality Aware, 10 leaders: 8.089160e-04 +Pairwise Multileader Locality, 10 leaders: 5.609296e-04 +Nonblocking Multileader, 10 leaders: 3.768847e-04 +Nonblocking Locality Aware, 10 leaders: 3.694790e-04 +Nonblocking Multileader Locality, 10 leaders: 4.477128e-04 +Pairwise Multileader, 20 leaders: 1.106451e-03 +Pairwise Locality Aware, 20 leaders: 1.187135e-03 +Pairwise Multileader Locality, 20 leaders: 3.917460e-04 +Nonblocking Multileader, 20 leaders: 1.386556e-03 +Nonblocking Locality Aware, 20 leaders: 7.430885e-02 +Nonblocking Multileader Locality, 20 leaders: 3.286713e-04 +Size 16 +PMPI: 9.912791e-04 +Pairwise: 3.809842e-03 +NonBlocking: 1.155195e-01 +Pairwise Hierarchical: 8.307472e-03 +Pairwise Node Aware: 7.466496e-04 +Nonblocking Hierarchical: 7.400386e-03 +Nonblocking Node Aware: 5.245924e-04 +Pairwise Multileader, 4 leaders: 2.348767e-03 +Pairwise Locality Aware, 4 leaders: 5.702595e-04 +Pairwise Multileader Locality, 4 leaders: 2.851684e-03 +Nonblocking Multileader, 4 leaders: 1.904928e-03 +Nonblocking Locality Aware, 4 leaders: 5.349897e-04 +Nonblocking Multileader Locality, 4 leaders: 2.335752e-03 +Pairwise Multileader, 10 leaders: 1.495465e-03 +Pairwise Locality Aware, 10 leaders: 7.068559e-04 +Pairwise Multileader Locality, 10 leaders: 1.063078e-03 +Nonblocking Multileader, 10 leaders: 8.250263e-04 +Nonblocking Locality Aware, 10 leaders: 3.615806e-03 +Nonblocking Multileader Locality, 10 leaders: 9.345402e-04 +Pairwise Multileader, 20 leaders: 2.698954e-03 +Pairwise Locality Aware, 20 leaders: 1.194846e-03 +Pairwise Multileader Locality, 20 leaders: 6.708973e-04 +Nonblocking Multileader, 20 leaders: 1.258568e-03 +Nonblocking Locality Aware, 20 leaders: 5.235979e-02 +Nonblocking Multileader Locality, 20 leaders: 5.355069e-04 +Size 32 +PMPI: 2.054689e-03 +Pairwise: 4.111730e-03 +NonBlocking: 1.245064e-01 +Pairwise Hierarchical: 1.866676e-02 +Pairwise Node Aware: 1.109505e-03 +Nonblocking Hierarchical: 1.584353e-02 +Nonblocking Node Aware: 1.066202e-03 +Pairwise Multileader, 4 leaders: 6.475520e-03 +Pairwise Locality Aware, 4 leaders: 1.271625e-03 +Pairwise Multileader Locality, 4 leaders: 5.622706e-03 +Nonblocking Multileader, 4 leaders: 4.483615e-03 +Nonblocking Locality Aware, 4 leaders: 8.665212e-04 +Nonblocking Multileader Locality, 4 leaders: 4.586644e-03 +Pairwise Multileader, 10 leaders: 2.492089e-03 +Pairwise Locality Aware, 10 leaders: 1.045264e-03 +Pairwise Multileader Locality, 10 leaders: 2.313743e-03 +Nonblocking Multileader, 10 leaders: 1.531677e-03 +Nonblocking Locality Aware, 10 leaders: 1.064783e-03 +Nonblocking Multileader Locality, 10 leaders: 2.162910e-03 +Pairwise Multileader, 20 leaders: 1.992061e-03 +Pairwise Locality Aware, 20 leaders: 1.339261e-03 +Pairwise Multileader Locality, 20 leaders: 1.310528e-03 +Nonblocking Multileader, 20 leaders: 7.948659e-04 +Nonblocking Locality Aware, 20 leaders: 8.674508e-02 +Nonblocking Multileader Locality, 20 leaders: 1.010575e-03 +Size 64 +PMPI: 3.313163e-03 +Pairwise: 6.329827e-03 +NonBlocking: 1.250520e-01 +Pairwise Hierarchical: 3.930357e-02 +Pairwise Node Aware: 1.986875e-03 +Nonblocking Hierarchical: 3.252374e-02 +Nonblocking Node Aware: 1.402364e-03 +Pairwise Multileader, 4 leaders: 1.263441e-02 +Pairwise Locality Aware, 4 leaders: 1.831435e-03 +Pairwise Multileader Locality, 4 leaders: 1.014681e-02 +Nonblocking Multileader, 4 leaders: 7.813604e-03 +Nonblocking Locality Aware, 4 leaders: 1.674057e-03 +Nonblocking Multileader Locality, 4 leaders: 1.003686e-02 +Pairwise Multileader, 10 leaders: 5.883706e-03 +Pairwise Locality Aware, 10 leaders: 2.953156e-03 +Pairwise Multileader Locality, 10 leaders: 4.641579e-03 +Nonblocking Multileader, 10 leaders: 3.364776e-03 +Nonblocking Locality Aware, 10 leaders: 1.965406e-03 +Nonblocking Multileader Locality, 10 leaders: 3.810626e-03 +Pairwise Multileader, 20 leaders: 3.274109e-03 +Pairwise Locality Aware, 20 leaders: 2.013693e-03 +Pairwise Multileader Locality, 20 leaders: 2.724827e-03 +Nonblocking Multileader, 20 leaders: 1.549604e-03 +Nonblocking Locality Aware, 20 leaders: 8.704006e-02 +Nonblocking Multileader Locality, 20 leaders: 2.156191e-03 +Size 128 +PMPI: 6.538797e-03 +Pairwise: 4.863436e-03 +NonBlocking: 1.173612e-01 +Pairwise Hierarchical: 7.690234e-02 +Pairwise Node Aware: 4.423299e-03 +Nonblocking Hierarchical: 6.309093e-02 +Nonblocking Node Aware: 3.058238e-03 +Pairwise Multileader, 4 leaders: 2.392804e-02 +Pairwise Locality Aware, 4 leaders: 2.957542e-03 +Pairwise Multileader Locality, 4 leaders: 2.243023e-02 +Nonblocking Multileader, 4 leaders: 1.559045e-02 +Nonblocking Locality Aware, 4 leaders: 2.960869e-03 +Nonblocking Multileader Locality, 4 leaders: 1.725299e-02 +Pairwise Multileader, 10 leaders: 1.096788e-02 +Pairwise Locality Aware, 10 leaders: 3.654963e-03 +Pairwise Multileader Locality, 10 leaders: 1.001199e-02 +Nonblocking Multileader, 10 leaders: 7.700107e-03 +Nonblocking Locality Aware, 10 leaders: 4.120882e-03 +Nonblocking Multileader Locality, 10 leaders: 8.554072e-03 +Pairwise Multileader, 20 leaders: 7.440492e-03 +Pairwise Locality Aware, 20 leaders: 4.618556e-03 +Pairwise Multileader Locality, 20 leaders: 5.593056e-03 +Nonblocking Multileader, 20 leaders: 8.394695e-03 +Nonblocking Locality Aware, 20 leaders: 6.349180e-02 +Nonblocking Multileader Locality, 20 leaders: 4.537526e-03 +Size 256 +PMPI: 1.293471e-02 +Pairwise: 1.535854e-02 +NonBlocking: 1.384201e-01 +Pairwise Hierarchical: 1.434198e-01 +Pairwise Node Aware: 6.505959e-03 +Nonblocking Hierarchical: 1.238470e-01 +Nonblocking Node Aware: 5.964041e-03 +Pairwise Multileader, 4 leaders: 4.951468e-02 +Pairwise Locality Aware, 4 leaders: 6.587102e-03 +Pairwise Multileader Locality, 4 leaders: 5.159656e-02 +Nonblocking Multileader, 4 leaders: 3.560564e-02 +Nonblocking Locality Aware, 4 leaders: 5.494068e-03 +Nonblocking Multileader Locality, 4 leaders: 4.028127e-02 +Pairwise Multileader, 10 leaders: 2.760452e-02 +Pairwise Locality Aware, 10 leaders: 7.045363e-03 +Pairwise Multileader Locality, 10 leaders: 2.301573e-02 +Nonblocking Multileader, 10 leaders: 1.757306e-02 +Nonblocking Locality Aware, 10 leaders: 8.444488e-03 +Nonblocking Multileader Locality, 10 leaders: 1.928345e-02 +Pairwise Multileader, 20 leaders: 1.278349e-02 +Pairwise Locality Aware, 20 leaders: 9.856084e-03 +Pairwise Multileader Locality, 20 leaders: 1.109212e-02 +Nonblocking Multileader, 20 leaders: 8.069740e-03 +Nonblocking Locality Aware, 20 leaders: 8.469688e-03 +Nonblocking Multileader Locality, 20 leaders: 9.558965e-03 +Size 512 +PMPI: 1.541982e-02 +Pairwise: 1.650264e-02 +NonBlocking: 1.163382e-01 +Pairwise Hierarchical: 2.734804e-01 +Pairwise Node Aware: 1.177432e-02 +Nonblocking Hierarchical: 2.466899e-01 +Nonblocking Node Aware: 1.216376e-02 +Pairwise Multileader, 4 leaders: 9.185078e-02 +Pairwise Locality Aware, 4 leaders: 1.268377e-02 +Pairwise Multileader Locality, 4 leaders: 1.001415e-01 +Nonblocking Multileader, 4 leaders: 6.708674e-02 +Nonblocking Locality Aware, 4 leaders: 1.151823e-02 +Nonblocking Multileader Locality, 4 leaders: 7.942823e-02 +Pairwise Multileader, 10 leaders: 4.988147e-02 +Pairwise Locality Aware, 10 leaders: 1.316699e-02 +Pairwise Multileader Locality, 10 leaders: 4.979936e-02 +Nonblocking Multileader, 10 leaders: 3.528106e-02 +Nonblocking Locality Aware, 10 leaders: 1.142995e-02 +Nonblocking Multileader Locality, 10 leaders: 4.164120e-02 +Pairwise Multileader, 20 leaders: 2.831608e-02 +Pairwise Locality Aware, 20 leaders: 2.035886e-02 +Pairwise Multileader Locality, 20 leaders: 2.430431e-02 +Nonblocking Multileader, 20 leaders: 1.567085e-02 +Nonblocking Locality Aware, 20 leaders: 1.447775e-02 +Nonblocking Multileader Locality, 20 leaders: 2.014235e-02 +Test 2 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 2.214887e-04 +Pairwise: 2.838046e-03 +NonBlocking: 1.032664e-03 +Pairwise Hierarchical: 6.317858e-04 +Pairwise Node Aware: 3.107386e-04 +Nonblocking Hierarchical: 5.433757e-04 +Nonblocking Node Aware: 2.330428e-04 +Pairwise Multileader, 4 leaders: 2.964279e-04 +Pairwise Locality Aware, 4 leaders: 2.703491e-04 +Pairwise Multileader Locality, 4 leaders: 3.100239e-04 +Nonblocking Multileader, 4 leaders: 1.807736e-04 +Nonblocking Locality Aware, 4 leaders: 1.520649e-04 +Nonblocking Multileader Locality, 4 leaders: 1.847236e-04 +Pairwise Multileader, 10 leaders: 4.716922e-04 +Pairwise Locality Aware, 10 leaders: 4.829287e-04 +Pairwise Multileader Locality, 10 leaders: 1.358766e-04 +Nonblocking Multileader, 10 leaders: 1.247667e-04 +Nonblocking Locality Aware, 10 leaders: 3.667670e-04 +Nonblocking Multileader Locality, 10 leaders: 1.093854e-04 +Pairwise Multileader, 20 leaders: 1.421416e-03 +Pairwise Locality Aware, 20 leaders: 1.492416e-03 +Pairwise Multileader Locality, 20 leaders: 1.279418e-04 +Nonblocking Multileader, 20 leaders: 1.758136e-04 +Nonblocking Locality Aware, 20 leaders: 7.610119e-03 +Nonblocking Multileader Locality, 20 leaders: 9.252772e-05 +Size 2 +PMPI: 2.125257e-04 +Pairwise: 3.465993e-03 +NonBlocking: 1.056662e-03 +Pairwise Hierarchical: 1.324412e-03 +Pairwise Node Aware: 2.863025e-04 +Nonblocking Hierarchical: 1.073829e-03 +Nonblocking Node Aware: 2.274392e-04 +Pairwise Multileader, 4 leaders: 4.333410e-04 +Pairwise Locality Aware, 4 leaders: 2.908006e-04 +Pairwise Multileader Locality, 4 leaders: 4.099511e-04 +Nonblocking Multileader, 4 leaders: 2.625284e-04 +Nonblocking Locality Aware, 4 leaders: 1.625407e-04 +Nonblocking Multileader Locality, 4 leaders: 3.074112e-04 +Pairwise Multileader, 10 leaders: 3.925524e-04 +Pairwise Locality Aware, 10 leaders: 5.098376e-04 +Pairwise Multileader Locality, 10 leaders: 1.701492e-04 +Nonblocking Multileader, 10 leaders: 1.526971e-04 +Nonblocking Locality Aware, 10 leaders: 2.218259e-04 +Nonblocking Multileader Locality, 10 leaders: 1.375521e-04 +Pairwise Multileader, 20 leaders: 8.883113e-04 +Pairwise Locality Aware, 20 leaders: 7.288491e-04 +Pairwise Multileader Locality, 20 leaders: 1.499418e-04 +Nonblocking Multileader, 20 leaders: 3.107042e-04 +Nonblocking Locality Aware, 20 leaders: 8.628496e-04 +Nonblocking Multileader Locality, 20 leaders: 1.041872e-04 +Size 4 +PMPI: 3.324858e-04 +Pairwise: 5.151928e-03 +NonBlocking: 9.253638e-02 +Pairwise Hierarchical: 1.960660e-03 +Pairwise Node Aware: 4.371022e-04 +Nonblocking Hierarchical: 1.723078e-03 +Nonblocking Node Aware: 2.804271e-04 +Pairwise Multileader, 4 leaders: 6.927244e-04 +Pairwise Locality Aware, 4 leaders: 2.544534e-04 +Pairwise Multileader Locality, 4 leaders: 6.247123e-04 +Nonblocking Multileader, 4 leaders: 4.328726e-04 +Nonblocking Locality Aware, 4 leaders: 1.785666e-04 +Nonblocking Multileader Locality, 4 leaders: 5.501494e-04 +Pairwise Multileader, 10 leaders: 5.523226e-04 +Pairwise Locality Aware, 10 leaders: 5.770920e-04 +Pairwise Multileader Locality, 10 leaders: 2.837856e-04 +Nonblocking Multileader, 10 leaders: 2.200546e-04 +Nonblocking Locality Aware, 10 leaders: 5.024135e-03 +Nonblocking Multileader Locality, 10 leaders: 2.350511e-04 +Pairwise Multileader, 20 leaders: 8.866488e-04 +Pairwise Locality Aware, 20 leaders: 8.564249e-04 +Pairwise Multileader Locality, 20 leaders: 1.983151e-04 +Nonblocking Multileader, 20 leaders: 3.145908e-03 +Nonblocking Locality Aware, 20 leaders: 7.432079e-02 +Nonblocking Multileader Locality, 20 leaders: 1.508795e-04 +Size 8 +PMPI: 5.977344e-04 +Pairwise: 4.489872e-03 +NonBlocking: 1.154815e-01 +Pairwise Hierarchical: 3.775355e-03 +Pairwise Node Aware: 5.960960e-04 +Nonblocking Hierarchical: 3.357488e-03 +Nonblocking Node Aware: 4.613724e-04 +Pairwise Multileader, 4 leaders: 1.068441e-03 +Pairwise Locality Aware, 4 leaders: 4.450174e-04 +Pairwise Multileader Locality, 4 leaders: 1.472017e-03 +Nonblocking Multileader, 4 leaders: 7.981042e-04 +Nonblocking Locality Aware, 4 leaders: 4.151264e-04 +Nonblocking Multileader Locality, 4 leaders: 1.159348e-03 +Pairwise Multileader, 10 leaders: 7.777151e-04 +Pairwise Locality Aware, 10 leaders: 5.696940e-04 +Pairwise Multileader Locality, 10 leaders: 5.413907e-04 +Nonblocking Multileader, 10 leaders: 4.023156e-04 +Nonblocking Locality Aware, 10 leaders: 1.566918e-02 +Nonblocking Multileader Locality, 10 leaders: 4.447015e-04 +Pairwise Multileader, 20 leaders: 1.233923e-03 +Pairwise Locality Aware, 20 leaders: 1.016545e-03 +Pairwise Multileader Locality, 20 leaders: 4.398277e-04 +Nonblocking Multileader, 20 leaders: 3.792676e-04 +Nonblocking Locality Aware, 20 leaders: 1.644715e-02 +Nonblocking Multileader Locality, 20 leaders: 2.280371e-04 +Size 16 +PMPI: 1.032109e-03 +Pairwise: 5.196251e-03 +NonBlocking: 9.257102e-02 +Pairwise Hierarchical: 8.450597e-03 +Pairwise Node Aware: 7.487372e-04 +Nonblocking Hierarchical: 7.444101e-03 +Nonblocking Node Aware: 5.544399e-04 +Pairwise Multileader, 4 leaders: 2.352466e-03 +Pairwise Locality Aware, 4 leaders: 4.408694e-04 +Pairwise Multileader Locality, 4 leaders: 2.817163e-03 +Nonblocking Multileader, 4 leaders: 1.842003e-03 +Nonblocking Locality Aware, 4 leaders: 3.552463e-04 +Nonblocking Multileader Locality, 4 leaders: 2.349206e-03 +Pairwise Multileader, 10 leaders: 1.325761e-03 +Pairwise Locality Aware, 10 leaders: 7.681876e-04 +Pairwise Multileader Locality, 10 leaders: 1.068569e-03 +Nonblocking Multileader, 10 leaders: 7.994286e-04 +Nonblocking Locality Aware, 10 leaders: 4.757434e-02 +Nonblocking Multileader Locality, 10 leaders: 9.365669e-04 +Pairwise Multileader, 20 leaders: 1.293266e-03 +Pairwise Locality Aware, 20 leaders: 1.132215e-03 +Pairwise Multileader Locality, 20 leaders: 6.444818e-04 +Nonblocking Multileader, 20 leaders: 1.835491e-02 +Nonblocking Locality Aware, 20 leaders: 6.203237e-02 +Nonblocking Multileader Locality, 20 leaders: 4.906506e-04 +Size 32 +PMPI: 2.075504e-03 +Pairwise: 6.462065e-03 +NonBlocking: 1.614877e-01 +Pairwise Hierarchical: 1.878214e-02 +Pairwise Node Aware: 1.192358e-03 +Nonblocking Hierarchical: 1.591597e-02 +Nonblocking Node Aware: 1.035629e-03 +Pairwise Multileader, 4 leaders: 6.234181e-03 +Pairwise Locality Aware, 4 leaders: 1.200711e-03 +Pairwise Multileader Locality, 4 leaders: 5.529624e-03 +Nonblocking Multileader, 4 leaders: 4.476874e-03 +Nonblocking Locality Aware, 4 leaders: 9.155997e-04 +Nonblocking Multileader Locality, 4 leaders: 4.654749e-03 +Pairwise Multileader, 10 leaders: 2.643274e-03 +Pairwise Locality Aware, 10 leaders: 1.302353e-03 +Pairwise Multileader Locality, 10 leaders: 2.275202e-03 +Nonblocking Multileader, 10 leaders: 1.733109e-03 +Nonblocking Locality Aware, 10 leaders: 2.329814e-03 +Nonblocking Multileader Locality, 10 leaders: 1.913171e-03 +Pairwise Multileader, 20 leaders: 2.319193e-03 +Pairwise Locality Aware, 20 leaders: 2.093493e-03 +Pairwise Multileader Locality, 20 leaders: 1.191066e-03 +Nonblocking Multileader, 20 leaders: 3.763940e-03 +Nonblocking Locality Aware, 20 leaders: 8.724325e-02 +Nonblocking Multileader Locality, 20 leaders: 1.019075e-03 +Size 64 +PMPI: 3.254374e-03 +Pairwise: 5.828704e-03 +NonBlocking: 1.162837e-01 +Pairwise Hierarchical: 3.924001e-02 +Pairwise Node Aware: 2.150646e-03 +Nonblocking Hierarchical: 3.261757e-02 +Nonblocking Node Aware: 1.597791e-03 +Pairwise Multileader, 4 leaders: 1.278350e-02 +Pairwise Locality Aware, 4 leaders: 2.057879e-03 +Pairwise Multileader Locality, 4 leaders: 1.005311e-02 +Nonblocking Multileader, 4 leaders: 7.839797e-03 +Nonblocking Locality Aware, 4 leaders: 2.302442e-03 +Nonblocking Multileader Locality, 4 leaders: 8.440588e-03 +Pairwise Multileader, 10 leaders: 5.211329e-03 +Pairwise Locality Aware, 10 leaders: 2.403905e-03 +Pairwise Multileader Locality, 10 leaders: 5.636438e-03 +Nonblocking Multileader, 10 leaders: 3.635851e-03 +Nonblocking Locality Aware, 10 leaders: 2.080487e-03 +Nonblocking Multileader Locality, 10 leaders: 4.036470e-03 +Pairwise Multileader, 20 leaders: 3.440624e-03 +Pairwise Locality Aware, 20 leaders: 1.972467e-03 +Pairwise Multileader Locality, 20 leaders: 2.662819e-03 +Nonblocking Multileader, 20 leaders: 1.628011e-03 +Nonblocking Locality Aware, 20 leaders: 7.481692e-02 +Nonblocking Multileader Locality, 20 leaders: 2.717164e-03 +Size 128 +PMPI: 6.411299e-03 +Pairwise: 5.826150e-03 +NonBlocking: 1.393852e-01 +Pairwise Hierarchical: 7.709676e-02 +Pairwise Node Aware: 3.897668e-03 +Nonblocking Hierarchical: 6.373131e-02 +Nonblocking Node Aware: 3.146822e-03 +Pairwise Multileader, 4 leaders: 2.462343e-02 +Pairwise Locality Aware, 4 leaders: 3.420981e-03 +Pairwise Multileader Locality, 4 leaders: 2.205421e-02 +Nonblocking Multileader, 4 leaders: 1.570378e-02 +Nonblocking Locality Aware, 4 leaders: 3.845917e-03 +Nonblocking Multileader Locality, 4 leaders: 1.729058e-02 +Pairwise Multileader, 10 leaders: 1.025996e-02 +Pairwise Locality Aware, 10 leaders: 3.419971e-03 +Pairwise Multileader Locality, 10 leaders: 9.706953e-03 +Nonblocking Multileader, 10 leaders: 7.414857e-03 +Nonblocking Locality Aware, 10 leaders: 4.146005e-03 +Nonblocking Multileader Locality, 10 leaders: 8.497596e-03 +Pairwise Multileader, 20 leaders: 5.957041e-03 +Pairwise Locality Aware, 20 leaders: 3.431499e-03 +Pairwise Multileader Locality, 20 leaders: 5.086868e-03 +Nonblocking Multileader, 20 leaders: 3.508181e-03 +Nonblocking Locality Aware, 20 leaders: 6.409861e-02 +Nonblocking Multileader Locality, 20 leaders: 4.689153e-03 +Size 256 +PMPI: 1.118628e-02 +Pairwise: 9.613161e-03 +NonBlocking: 1.207304e-01 +Pairwise Hierarchical: 1.433605e-01 +Pairwise Node Aware: 6.039558e-03 +Nonblocking Hierarchical: 1.243666e-01 +Nonblocking Node Aware: 5.685848e-03 +Pairwise Multileader, 4 leaders: 4.940397e-02 +Pairwise Locality Aware, 4 leaders: 6.733034e-03 +Pairwise Multileader Locality, 4 leaders: 5.167162e-02 +Nonblocking Multileader, 4 leaders: 3.557722e-02 +Nonblocking Locality Aware, 4 leaders: 4.848158e-03 +Nonblocking Multileader Locality, 4 leaders: 4.027998e-02 +Pairwise Multileader, 10 leaders: 2.734588e-02 +Pairwise Locality Aware, 10 leaders: 6.779358e-03 +Pairwise Multileader Locality, 10 leaders: 2.378527e-02 +Nonblocking Multileader, 10 leaders: 1.976987e-02 +Nonblocking Locality Aware, 10 leaders: 8.457894e-03 +Nonblocking Multileader Locality, 10 leaders: 1.945731e-02 +Pairwise Multileader, 20 leaders: 1.249134e-02 +Pairwise Locality Aware, 20 leaders: 6.326986e-03 +Pairwise Multileader Locality, 20 leaders: 1.086468e-02 +Nonblocking Multileader, 20 leaders: 7.618076e-03 +Nonblocking Locality Aware, 20 leaders: 7.058857e-03 +Nonblocking Multileader Locality, 20 leaders: 9.655613e-03 +Size 512 +PMPI: 1.576222e-02 +Pairwise: 1.663537e-02 +NonBlocking: 1.172828e-01 +Pairwise Hierarchical: 2.738185e-01 +Pairwise Node Aware: 1.387457e-02 +Nonblocking Hierarchical: 2.461831e-01 +Nonblocking Node Aware: 1.297257e-02 +Pairwise Multileader, 4 leaders: 9.188481e-02 +Pairwise Locality Aware, 4 leaders: 1.335440e-02 +Pairwise Multileader Locality, 4 leaders: 9.936916e-02 +Nonblocking Multileader, 4 leaders: 6.745814e-02 +Nonblocking Locality Aware, 4 leaders: 1.258176e-02 +Nonblocking Multileader Locality, 4 leaders: 7.984801e-02 +Pairwise Multileader, 10 leaders: 4.975483e-02 +Pairwise Locality Aware, 10 leaders: 1.463389e-02 +Pairwise Multileader Locality, 10 leaders: 4.880029e-02 +Nonblocking Multileader, 10 leaders: 3.539056e-02 +Nonblocking Locality Aware, 10 leaders: 1.404236e-02 +Nonblocking Multileader Locality, 10 leaders: 4.130556e-02 +Pairwise Multileader, 20 leaders: 2.660438e-02 +Pairwise Locality Aware, 20 leaders: 1.794670e-02 +Pairwise Multileader Locality, 20 leaders: 2.420113e-02 +Nonblocking Multileader, 20 leaders: 1.736299e-02 +Nonblocking Locality Aware, 20 leaders: 1.965976e-02 +Nonblocking Multileader Locality, 20 leaders: 2.033169e-02 +Test 3 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.728427e-04 +Pairwise: 2.098535e-03 +NonBlocking: 7.596941e-04 +Pairwise Hierarchical: 5.787336e-04 +Pairwise Node Aware: 2.778316e-04 +Nonblocking Hierarchical: 5.359879e-04 +Nonblocking Node Aware: 2.275719e-04 +Pairwise Multileader, 4 leaders: 2.988014e-04 +Pairwise Locality Aware, 4 leaders: 2.703987e-04 +Pairwise Multileader Locality, 4 leaders: 2.427323e-04 +Nonblocking Multileader, 4 leaders: 1.595459e-04 +Nonblocking Locality Aware, 4 leaders: 1.966755e-04 +Nonblocking Multileader Locality, 4 leaders: 1.914431e-04 +Pairwise Multileader, 10 leaders: 4.654750e-04 +Pairwise Locality Aware, 10 leaders: 4.769053e-04 +Pairwise Multileader Locality, 10 leaders: 1.479593e-04 +Nonblocking Multileader, 10 leaders: 1.202069e-04 +Nonblocking Locality Aware, 10 leaders: 4.660533e-04 +Nonblocking Multileader Locality, 10 leaders: 1.043860e-04 +Pairwise Multileader, 20 leaders: 8.890786e-04 +Pairwise Locality Aware, 20 leaders: 1.069580e-03 +Pairwise Multileader Locality, 20 leaders: 1.390102e-04 +Nonblocking Multileader, 20 leaders: 7.030271e-04 +Nonblocking Locality Aware, 20 leaders: 9.705513e-03 +Nonblocking Multileader Locality, 20 leaders: 8.366061e-05 +Size 2 +PMPI: 1.940049e-04 +Pairwise: 2.545955e-03 +NonBlocking: 6.991589e-04 +Pairwise Hierarchical: 1.236420e-03 +Pairwise Node Aware: 2.796836e-04 +Nonblocking Hierarchical: 1.099948e-03 +Nonblocking Node Aware: 2.253291e-04 +Pairwise Multileader, 4 leaders: 4.158071e-04 +Pairwise Locality Aware, 4 leaders: 2.816719e-04 +Pairwise Multileader Locality, 4 leaders: 4.057080e-04 +Nonblocking Multileader, 4 leaders: 2.653927e-04 +Nonblocking Locality Aware, 4 leaders: 1.720400e-04 +Nonblocking Multileader Locality, 4 leaders: 3.072802e-04 +Pairwise Multileader, 10 leaders: 4.324444e-04 +Pairwise Locality Aware, 10 leaders: 5.150189e-04 +Pairwise Multileader Locality, 10 leaders: 1.814841e-04 +Nonblocking Multileader, 10 leaders: 1.489173e-04 +Nonblocking Locality Aware, 10 leaders: 1.881007e-03 +Nonblocking Multileader Locality, 10 leaders: 1.370856e-04 +Pairwise Multileader, 20 leaders: 8.668556e-04 +Pairwise Locality Aware, 20 leaders: 9.142523e-04 +Pairwise Multileader Locality, 20 leaders: 1.611620e-04 +Nonblocking Multileader, 20 leaders: 2.306013e-04 +Nonblocking Locality Aware, 20 leaders: 8.804238e-04 +Nonblocking Multileader Locality, 20 leaders: 1.049771e-04 +Size 4 +PMPI: 3.233471e-04 +Pairwise: 4.072996e-03 +NonBlocking: 9.213809e-02 +Pairwise Hierarchical: 2.105269e-03 +Pairwise Node Aware: 4.174135e-04 +Nonblocking Hierarchical: 1.746279e-03 +Nonblocking Node Aware: 2.633704e-04 +Pairwise Multileader, 4 leaders: 7.296863e-04 +Pairwise Locality Aware, 4 leaders: 2.437505e-04 +Pairwise Multileader Locality, 4 leaders: 6.273263e-04 +Nonblocking Multileader, 4 leaders: 4.253166e-04 +Nonblocking Locality Aware, 4 leaders: 1.840610e-04 +Nonblocking Multileader Locality, 4 leaders: 5.117744e-04 +Pairwise Multileader, 10 leaders: 5.378483e-04 +Pairwise Locality Aware, 10 leaders: 7.758066e-04 +Pairwise Multileader Locality, 10 leaders: 2.879688e-04 +Nonblocking Multileader, 10 leaders: 4.249038e-04 +Nonblocking Locality Aware, 10 leaders: 7.675693e-03 +Nonblocking Multileader Locality, 10 leaders: 2.709212e-04 +Pairwise Multileader, 20 leaders: 1.239473e-03 +Pairwise Locality Aware, 20 leaders: 8.774427e-04 +Pairwise Multileader Locality, 20 leaders: 2.286514e-04 +Nonblocking Multileader, 20 leaders: 7.501507e-04 +Nonblocking Locality Aware, 20 leaders: 8.654075e-02 +Nonblocking Multileader Locality, 20 leaders: 1.846036e-04 +Size 8 +PMPI: 5.465270e-04 +Pairwise: 4.645316e-03 +NonBlocking: 9.232668e-02 +Pairwise Hierarchical: 3.751583e-03 +Pairwise Node Aware: 5.332107e-04 +Nonblocking Hierarchical: 3.321088e-03 +Nonblocking Node Aware: 3.949184e-04 +Pairwise Multileader, 4 leaders: 1.177097e-03 +Pairwise Locality Aware, 4 leaders: 3.533494e-04 +Pairwise Multileader Locality, 4 leaders: 1.407650e-03 +Nonblocking Multileader, 4 leaders: 7.973804e-04 +Nonblocking Locality Aware, 4 leaders: 3.057541e-04 +Nonblocking Multileader Locality, 4 leaders: 1.144804e-03 +Pairwise Multileader, 10 leaders: 8.399029e-04 +Pairwise Locality Aware, 10 leaders: 6.669880e-04 +Pairwise Multileader Locality, 10 leaders: 5.277113e-04 +Nonblocking Multileader, 10 leaders: 3.914087e-04 +Nonblocking Locality Aware, 10 leaders: 3.445879e-04 +Nonblocking Multileader Locality, 10 leaders: 4.430902e-04 +Pairwise Multileader, 20 leaders: 9.909089e-04 +Pairwise Locality Aware, 20 leaders: 9.775583e-04 +Pairwise Multileader Locality, 20 leaders: 3.583262e-04 +Nonblocking Multileader, 20 leaders: 2.840332e-04 +Nonblocking Locality Aware, 20 leaders: 1.457201e-02 +Nonblocking Multileader Locality, 20 leaders: 2.395928e-04 +Size 16 +PMPI: 1.002388e-03 +Pairwise: 3.630068e-03 +NonBlocking: 9.262300e-02 +Pairwise Hierarchical: 8.412258e-03 +Pairwise Node Aware: 7.049661e-04 +Nonblocking Hierarchical: 7.450379e-03 +Nonblocking Node Aware: 5.021852e-04 +Pairwise Multileader, 4 leaders: 2.252389e-03 +Pairwise Locality Aware, 4 leaders: 5.474218e-04 +Pairwise Multileader Locality, 4 leaders: 2.861095e-03 +Nonblocking Multileader, 4 leaders: 1.845213e-03 +Nonblocking Locality Aware, 4 leaders: 4.392547e-04 +Nonblocking Multileader Locality, 4 leaders: 2.364634e-03 +Pairwise Multileader, 10 leaders: 1.298737e-03 +Pairwise Locality Aware, 10 leaders: 6.640917e-04 +Pairwise Multileader Locality, 10 leaders: 1.077169e-03 +Nonblocking Multileader, 10 leaders: 7.762106e-04 +Nonblocking Locality Aware, 10 leaders: 5.168915e-04 +Nonblocking Multileader Locality, 10 leaders: 9.400041e-04 +Pairwise Multileader, 20 leaders: 1.465318e-03 +Pairwise Locality Aware, 20 leaders: 9.904321e-04 +Pairwise Multileader Locality, 20 leaders: 6.641315e-04 +Nonblocking Multileader, 20 leaders: 3.266309e-03 +Nonblocking Locality Aware, 20 leaders: 5.795880e-02 +Nonblocking Multileader Locality, 20 leaders: 5.628781e-04 +Size 32 +PMPI: 2.135775e-03 +Pairwise: 3.925199e-03 +NonBlocking: 1.156990e-01 +Pairwise Hierarchical: 1.891126e-02 +Pairwise Node Aware: 1.097124e-03 +Nonblocking Hierarchical: 1.593860e-02 +Nonblocking Node Aware: 9.544356e-04 +Pairwise Multileader, 4 leaders: 5.878700e-03 +Pairwise Locality Aware, 4 leaders: 8.687681e-04 +Pairwise Multileader Locality, 4 leaders: 5.580218e-03 +Nonblocking Multileader, 4 leaders: 4.442671e-03 +Nonblocking Locality Aware, 4 leaders: 8.690353e-04 +Nonblocking Multileader Locality, 4 leaders: 4.627992e-03 +Pairwise Multileader, 10 leaders: 2.444620e-03 +Pairwise Locality Aware, 10 leaders: 1.007481e-03 +Pairwise Multileader Locality, 10 leaders: 2.296988e-03 +Nonblocking Multileader, 10 leaders: 1.509010e-03 +Nonblocking Locality Aware, 10 leaders: 9.715649e-04 +Nonblocking Multileader Locality, 10 leaders: 1.926075e-03 +Pairwise Multileader, 20 leaders: 1.650414e-03 +Pairwise Locality Aware, 20 leaders: 1.341060e-03 +Pairwise Multileader Locality, 20 leaders: 1.197324e-03 +Nonblocking Multileader, 20 leaders: 1.156617e-03 +Nonblocking Locality Aware, 20 leaders: 8.678113e-02 +Nonblocking Multileader Locality, 20 leaders: 1.017742e-03 +Size 64 +PMPI: 3.230561e-03 +Pairwise: 5.288876e-03 +NonBlocking: 1.160978e-01 +Pairwise Hierarchical: 3.946867e-02 +Pairwise Node Aware: 2.032772e-03 +Nonblocking Hierarchical: 3.257106e-02 +Nonblocking Node Aware: 1.409561e-03 +Pairwise Multileader, 4 leaders: 1.222134e-02 +Pairwise Locality Aware, 4 leaders: 1.414608e-03 +Pairwise Multileader Locality, 4 leaders: 1.019949e-02 +Nonblocking Multileader, 4 leaders: 7.879105e-03 +Nonblocking Locality Aware, 4 leaders: 1.329934e-03 +Nonblocking Multileader Locality, 4 leaders: 8.396393e-03 +Pairwise Multileader, 10 leaders: 5.097948e-03 +Pairwise Locality Aware, 10 leaders: 1.877025e-03 +Pairwise Multileader Locality, 10 leaders: 4.388462e-03 +Nonblocking Multileader, 10 leaders: 3.380323e-03 +Nonblocking Locality Aware, 10 leaders: 2.192748e-03 +Nonblocking Multileader Locality, 10 leaders: 3.720316e-03 +Pairwise Multileader, 20 leaders: 3.431499e-03 +Pairwise Locality Aware, 20 leaders: 2.011847e-03 +Pairwise Multileader Locality, 20 leaders: 2.655069e-03 +Nonblocking Multileader, 20 leaders: 1.678925e-03 +Nonblocking Locality Aware, 20 leaders: 7.473661e-02 +Nonblocking Multileader Locality, 20 leaders: 2.180670e-03 +Size 128 +PMPI: 6.423394e-03 +Pairwise: 6.930314e-03 +NonBlocking: 1.171411e-01 +Pairwise Hierarchical: 7.673828e-02 +Pairwise Node Aware: 4.127291e-03 +Nonblocking Hierarchical: 6.370601e-02 +Nonblocking Node Aware: 2.718854e-03 +Pairwise Multileader, 4 leaders: 2.409791e-02 +Pairwise Locality Aware, 4 leaders: 2.853744e-03 +Pairwise Multileader Locality, 4 leaders: 2.236098e-02 +Nonblocking Multileader, 4 leaders: 1.558577e-02 +Nonblocking Locality Aware, 4 leaders: 3.210225e-03 +Nonblocking Multileader Locality, 4 leaders: 1.739152e-02 +Pairwise Multileader, 10 leaders: 1.042346e-02 +Pairwise Locality Aware, 10 leaders: 3.228134e-03 +Pairwise Multileader Locality, 10 leaders: 9.552376e-03 +Nonblocking Multileader, 10 leaders: 7.575872e-03 +Nonblocking Locality Aware, 10 leaders: 3.670628e-03 +Nonblocking Multileader Locality, 10 leaders: 8.418292e-03 +Pairwise Multileader, 20 leaders: 6.033748e-03 +Pairwise Locality Aware, 20 leaders: 2.897773e-03 +Pairwise Multileader Locality, 20 leaders: 5.062724e-03 +Nonblocking Multileader, 20 leaders: 3.361508e-03 +Nonblocking Locality Aware, 20 leaders: 7.377597e-03 +Nonblocking Multileader Locality, 20 leaders: 4.379151e-03 +Size 256 +PMPI: 1.130428e-02 +Pairwise: 8.013764e-03 +NonBlocking: 1.190892e-01 +Pairwise Hierarchical: 1.440216e-01 +Pairwise Node Aware: 5.721219e-03 +Nonblocking Hierarchical: 1.256300e-01 +Nonblocking Node Aware: 5.585076e-03 +Pairwise Multileader, 4 leaders: 4.976899e-02 +Pairwise Locality Aware, 4 leaders: 6.300151e-03 +Pairwise Multileader Locality, 4 leaders: 5.122715e-02 +Nonblocking Multileader, 4 leaders: 3.542991e-02 +Nonblocking Locality Aware, 4 leaders: 5.351200e-03 +Nonblocking Multileader Locality, 4 leaders: 4.010479e-02 +Pairwise Multileader, 10 leaders: 2.724702e-02 +Pairwise Locality Aware, 10 leaders: 8.099885e-03 +Pairwise Multileader Locality, 10 leaders: 2.288613e-02 +Nonblocking Multileader, 10 leaders: 1.761345e-02 +Nonblocking Locality Aware, 10 leaders: 9.320589e-03 +Nonblocking Multileader Locality, 10 leaders: 1.939467e-02 +Pairwise Multileader, 20 leaders: 1.185856e-02 +Pairwise Locality Aware, 20 leaders: 6.118701e-03 +Pairwise Multileader Locality, 20 leaders: 1.093665e-02 +Nonblocking Multileader, 20 leaders: 7.682083e-03 +Nonblocking Locality Aware, 20 leaders: 6.549160e-03 +Nonblocking Multileader Locality, 20 leaders: 9.143374e-03 +Size 512 +PMPI: 1.548597e-02 +Pairwise: 1.517967e-02 +NonBlocking: 1.180874e-01 +Pairwise Hierarchical: 2.726921e-01 +Pairwise Node Aware: 1.281712e-02 +Nonblocking Hierarchical: 2.440080e-01 +Nonblocking Node Aware: 1.314455e-02 +Pairwise Multileader, 4 leaders: 9.224580e-02 +Pairwise Locality Aware, 4 leaders: 1.173213e-02 +Pairwise Multileader Locality, 4 leaders: 1.008224e-01 +Nonblocking Multileader, 4 leaders: 6.786326e-02 +Nonblocking Locality Aware, 4 leaders: 1.045099e-02 +Nonblocking Multileader Locality, 4 leaders: 8.002820e-02 +Pairwise Multileader, 10 leaders: 4.973290e-02 +Pairwise Locality Aware, 10 leaders: 1.504152e-02 +Pairwise Multileader Locality, 10 leaders: 4.903637e-02 +Nonblocking Multileader, 10 leaders: 3.536666e-02 +Nonblocking Locality Aware, 10 leaders: 1.161280e-02 +Nonblocking Multileader Locality, 10 leaders: 4.159847e-02 +Pairwise Multileader, 20 leaders: 2.635140e-02 +Pairwise Locality Aware, 20 leaders: 1.627109e-02 +Pairwise Multileader Locality, 20 leaders: 2.427999e-02 +Nonblocking Multileader, 20 leaders: 1.734174e-02 +Nonblocking Locality Aware, 20 leaders: 2.222668e-02 +Nonblocking Multileader Locality, 20 leaders: 2.013535e-02 diff --git a/runscripts/dane_results/test_alltoall b/runscripts/dane_results/test_alltoall new file mode 100644 index 000000000..7172431e5 --- /dev/null +++ b/runscripts/dane_results/test_alltoall @@ -0,0 +1,27 @@ +#!/bin/bash + +#SBATCH --nodes=2 +#SBATCH --ntasks-per-node=100 +#SBATCH --output=alltoall_N2.%j.out +#SBATCH --error=alltoall_N2.%j.err +#SBATCH --account=UNM +#SBATCH --time=00:30:00 +##SBATCH --partition=pdebug +#SBATCH --cpus-per-task=1 +#SBATCH --exclusive + +cd $HOME/AdvanceAlltoall/build/benchmarks +nnodes=2 +ppn=100 +ntasks=$(( ${nnodes} * ${ppn} )) + +echo "Test 1 Running Alltoall Benchmark on ${nnodes} nodes and ${ntasks} tasks" +srun -n ${ntasks} -N ${nnodes} ./alltoall_topology + +echo "Test 2 Running Alltoall Benchmark on ${nnodes} nodes and ${ntasks} tasks" +srun -n ${ntasks} -N ${nnodes} ./alltoall_topology + +echo "Test 3 Running Alltoall Benchmark on ${nnodes} nodes and ${ntasks} tasks" +srun -n ${ntasks} -N ${nnodes} ./alltoall_topology + + diff --git a/runscripts/large_variation/alltoall_N16.2360546.err b/runscripts/large_variation/alltoall_N16.2360546.err new file mode 100644 index 000000000..13c846319 --- /dev/null +++ b/runscripts/large_variation/alltoall_N16.2360546.err @@ -0,0 +1,395 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2360546.0 ON dane170 CANCELLED AT 2025-04-28T15:14:21 *** +slurmstepd: error: *** STEP 2360546.0 ON dane170 CANCELLED AT 2025-04-28T15:14:21 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane882: task 905: Killed +srun: error: dane1223: tasks 1300-1399: Killed +srun: error: dane655: tasks 600-699: Killed +srun: error: dane776: tasks 800-899: Killed +srun: error: dane1477: tasks 1500-1599: Killed +srun: error: dane1078: tasks 1100-1199: Killed +srun: error: dane882: tasks 900-904,906-999: Killed +srun: error: dane1135: tasks 1200-1299: Killed +srun: error: dane689: tasks 700-799: Killed +srun: error: dane998: tasks 1000-1099: Killed +srun: error: dane648: tasks 500-599: Killed +srun: error: dane1449: tasks 1400-1499: Killed +srun: error: dane316: tasks 100-199: Killed +srun: error: dane170: tasks 0-99: Killed +srun: error: dane487: tasks 400-499: Killed +srun: error: dane391: tasks 200-299: Killed +srun: error: dane419: tasks 300-399: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2360546.1 ON dane170 CANCELLED AT 2025-04-28T15:18:09 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1477: task 1505: Killed +srun: error: dane487: task 405: Killed +srun: error: dane1223: tasks 1300-1399: Killed +srun: error: dane655: tasks 600-699: Killed +srun: error: dane776: tasks 800-899: Killed +srun: error: dane1477: tasks 1500-1504,1506-1599: Killed +srun: error: dane882: tasks 900-999: Killed +srun: error: dane1078: tasks 1100-1199: Killed +srun: error: dane1135: tasks 1200-1299: Killed +srun: error: dane998: tasks 1000-1099: Killed +srun: error: dane689: tasks 700-799: Killed +srun: error: dane648: tasks 500-599: Killed +srun: error: dane1449: tasks 1400-1499: Killed +srun: error: dane316: tasks 100-199: Killed +srun: error: dane170: tasks 0-99: Killed +srun: error: dane391: tasks 200-299: Killed +srun: error: dane487: tasks 400-404,406-499: Killed +srun: error: dane419: tasks 300-399: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2360546.2 ON dane170 CANCELLED AT 2025-04-28T15:21:48 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1223: tasks 1305-1306,1309,1313,1317: Killed +srun: error: dane1223: tasks 1300-1304,1307-1308,1310-1312,1314-1316,1318-1399: Killed +srun: error: dane655: tasks 600-699: Killed +srun: error: dane776: tasks 800-899: Killed +srun: error: dane1477: tasks 1500-1599: Killed +srun: error: dane882: tasks 900-999: Killed +srun: error: dane1078: tasks 1100-1199: Killed +srun: error: dane998: tasks 1000-1099: Killed +srun: error: dane1135: tasks 1200-1299: Killed +srun: error: dane689: tasks 700-799: Killed +srun: error: dane648: tasks 500-599: Killed +srun: error: dane1449: tasks 1400-1499: Killed +srun: error: dane316: tasks 100-199: Killed +srun: error: dane170: tasks 0-99: Killed +srun: error: dane487: tasks 400-499: Killed +srun: error: dane419: tasks 300-399: Killed +srun: error: dane391: tasks 200-299: Killed diff --git a/runscripts/large_variation/alltoall_N16.2360546.out b/runscripts/large_variation/alltoall_N16.2360546.out new file mode 100644 index 000000000..264d388bd --- /dev/null +++ b/runscripts/large_variation/alltoall_N16.2360546.out @@ -0,0 +1,519 @@ +Test 1 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 4.282932e-04 +Pairwise: 9.293447e-03 +NonBlocking: 1.217895e-02 +Pairwise Hierarchical: 9.871259e-03 +Pairwise Multileader: 9.148265e-03 +Pairwise Node Aware: 9.376041e-03 +Pairwise Locality Aware: 1.040587e-02 +Pairwise Multileader Locality: 1.100899e-02 +Nonblocking Hierarchical: 9.090560e-03 +Nonblocking Multileader: 1.208765e-02 +Nonblocking Node Aware: 6.982159e-03 +Nonblocking Locality Aware: 8.861602e-03 +Nonblocking Multileader Locality: 9.424993e-03 +Size 2 +PMPI: 6.378232e-04 +Pairwise: 1.276287e-02 +NonBlocking: 1.132682e-02 +Pairwise Hierarchical: 7.801236e-03 +Pairwise Multileader: 1.020924e-02 +Pairwise Node Aware: 1.203634e-02 +Pairwise Locality Aware: 1.227633e-02 +Pairwise Multileader Locality: 1.133643e-02 +Nonblocking Hierarchical: 8.685706e-03 +Nonblocking Multileader: 1.027055e-02 +Nonblocking Node Aware: 9.105569e-03 +Nonblocking Locality Aware: 8.265329e-03 +Nonblocking Multileader Locality: 1.193064e-02 +Size 4 +PMPI: 1.207764e-03 +Pairwise: 2.266764e-02 +NonBlocking: 1.970509e-02 +Pairwise Hierarchical: 2.402098e-02 +Pairwise Multileader: 1.586039e-02 +Pairwise Node Aware: 2.285025e-02 +Pairwise Locality Aware: 2.632511e-02 +Pairwise Multileader Locality: 2.291351e-02 +Nonblocking Hierarchical: 2.019869e-02 +Nonblocking Multileader: 2.315649e-02 +Nonblocking Node Aware: 2.982713e-02 +Nonblocking Locality Aware: 2.480363e-02 +Nonblocking Multileader Locality: 2.427742e-02 +Size 8 +PMPI: 1.475890e-03 +Pairwise: 1.755944e-02 +NonBlocking: 2.403059e-02 +Pairwise Hierarchical: 2.247145e-02 +Pairwise Multileader: 2.214480e-02 +Pairwise Node Aware: 1.856081e-02 +Pairwise Locality Aware: 1.712571e-02 +Pairwise Multileader Locality: 1.740167e-02 +Nonblocking Hierarchical: 1.751468e-02 +Nonblocking Multileader: 2.431199e-02 +Nonblocking Node Aware: 1.897780e-02 +Nonblocking Locality Aware: 1.876411e-02 +Nonblocking Multileader Locality: 1.581275e-02 +Size 16 +PMPI: 3.098835e-03 +Pairwise: 2.162809e-02 +NonBlocking: 1.886259e-02 +Pairwise Hierarchical: 1.978712e-02 +Pairwise Multileader: 1.997503e-02 +Pairwise Node Aware: 2.560836e-02 +Pairwise Locality Aware: 1.732545e-02 +Pairwise Multileader Locality: 1.917038e-02 +Nonblocking Hierarchical: 2.160126e-02 +Nonblocking Multileader: 1.921703e-02 +Nonblocking Node Aware: 2.100685e-02 +Nonblocking Locality Aware: 2.289450e-02 +Nonblocking Multileader Locality: 2.161624e-02 +Size 32 +PMPI: 1.280756e-02 +Pairwise: 1.984187e-02 +NonBlocking: 2.049493e-02 +Pairwise Hierarchical: 2.050396e-02 +Pairwise Multileader: 2.001601e-02 +Pairwise Node Aware: 2.424881e-02 +Pairwise Locality Aware: 2.100428e-02 +Pairwise Multileader Locality: 1.407969e-02 +Nonblocking Hierarchical: 2.337192e-02 +Nonblocking Multileader: 2.563717e-02 +Nonblocking Node Aware: 2.338766e-02 +Nonblocking Locality Aware: 2.405605e-02 +Nonblocking Multileader Locality: 2.310948e-02 +Size 64 +PMPI: 2.268497e-02 +Pairwise: 2.125047e-02 +NonBlocking: 2.233020e-02 +Pairwise Hierarchical: 2.655068e-02 +Pairwise Multileader: 2.860774e-02 +Pairwise Node Aware: 2.371891e-02 +Pairwise Locality Aware: 1.773084e-02 +Pairwise Multileader Locality: 2.541405e-02 +Nonblocking Hierarchical: 2.285157e-02 +Nonblocking Multileader: 2.398823e-02 +Nonblocking Node Aware: 2.019647e-02 +Nonblocking Locality Aware: 1.776544e-02 +Nonblocking Multileader Locality: 2.991044e-02 +Size 128 +PMPI: 1.599999e-02 +Pairwise: 2.643707e-02 +NonBlocking: 2.731203e-02 +Pairwise Hierarchical: 2.579598e-02 +Pairwise Multileader: 3.310370e-02 +Pairwise Node Aware: 2.697694e-02 +Pairwise Locality Aware: 2.431725e-02 +Pairwise Multileader Locality: 3.144107e-02 +Nonblocking Hierarchical: 2.084378e-02 +Nonblocking Multileader: 2.949322e-02 +Nonblocking Node Aware: 2.233088e-02 +Nonblocking Locality Aware: 2.688385e-02 +Nonblocking Multileader Locality: 2.459082e-02 +Size 256 +PMPI: 3.270835e-02 +Pairwise: 3.507441e-02 +NonBlocking: 3.615302e-02 +Pairwise Hierarchical: 3.327147e-02 +Pairwise Multileader: 3.601889e-02 +Pairwise Node Aware: 3.489797e-02 +Pairwise Locality Aware: 2.915324e-02 +Pairwise Multileader Locality: 3.830753e-02 +Nonblocking Hierarchical: 2.873065e-02 +Nonblocking Multileader: 3.587592e-02 +Nonblocking Node Aware: 3.171972e-02 +Nonblocking Locality Aware: 3.005376e-02 +Nonblocking Multileader Locality: 3.786503e-02 +Size 512 +PMPI: 1.494026e-01 +Pairwise: 4.674055e-02 +NonBlocking: 5.413103e-02 +Pairwise Hierarchical: 4.952328e-02 +Pairwise Multileader: 5.157208e-02 +Pairwise Node Aware: 4.750038e-02 +Pairwise Locality Aware: 4.661398e-02 +Pairwise Multileader Locality: 4.989347e-02 +Nonblocking Hierarchical: 4.725368e-02 +Nonblocking Multileader: 4.857639e-02 +Nonblocking Node Aware: 6.342228e-02 +Nonblocking Locality Aware: 5.866364e-02 +Nonblocking Multileader Locality: 5.474434e-02 +Size 1024 +PMPI: 8.967270e-02 +Pairwise: 9.291733e-02 +NonBlocking: 8.621158e-02 +Pairwise Hierarchical: 8.332126e-02 +Pairwise Multileader: 8.697446e-02 +Pairwise Node Aware: 9.960673e-02 +Pairwise Locality Aware: 8.881831e-02 +Pairwise Multileader Locality: 8.509099e-02 +Nonblocking Hierarchical: 9.376272e-02 +Nonblocking Multileader: 8.180718e-02 +Nonblocking Node Aware: 9.565328e-02 +Nonblocking Locality Aware: 8.816266e-02 +Nonblocking Multileader Locality: 8.239357e-02 +Size 2048 +PMPI: 2.217232e-01 +Pairwise: 1.647097e-01 +NonBlocking: 1.924019e-01 +Pairwise Hierarchical: 1.846667e-01 +Pairwise Multileader: 1.667900e-01 +Pairwise Node Aware: 2.151142e-01 +Pairwise Locality Aware: 1.798965e-01 +Pairwise Multileader Locality: 1.500419e-01 +Nonblocking Hierarchical: 1.889910e-01 +Nonblocking Multileader: 1.621058e-01 +Nonblocking Node Aware: 2.077194e-01 +Nonblocking Locality Aware: 1.630314e-01 +Nonblocking Multileader Locality: 1.736321e-01 +Size 4096 +PMPI: 3.266404e-01 +Pairwise: 3.777982e-01 +NonBlocking: 3.337358e-01 +Test 2 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 5.906249e-04 +Pairwise: 9.389604e-03 +NonBlocking: 1.106836e-02 +Pairwise Hierarchical: 1.104992e-02 +Pairwise Multileader: 9.641287e-03 +Pairwise Node Aware: 7.945767e-03 +Pairwise Locality Aware: 9.775229e-03 +Pairwise Multileader Locality: 1.197446e-02 +Nonblocking Hierarchical: 1.108573e-02 +Nonblocking Multileader: 1.036497e-02 +Nonblocking Node Aware: 8.961475e-03 +Nonblocking Locality Aware: 8.968312e-03 +Nonblocking Multileader Locality: 9.462178e-03 +Size 2 +PMPI: 9.627953e-04 +Pairwise: 8.519081e-03 +NonBlocking: 8.686210e-03 +Pairwise Hierarchical: 9.952923e-03 +Pairwise Multileader: 1.002504e-02 +Pairwise Node Aware: 8.982132e-03 +Pairwise Locality Aware: 1.228037e-02 +Pairwise Multileader Locality: 8.339155e-03 +Nonblocking Hierarchical: 1.064271e-02 +Nonblocking Multileader: 1.001275e-02 +Nonblocking Node Aware: 8.322625e-03 +Nonblocking Locality Aware: 8.064708e-03 +Nonblocking Multileader Locality: 9.028610e-03 +Size 4 +PMPI: 1.167798e-03 +Pairwise: 1.959129e-02 +NonBlocking: 2.021232e-02 +Pairwise Hierarchical: 2.065518e-02 +Pairwise Multileader: 1.775740e-02 +Pairwise Node Aware: 2.262259e-02 +Pairwise Locality Aware: 2.075905e-02 +Pairwise Multileader Locality: 2.089411e-02 +Nonblocking Hierarchical: 2.481823e-02 +Nonblocking Multileader: 2.050023e-02 +Nonblocking Node Aware: 1.577196e-02 +Nonblocking Locality Aware: 1.939185e-02 +Nonblocking Multileader Locality: 1.232248e-02 +Size 8 +PMPI: 1.670075e-03 +Pairwise: 2.178921e-02 +NonBlocking: 1.532307e-02 +Pairwise Hierarchical: 1.984606e-02 +Pairwise Multileader: 1.924886e-02 +Pairwise Node Aware: 2.033274e-02 +Pairwise Locality Aware: 1.895593e-02 +Pairwise Multileader Locality: 2.023322e-02 +Nonblocking Hierarchical: 1.326046e-02 +Nonblocking Multileader: 2.022780e-02 +Nonblocking Node Aware: 1.881513e-02 +Nonblocking Locality Aware: 1.814840e-02 +Nonblocking Multileader Locality: 1.909157e-02 +Size 16 +PMPI: 2.954349e-03 +Pairwise: 2.051507e-02 +NonBlocking: 1.772175e-02 +Pairwise Hierarchical: 1.583170e-02 +Pairwise Multileader: 1.739515e-02 +Pairwise Node Aware: 1.899328e-02 +Pairwise Locality Aware: 1.949912e-02 +Pairwise Multileader Locality: 2.416996e-02 +Nonblocking Hierarchical: 3.871919e-02 +Nonblocking Multileader: 2.363410e-02 +Nonblocking Node Aware: 2.695379e-02 +Nonblocking Locality Aware: 3.173716e-02 +Nonblocking Multileader Locality: 1.974038e-02 +Size 32 +PMPI: 7.540150e-03 +Pairwise: 2.261965e-02 +NonBlocking: 1.552025e-02 +Pairwise Hierarchical: 2.450485e-02 +Pairwise Multileader: 1.823606e-02 +Pairwise Node Aware: 1.661880e-02 +Pairwise Locality Aware: 2.077981e-02 +Pairwise Multileader Locality: 1.833457e-02 +Nonblocking Hierarchical: 1.809792e-02 +Nonblocking Multileader: 1.963959e-02 +Nonblocking Node Aware: 1.933943e-02 +Nonblocking Locality Aware: 1.377387e-02 +Nonblocking Multileader Locality: 2.096363e-02 +Size 64 +PMPI: 9.731388e-03 +Pairwise: 1.704428e-02 +NonBlocking: 2.726709e-02 +Pairwise Hierarchical: 1.750229e-02 +Pairwise Multileader: 2.016166e-02 +Pairwise Node Aware: 2.219416e-02 +Pairwise Locality Aware: 2.195769e-02 +Pairwise Multileader Locality: 2.404145e-02 +Nonblocking Hierarchical: 1.918230e-02 +Nonblocking Multileader: 1.679621e-02 +Nonblocking Node Aware: 2.733933e-02 +Nonblocking Locality Aware: 1.707185e-02 +Nonblocking Multileader Locality: 1.909498e-02 +Size 128 +PMPI: 1.612158e-02 +Pairwise: 2.002500e-02 +NonBlocking: 2.664122e-02 +Pairwise Hierarchical: 2.555822e-02 +Pairwise Multileader: 2.247479e-02 +Pairwise Node Aware: 2.283838e-02 +Pairwise Locality Aware: 2.362219e-02 +Pairwise Multileader Locality: 2.433763e-02 +Nonblocking Hierarchical: 2.994501e-02 +Nonblocking Multileader: 2.618880e-02 +Nonblocking Node Aware: 2.288979e-02 +Nonblocking Locality Aware: 2.496351e-02 +Nonblocking Multileader Locality: 2.316534e-02 +Size 256 +PMPI: 3.668101e-02 +Pairwise: 3.274677e-02 +NonBlocking: 3.451955e-02 +Pairwise Hierarchical: 3.026602e-02 +Pairwise Multileader: 3.203297e-02 +Pairwise Node Aware: 3.694673e-02 +Pairwise Locality Aware: 3.343804e-02 +Pairwise Multileader Locality: 3.366663e-02 +Nonblocking Hierarchical: 3.114570e-02 +Nonblocking Multileader: 3.456998e-02 +Nonblocking Node Aware: 3.420355e-02 +Nonblocking Locality Aware: 2.968354e-02 +Nonblocking Multileader Locality: 3.651989e-02 +Size 512 +PMPI: 1.700667e-01 +Pairwise: 5.482549e-02 +NonBlocking: 5.525276e-02 +Pairwise Hierarchical: 4.635852e-02 +Pairwise Multileader: 5.510746e-02 +Pairwise Node Aware: 4.793247e-02 +Pairwise Locality Aware: 5.166328e-02 +Pairwise Multileader Locality: 4.409791e-02 +Nonblocking Hierarchical: 4.812933e-02 +Nonblocking Multileader: 5.252889e-02 +Nonblocking Node Aware: 5.764621e-02 +Nonblocking Locality Aware: 6.543498e-02 +Nonblocking Multileader Locality: 4.804923e-02 +Size 1024 +PMPI: 1.006294e-01 +Pairwise: 8.286927e-02 +NonBlocking: 8.653426e-02 +Pairwise Hierarchical: 8.133422e-02 +Pairwise Multileader: 8.601003e-02 +Pairwise Node Aware: 8.442668e-02 +Pairwise Locality Aware: 1.389551e-01 +Pairwise Multileader Locality: 2.220170e-01 +Nonblocking Hierarchical: 2.115393e-01 +Nonblocking Multileader: 6.451607e-01 +Nonblocking Node Aware: 1.076519e-01 +Nonblocking Locality Aware: 8.231007e-02 +Nonblocking Multileader Locality: 9.016114e-02 +Size 2048 +PMPI: 1.678849e-01 +Pairwise: 1.576742e-01 +NonBlocking: 1.656308e-01 +Pairwise Hierarchical: 1.680000e-01 +Pairwise Multileader: 1.614030e-01 +Pairwise Node Aware: 1.612915e-01 +Pairwise Locality Aware: 1.686310e-01 +Pairwise Multileader Locality: 1.877647e-01 +Nonblocking Hierarchical: 1.584732e-01 +Nonblocking Multileader: 1.638112e-01 +Nonblocking Node Aware: 1.660944e-01 +Nonblocking Locality Aware: 1.617895e-01 +Nonblocking Multileader Locality: 1.951037e-01 +Size 4096 +PMPI: 3.978920e-01 +Pairwise: 3.310639e-01 +NonBlocking: 3.299132e-01 +Test 3 Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 4.359023e-04 +Pairwise: 1.175371e-02 +NonBlocking: 1.322705e-02 +Pairwise Hierarchical: 1.003485e-02 +Pairwise Multileader: 8.549226e-03 +Pairwise Node Aware: 1.219943e-02 +Pairwise Locality Aware: 1.074112e-02 +Pairwise Multileader Locality: 1.033318e-02 +Nonblocking Hierarchical: 1.280690e-02 +Nonblocking Multileader: 8.014599e-03 +Nonblocking Node Aware: 1.106987e-02 +Nonblocking Locality Aware: 9.330795e-03 +Nonblocking Multileader Locality: 7.867685e-03 +Size 2 +PMPI: 7.578336e-04 +Pairwise: 9.969294e-03 +NonBlocking: 8.248979e-03 +Pairwise Hierarchical: 1.002339e-02 +Pairwise Multileader: 1.212365e-02 +Pairwise Node Aware: 1.046183e-02 +Pairwise Locality Aware: 7.309752e-03 +Pairwise Multileader Locality: 1.035007e-02 +Nonblocking Hierarchical: 1.096658e-02 +Nonblocking Multileader: 9.086163e-03 +Nonblocking Node Aware: 8.009307e-03 +Nonblocking Locality Aware: 1.067621e-02 +Nonblocking Multileader Locality: 9.565277e-03 +Size 4 +PMPI: 1.246887e-03 +Pairwise: 2.147951e-02 +NonBlocking: 2.437786e-02 +Pairwise Hierarchical: 2.087918e-02 +Pairwise Multileader: 2.643046e-02 +Pairwise Node Aware: 1.798223e-02 +Pairwise Locality Aware: 1.714161e-02 +Pairwise Multileader Locality: 2.426967e-02 +Nonblocking Hierarchical: 2.772896e-02 +Nonblocking Multileader: 2.958378e-02 +Nonblocking Node Aware: 2.555775e-02 +Nonblocking Locality Aware: 1.568129e-02 +Nonblocking Multileader Locality: 1.678880e-02 +Size 8 +PMPI: 2.289429e-03 +Pairwise: 1.675915e-02 +NonBlocking: 1.915911e-02 +Pairwise Hierarchical: 1.649006e-02 +Pairwise Multileader: 2.121319e-02 +Pairwise Node Aware: 2.517709e-02 +Pairwise Locality Aware: 1.371041e-02 +Pairwise Multileader Locality: 1.435492e-02 +Nonblocking Hierarchical: 2.118215e-02 +Nonblocking Multileader: 2.274397e-02 +Nonblocking Node Aware: 2.539487e-02 +Nonblocking Locality Aware: 1.747634e-02 +Nonblocking Multileader Locality: 1.729574e-02 +Size 16 +PMPI: 3.461136e-03 +Pairwise: 2.340712e-02 +NonBlocking: 2.035067e-02 +Pairwise Hierarchical: 1.749823e-02 +Pairwise Multileader: 1.873797e-02 +Pairwise Node Aware: 1.962757e-02 +Pairwise Locality Aware: 2.846257e-02 +Pairwise Multileader Locality: 2.549756e-02 +Nonblocking Hierarchical: 2.245884e-02 +Nonblocking Multileader: 2.386617e-02 +Nonblocking Node Aware: 2.946771e-02 +Nonblocking Locality Aware: 2.237715e-02 +Nonblocking Multileader Locality: 1.865713e-02 +Size 32 +PMPI: 5.288029e-03 +Pairwise: 2.115582e-02 +NonBlocking: 2.125613e-02 +Pairwise Hierarchical: 2.279939e-02 +Pairwise Multileader: 2.649328e-02 +Pairwise Node Aware: 2.385951e-02 +Pairwise Locality Aware: 2.344703e-02 +Pairwise Multileader Locality: 2.371728e-02 +Nonblocking Hierarchical: 2.381530e-02 +Nonblocking Multileader: 2.141458e-02 +Nonblocking Node Aware: 1.934189e-02 +Nonblocking Locality Aware: 2.672495e-02 +Nonblocking Multileader Locality: 2.539157e-02 +Size 64 +PMPI: 9.568081e-03 +Pairwise: 2.260344e-02 +NonBlocking: 2.461607e-02 +Pairwise Hierarchical: 2.417563e-02 +Pairwise Multileader: 2.225716e-02 +Pairwise Node Aware: 2.368850e-02 +Pairwise Locality Aware: 2.123325e-02 +Pairwise Multileader Locality: 2.758755e-02 +Nonblocking Hierarchical: 2.519123e-02 +Nonblocking Multileader: 1.972512e-02 +Nonblocking Node Aware: 2.412973e-02 +Nonblocking Locality Aware: 2.709909e-02 +Nonblocking Multileader Locality: 2.206732e-02 +Size 128 +PMPI: 1.879865e-02 +Pairwise: 2.910541e-02 +NonBlocking: 2.730888e-02 +Pairwise Hierarchical: 2.133854e-02 +Pairwise Multileader: 3.291651e-02 +Pairwise Node Aware: 2.621247e-02 +Pairwise Locality Aware: 2.171749e-02 +Pairwise Multileader Locality: 2.920035e-02 +Nonblocking Hierarchical: 2.607121e-02 +Nonblocking Multileader: 2.899415e-02 +Nonblocking Node Aware: 2.551796e-02 +Nonblocking Locality Aware: 2.610320e-02 +Nonblocking Multileader Locality: 2.777311e-02 +Size 256 +PMPI: 3.411183e-02 +Pairwise: 4.349078e-02 +NonBlocking: 3.825512e-02 +Pairwise Hierarchical: 3.610801e-02 +Pairwise Multileader: 4.681920e-02 +Pairwise Node Aware: 3.363502e-02 +Pairwise Locality Aware: 3.707406e-02 +Pairwise Multileader Locality: 3.632071e-02 +Nonblocking Hierarchical: 3.409117e-02 +Nonblocking Multileader: 3.456300e-02 +Nonblocking Node Aware: 3.670321e-02 +Nonblocking Locality Aware: 4.227703e-02 +Nonblocking Multileader Locality: 3.515215e-02 +Size 512 +PMPI: 2.015927e-01 +Pairwise: 5.454460e-02 +NonBlocking: 5.554961e-02 +Pairwise Hierarchical: 5.517336e-02 +Pairwise Multileader: 5.186946e-02 +Pairwise Node Aware: 4.973353e-02 +Pairwise Locality Aware: 5.122765e-02 +Pairwise Multileader Locality: 5.270199e-02 +Nonblocking Hierarchical: 5.147902e-02 +Nonblocking Multileader: 5.459856e-02 +Nonblocking Node Aware: 5.512122e-02 +Nonblocking Locality Aware: 5.485710e-02 +Nonblocking Multileader Locality: 5.022135e-02 +Size 1024 +PMPI: 1.541836e-01 +Pairwise: 9.541278e-02 +NonBlocking: 9.555984e-02 +Pairwise Hierarchical: 1.097366e-01 +Pairwise Multileader: 9.320651e-02 +Pairwise Node Aware: 1.309149e-01 +Pairwise Locality Aware: 8.726228e-02 +Pairwise Multileader Locality: 1.030589e-01 +Nonblocking Hierarchical: 8.441434e-02 +Nonblocking Multileader: 7.992716e-02 +Nonblocking Node Aware: 9.265002e-02 +Nonblocking Locality Aware: 1.165579e-01 +Nonblocking Multileader Locality: 9.042878e-02 +Size 2048 +PMPI: 1.670399e-01 +Pairwise: 1.699790e-01 +NonBlocking: 1.527307e-01 +Pairwise Hierarchical: 1.532082e-01 +Pairwise Multileader: 2.178731e-01 +Pairwise Node Aware: 1.590399e-01 +Pairwise Locality Aware: 1.657213e-01 +Pairwise Multileader Locality: 1.616196e-01 +Nonblocking Hierarchical: 1.690065e-01 +Nonblocking Multileader: 2.160147e-01 +Nonblocking Node Aware: 1.995846e-01 +Nonblocking Locality Aware: 1.815793e-01 +Nonblocking Multileader Locality: 1.598647e-01 +Size 4096 +PMPI: 3.687642e-01 +Pairwise: 3.352640e-01 +NonBlocking: 3.321205e-01 diff --git a/runscripts/large_variation/alltoall_N2.2360533.err b/runscripts/large_variation/alltoall_N2.2360533.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/large_variation/alltoall_N2.2360533.out b/runscripts/large_variation/alltoall_N2.2360533.out new file mode 100644 index 000000000..d9f5a717b --- /dev/null +++ b/runscripts/large_variation/alltoall_N2.2360533.out @@ -0,0 +1,633 @@ +Test 1 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 4.375190e-05 +Pairwise: 3.791252e-04 +NonBlocking: 3.823517e-04 +Pairwise Hierarchical: 3.953741e-04 +Pairwise Multileader: 3.878461e-04 +Pairwise Node Aware: 3.746982e-04 +Pairwise Locality Aware: 3.854166e-04 +Pairwise Multileader Locality: 4.064942e-04 +Nonblocking Hierarchical: 3.766102e-04 +Nonblocking Multileader: 3.740545e-04 +Nonblocking Node Aware: 4.007346e-04 +Nonblocking Locality Aware: 3.837829e-04 +Nonblocking Multileader Locality: 3.742000e-04 +Size 2 +PMPI: 4.646209e-05 +Pairwise: 4.056553e-04 +NonBlocking: 3.769293e-04 +Pairwise Hierarchical: 3.779658e-04 +Pairwise Multileader: 4.022200e-04 +Pairwise Node Aware: 3.792628e-04 +Pairwise Locality Aware: 3.756040e-04 +Pairwise Multileader Locality: 3.893414e-04 +Nonblocking Hierarchical: 3.945317e-04 +Nonblocking Multileader: 3.746383e-04 +Nonblocking Node Aware: 3.816412e-04 +Nonblocking Locality Aware: 4.052135e-04 +Nonblocking Multileader Locality: 3.744594e-04 +Size 4 +PMPI: 5.686919e-05 +Pairwise: 8.301264e-04 +NonBlocking: 7.848931e-04 +Pairwise Hierarchical: 7.793687e-04 +Pairwise Multileader: 8.239857e-04 +Pairwise Node Aware: 8.006758e-04 +Pairwise Locality Aware: 7.745720e-04 +Pairwise Multileader Locality: 8.006273e-04 +Nonblocking Hierarchical: 8.151440e-04 +Nonblocking Multileader: 7.829813e-04 +Nonblocking Node Aware: 7.914475e-04 +Nonblocking Locality Aware: 8.310563e-04 +Nonblocking Multileader Locality: 7.816976e-04 +Size 8 +PMPI: 7.811394e-05 +Pairwise: 8.304180e-04 +NonBlocking: 8.232448e-04 +Pairwise Hierarchical: 8.037649e-04 +Pairwise Multileader: 8.172116e-04 +Pairwise Node Aware: 8.258725e-04 +Pairwise Locality Aware: 7.899264e-04 +Pairwise Multileader Locality: 7.942722e-04 +Nonblocking Hierarchical: 8.337596e-04 +Nonblocking Multileader: 8.186268e-04 +Nonblocking Node Aware: 7.881949e-04 +Nonblocking Locality Aware: 8.064784e-04 +Nonblocking Multileader Locality: 8.325940e-04 +Size 16 +PMPI: 1.344187e-04 +Pairwise: 7.954180e-04 +NonBlocking: 8.353857e-04 +Pairwise Hierarchical: 7.928681e-04 +Pairwise Multileader: 7.885140e-04 +Pairwise Node Aware: 8.404886e-04 +Pairwise Locality Aware: 8.015752e-04 +Pairwise Multileader Locality: 7.878448e-04 +Nonblocking Hierarchical: 8.054773e-04 +Nonblocking Multileader: 8.221236e-04 +Nonblocking Node Aware: 7.864296e-04 +Nonblocking Locality Aware: 8.000147e-04 +Nonblocking Multileader Locality: 8.555585e-04 +Size 32 +PMPI: 2.542539e-04 +Pairwise: 8.079384e-04 +NonBlocking: 8.574642e-04 +Pairwise Hierarchical: 8.076315e-04 +Pairwise Multileader: 8.057548e-04 +Pairwise Node Aware: 8.358106e-04 +Pairwise Locality Aware: 8.281310e-04 +Pairwise Multileader Locality: 8.029516e-04 +Nonblocking Hierarchical: 8.135216e-04 +Nonblocking Multileader: 8.447716e-04 +Nonblocking Node Aware: 8.153054e-04 +Nonblocking Locality Aware: 8.043948e-04 +Nonblocking Multileader Locality: 8.109230e-04 +Size 64 +PMPI: 4.558662e-04 +Pairwise: 8.342231e-04 +NonBlocking: 8.331828e-04 +Pairwise Hierarchical: 8.956085e-04 +Pairwise Multileader: 8.587593e-04 +Pairwise Node Aware: 8.243141e-04 +Pairwise Locality Aware: 8.429368e-04 +Pairwise Multileader Locality: 8.896761e-04 +Nonblocking Hierarchical: 8.350675e-04 +Nonblocking Multileader: 8.334505e-04 +Nonblocking Node Aware: 8.930946e-04 +Nonblocking Locality Aware: 8.396948e-04 +Nonblocking Multileader Locality: 8.318674e-04 +Size 128 +PMPI: 8.471267e-04 +Pairwise: 9.920896e-04 +NonBlocking: 9.542519e-04 +Pairwise Hierarchical: 9.819939e-04 +Pairwise Multileader: 1.001686e-03 +Pairwise Node Aware: 9.498385e-04 +Pairwise Locality Aware: 9.666463e-04 +Pairwise Multileader Locality: 1.041538e-03 +Nonblocking Hierarchical: 9.390610e-04 +Nonblocking Multileader: 9.583893e-04 +Nonblocking Node Aware: 1.032676e-03 +Nonblocking Locality Aware: 9.687636e-04 +Nonblocking Multileader Locality: 9.527534e-04 +Size 256 +PMPI: 1.601665e-03 +Pairwise: 1.443825e-03 +NonBlocking: 1.339274e-03 +Pairwise Hierarchical: 1.423694e-03 +Pairwise Multileader: 1.401379e-03 +Pairwise Node Aware: 1.341340e-03 +Pairwise Locality Aware: 1.404428e-03 +Pairwise Multileader Locality: 1.490970e-03 +Nonblocking Hierarchical: 1.352504e-03 +Nonblocking Multileader: 1.347402e-03 +Nonblocking Node Aware: 1.538948e-03 +Nonblocking Locality Aware: 1.614461e-03 +Nonblocking Multileader Locality: 1.486282e-03 +Size 512 +PMPI: 2.686080e-03 +Pairwise: 2.474082e-03 +NonBlocking: 2.281911e-03 +Pairwise Hierarchical: 2.327600e-03 +Pairwise Multileader: 2.454053e-03 +Pairwise Node Aware: 2.766713e-03 +Pairwise Locality Aware: 2.528074e-03 +Pairwise Multileader Locality: 2.270366e-03 +Nonblocking Hierarchical: 2.368435e-03 +Nonblocking Multileader: 2.370493e-03 +Nonblocking Node Aware: 2.356873e-03 +Nonblocking Locality Aware: 2.387340e-03 +Nonblocking Multileader Locality: 2.289798e-03 +Size 1024 +PMPI: 4.221197e-03 +Pairwise: 4.295177e-03 +NonBlocking: 4.325228e-03 +Pairwise Hierarchical: 4.298184e-03 +Pairwise Multileader: 4.301855e-03 +Pairwise Node Aware: 4.267055e-03 +Pairwise Locality Aware: 4.310926e-03 +Pairwise Multileader Locality: 4.179933e-03 +Nonblocking Hierarchical: 4.202171e-03 +Nonblocking Multileader: 4.424244e-03 +Nonblocking Node Aware: 4.144592e-03 +Nonblocking Locality Aware: 4.161625e-03 +Nonblocking Multileader Locality: 4.455994e-03 +Size 2048 +PMPI: 8.403349e-03 +Pairwise: 8.353347e-03 +NonBlocking: 8.637343e-03 +Pairwise Hierarchical: 8.445725e-03 +Pairwise Multileader: 8.447505e-03 +Pairwise Node Aware: 8.632120e-03 +Pairwise Locality Aware: 8.509862e-03 +Pairwise Multileader Locality: 8.386331e-03 +Nonblocking Hierarchical: 8.658713e-03 +Nonblocking Multileader: 8.628360e-03 +Nonblocking Node Aware: 8.404870e-03 +Nonblocking Locality Aware: 8.495575e-03 +Nonblocking Multileader Locality: 8.635070e-03 +Size 4096 +PMPI: 1.894216e-02 +Pairwise: 1.657053e-02 +NonBlocking: 1.660719e-02 +Pairwise Hierarchical: 1.642342e-02 +Pairwise Multileader: 1.706162e-02 +Pairwise Node Aware: 1.646036e-02 +Pairwise Locality Aware: 1.619797e-02 +Pairwise Multileader Locality: 1.678214e-02 +Nonblocking Hierarchical: 1.629255e-02 +Nonblocking Multileader: 1.616031e-02 +Nonblocking Node Aware: 1.633373e-02 +Nonblocking Locality Aware: 1.697505e-02 +Nonblocking Multileader Locality: 1.703308e-02 +Size 8192 +PMPI: 3.127351e-02 +Pairwise: 3.228295e-02 +NonBlocking: 3.106412e-02 +Pairwise Hierarchical: 3.330392e-02 +Pairwise Multileader: 3.094077e-02 +Pairwise Node Aware: 3.113744e-02 +Pairwise Locality Aware: 3.185280e-02 +Pairwise Multileader Locality: 3.144159e-02 +Nonblocking Hierarchical: 3.179958e-02 +Nonblocking Multileader: 3.104315e-02 +Nonblocking Node Aware: 3.167366e-02 +Nonblocking Locality Aware: 3.137687e-02 +Nonblocking Multileader Locality: 3.119654e-02 +Size 16384 +PMPI: 5.938262e-02 +Pairwise: 5.786373e-02 +NonBlocking: 5.709907e-02 +Pairwise Hierarchical: 5.707014e-02 +Pairwise Multileader: 5.691409e-02 +Pairwise Node Aware: 5.710144e-02 +Pairwise Locality Aware: 5.760841e-02 +Pairwise Multileader Locality: 5.627723e-02 +Nonblocking Hierarchical: 5.672671e-02 +Nonblocking Multileader: 5.786441e-02 +Nonblocking Node Aware: 5.711738e-02 +Nonblocking Locality Aware: 5.735809e-02 +Nonblocking Multileader Locality: 5.703309e-02 +Test 2 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 4.617489e-05 +Pairwise: 3.804211e-04 +NonBlocking: 3.782116e-04 +Pairwise Hierarchical: 4.047290e-04 +Pairwise Multileader: 3.892233e-04 +Pairwise Node Aware: 3.763856e-04 +Pairwise Locality Aware: 3.926461e-04 +Pairwise Multileader Locality: 3.863712e-04 +Nonblocking Hierarchical: 3.749300e-04 +Nonblocking Multileader: 3.822418e-04 +Nonblocking Node Aware: 4.025392e-04 +Nonblocking Locality Aware: 3.886400e-04 +Nonblocking Multileader Locality: 3.734271e-04 +Size 2 +PMPI: 4.679073e-05 +Pairwise: 4.039936e-04 +NonBlocking: 3.802093e-04 +Pairwise Hierarchical: 3.757949e-04 +Pairwise Multileader: 3.850415e-04 +Pairwise Node Aware: 3.959241e-04 +Pairwise Locality Aware: 3.846931e-04 +Pairwise Multileader Locality: 3.841124e-04 +Nonblocking Hierarchical: 3.821503e-04 +Nonblocking Multileader: 4.037121e-04 +Nonblocking Node Aware: 3.786177e-04 +Nonblocking Locality Aware: 3.755805e-04 +Nonblocking Multileader Locality: 4.005719e-04 +Size 4 +PMPI: 5.870472e-05 +Pairwise: 7.920544e-04 +NonBlocking: 8.115575e-04 +Pairwise Hierarchical: 8.220765e-04 +Pairwise Multileader: 7.768446e-04 +Pairwise Node Aware: 7.957869e-04 +Pairwise Locality Aware: 8.383811e-04 +Pairwise Multileader Locality: 7.841813e-04 +Nonblocking Hierarchical: 7.880482e-04 +Nonblocking Multileader: 8.403349e-04 +Nonblocking Node Aware: 7.893869e-04 +Nonblocking Locality Aware: 7.812866e-04 +Nonblocking Multileader Locality: 8.231084e-04 +Size 8 +PMPI: 7.894930e-05 +Pairwise: 7.984547e-04 +NonBlocking: 7.972724e-04 +Pairwise Hierarchical: 8.452495e-04 +Pairwise Multileader: 8.024248e-04 +Pairwise Node Aware: 7.915727e-04 +Pairwise Locality Aware: 8.366676e-04 +Pairwise Multileader Locality: 8.225056e-04 +Nonblocking Hierarchical: 7.836817e-04 +Nonblocking Multileader: 8.081886e-04 +Nonblocking Node Aware: 8.385668e-04 +Nonblocking Locality Aware: 7.903625e-04 +Nonblocking Multileader Locality: 7.979083e-04 +Size 16 +PMPI: 1.345806e-04 +Pairwise: 8.498046e-04 +NonBlocking: 8.032090e-04 +Pairwise Hierarchical: 7.886380e-04 +Pairwise Multileader: 8.317855e-04 +Pairwise Node Aware: 8.170806e-04 +Pairwise Locality Aware: 7.909076e-04 +Pairwise Multileader Locality: 8.090947e-04 +Nonblocking Hierarchical: 8.250497e-04 +Nonblocking Multileader: 7.858911e-04 +Nonblocking Node Aware: 7.925108e-04 +Nonblocking Locality Aware: 8.421875e-04 +Nonblocking Multileader Locality: 7.923886e-04 +Size 32 +PMPI: 2.498746e-04 +Pairwise: 8.191827e-04 +NonBlocking: 8.253652e-04 +Pairwise Hierarchical: 8.038026e-04 +Pairwise Multileader: 7.969913e-04 +Pairwise Node Aware: 7.961780e-04 +Pairwise Locality Aware: 8.171458e-04 +Pairwise Multileader Locality: 8.381294e-04 +Nonblocking Hierarchical: 7.898103e-04 +Nonblocking Multileader: 8.019882e-04 +Nonblocking Node Aware: 8.098128e-04 +Nonblocking Locality Aware: 8.060311e-04 +Nonblocking Multileader Locality: 8.049464e-04 +Size 64 +PMPI: 4.411608e-04 +Pairwise: 8.286488e-04 +NonBlocking: 8.525215e-04 +Pairwise Hierarchical: 8.816801e-04 +Pairwise Multileader: 8.290771e-04 +Pairwise Node Aware: 8.453146e-04 +Pairwise Locality Aware: 8.971647e-04 +Pairwise Multileader Locality: 8.454055e-04 +Nonblocking Hierarchical: 8.458743e-04 +Nonblocking Multileader: 8.842754e-04 +Nonblocking Node Aware: 8.480715e-04 +Nonblocking Locality Aware: 8.316867e-04 +Nonblocking Multileader Locality: 8.967481e-04 +Size 128 +PMPI: 8.902493e-04 +Pairwise: 9.507955e-04 +NonBlocking: 1.004420e-03 +Pairwise Hierarchical: 9.717751e-04 +Pairwise Multileader: 9.536572e-04 +Pairwise Node Aware: 9.919046e-04 +Pairwise Locality Aware: 9.757056e-04 +Pairwise Multileader Locality: 9.463416e-04 +Nonblocking Hierarchical: 9.602915e-04 +Nonblocking Multileader: 1.023107e-03 +Nonblocking Node Aware: 9.630026e-04 +Nonblocking Locality Aware: 9.500454e-04 +Nonblocking Multileader Locality: 9.933569e-04 +Size 256 +PMPI: 1.564945e-03 +Pairwise: 1.370042e-03 +NonBlocking: 1.402235e-03 +Pairwise Hierarchical: 1.481408e-03 +Pairwise Multileader: 1.407836e-03 +Pairwise Node Aware: 1.374258e-03 +Pairwise Locality Aware: 1.428022e-03 +Pairwise Multileader Locality: 1.338836e-03 +Nonblocking Hierarchical: 1.345687e-03 +Nonblocking Multileader: 1.402626e-03 +Nonblocking Node Aware: 1.384921e-03 +Nonblocking Locality Aware: 1.333430e-03 +Nonblocking Multileader Locality: 1.399249e-03 +Size 512 +PMPI: 2.278229e-03 +Pairwise: 2.299717e-03 +NonBlocking: 2.296912e-03 +Pairwise Hierarchical: 2.394694e-03 +Pairwise Multileader: 2.299704e-03 +Pairwise Node Aware: 2.312146e-03 +Pairwise Locality Aware: 2.340300e-03 +Pairwise Multileader Locality: 2.223334e-03 +Nonblocking Hierarchical: 2.268950e-03 +Nonblocking Multileader: 2.349343e-03 +Nonblocking Node Aware: 2.296302e-03 +Nonblocking Locality Aware: 2.292782e-03 +Nonblocking Multileader Locality: 2.336923e-03 +Size 1024 +PMPI: 4.357719e-03 +Pairwise: 4.235799e-03 +NonBlocking: 4.203937e-03 +Pairwise Hierarchical: 4.408462e-03 +Pairwise Multileader: 4.229369e-03 +Pairwise Node Aware: 4.210784e-03 +Pairwise Locality Aware: 4.365078e-03 +Pairwise Multileader Locality: 4.222913e-03 +Nonblocking Hierarchical: 4.159067e-03 +Nonblocking Multileader: 4.367919e-03 +Nonblocking Node Aware: 4.150328e-03 +Nonblocking Locality Aware: 4.178796e-03 +Nonblocking Multileader Locality: 4.387831e-03 +Size 2048 +PMPI: 8.430596e-03 +Pairwise: 8.368658e-03 +NonBlocking: 8.647557e-03 +Pairwise Hierarchical: 8.450467e-03 +Pairwise Multileader: 8.529207e-03 +Pairwise Node Aware: 8.501245e-03 +Pairwise Locality Aware: 8.908654e-03 +Pairwise Multileader Locality: 8.805890e-03 +Nonblocking Hierarchical: 8.570748e-03 +Nonblocking Multileader: 8.711566e-03 +Nonblocking Node Aware: 8.396669e-03 +Nonblocking Locality Aware: 8.859035e-03 +Nonblocking Multileader Locality: 8.908452e-03 +Size 4096 +PMPI: 1.843275e-02 +Pairwise: 1.668106e-02 +NonBlocking: 1.673347e-02 +Pairwise Hierarchical: 1.660560e-02 +Pairwise Multileader: 1.683593e-02 +Pairwise Node Aware: 1.648660e-02 +Pairwise Locality Aware: 1.612049e-02 +Pairwise Multileader Locality: 1.675979e-02 +Nonblocking Hierarchical: 1.613843e-02 +Nonblocking Multileader: 1.620685e-02 +Nonblocking Node Aware: 1.656585e-02 +Nonblocking Locality Aware: 1.644569e-02 +Nonblocking Multileader Locality: 1.728191e-02 +Size 8192 +PMPI: 3.366433e-02 +Pairwise: 3.073489e-02 +NonBlocking: 3.107276e-02 +Pairwise Hierarchical: 3.102353e-02 +Pairwise Multileader: 3.222303e-02 +Pairwise Node Aware: 3.335506e-02 +Pairwise Locality Aware: 3.199834e-02 +Pairwise Multileader Locality: 3.355747e-02 +Nonblocking Hierarchical: 3.270901e-02 +Nonblocking Multileader: 3.248227e-02 +Nonblocking Node Aware: 3.275482e-02 +Nonblocking Locality Aware: 3.297478e-02 +Nonblocking Multileader Locality: 4.113317e-02 +Size 16384 +PMPI: 6.428466e-02 +Pairwise: 6.934079e-02 +NonBlocking: 5.991636e-02 +Pairwise Hierarchical: 6.144461e-02 +Pairwise Multileader: 6.149068e-02 +Pairwise Node Aware: 6.083653e-02 +Pairwise Locality Aware: 6.045046e-02 +Pairwise Multileader Locality: 6.139229e-02 +Nonblocking Hierarchical: 6.207703e-02 +Nonblocking Multileader: 5.880504e-02 +Nonblocking Node Aware: 5.994270e-02 +Nonblocking Locality Aware: 5.999796e-02 +Nonblocking Multileader Locality: 5.641694e-02 +Test 3 Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 4.368923e-05 +Pairwise: 3.843156e-04 +NonBlocking: 4.091784e-04 +Pairwise Hierarchical: 3.811914e-04 +Pairwise Multileader: 3.789696e-04 +Pairwise Node Aware: 4.252757e-04 +Pairwise Locality Aware: 3.970160e-04 +Pairwise Multileader Locality: 3.781378e-04 +Nonblocking Hierarchical: 3.801430e-04 +Nonblocking Multileader: 3.986520e-04 +Nonblocking Node Aware: 3.769976e-04 +Nonblocking Locality Aware: 3.755801e-04 +Nonblocking Multileader Locality: 4.020314e-04 +Size 2 +PMPI: 4.690340e-05 +Pairwise: 3.768738e-04 +NonBlocking: 3.821663e-04 +Pairwise Hierarchical: 3.935429e-04 +Pairwise Multileader: 3.764448e-04 +Pairwise Node Aware: 3.754817e-04 +Pairwise Locality Aware: 4.055381e-04 +Pairwise Multileader Locality: 3.784541e-04 +Nonblocking Hierarchical: 3.767713e-04 +Nonblocking Multileader: 3.847225e-04 +Nonblocking Node Aware: 3.853688e-04 +Nonblocking Locality Aware: 3.908583e-04 +Nonblocking Multileader Locality: 3.774453e-04 +Size 4 +PMPI: 5.647905e-05 +Pairwise: 8.302303e-04 +NonBlocking: 7.829389e-04 +Pairwise Hierarchical: 7.841978e-04 +Pairwise Multileader: 8.176069e-04 +Pairwise Node Aware: 8.080877e-04 +Pairwise Locality Aware: 7.879549e-04 +Pairwise Multileader Locality: 8.254959e-04 +Nonblocking Hierarchical: 8.228730e-04 +Nonblocking Multileader: 7.825126e-04 +Nonblocking Node Aware: 7.871059e-04 +Nonblocking Locality Aware: 8.408596e-04 +Nonblocking Multileader Locality: 7.802278e-04 +Size 8 +PMPI: 7.677941e-05 +Pairwise: 8.325433e-04 +NonBlocking: 8.075719e-04 +Pairwise Hierarchical: 7.939578e-04 +Pairwise Multileader: 7.878292e-04 +Pairwise Node Aware: 8.322084e-04 +Pairwise Locality Aware: 8.126265e-04 +Pairwise Multileader Locality: 7.869397e-04 +Nonblocking Hierarchical: 8.269556e-04 +Nonblocking Multileader: 8.136487e-04 +Nonblocking Node Aware: 7.860005e-04 +Nonblocking Locality Aware: 8.117148e-04 +Nonblocking Multileader Locality: 8.479409e-04 +Size 16 +PMPI: 1.335167e-04 +Pairwise: 7.886359e-04 +NonBlocking: 8.397101e-04 +Pairwise Hierarchical: 7.984694e-04 +Pairwise Multileader: 7.889319e-04 +Pairwise Node Aware: 8.268161e-04 +Pairwise Locality Aware: 8.146394e-04 +Pairwise Multileader Locality: 7.938608e-04 +Nonblocking Hierarchical: 8.061265e-04 +Nonblocking Multileader: 8.421650e-04 +Nonblocking Node Aware: 7.888775e-04 +Nonblocking Locality Aware: 8.007522e-04 +Nonblocking Multileader Locality: 8.314140e-04 +Size 32 +PMPI: 2.480043e-04 +Pairwise: 8.065699e-04 +NonBlocking: 8.760280e-04 +Pairwise Hierarchical: 8.335987e-04 +Pairwise Multileader: 8.028058e-04 +Pairwise Node Aware: 8.285661e-04 +Pairwise Locality Aware: 8.614183e-04 +Pairwise Multileader Locality: 8.025583e-04 +Nonblocking Hierarchical: 8.028796e-04 +Nonblocking Multileader: 8.523092e-04 +Nonblocking Node Aware: 8.179500e-04 +Nonblocking Locality Aware: 8.069161e-04 +Nonblocking Multileader Locality: 8.284887e-04 +Size 64 +PMPI: 4.581206e-04 +Pairwise: 8.341368e-04 +NonBlocking: 8.580486e-04 +Pairwise Hierarchical: 8.718912e-04 +Pairwise Multileader: 8.360447e-04 +Pairwise Node Aware: 8.551332e-04 +Pairwise Locality Aware: 9.022327e-04 +Pairwise Multileader Locality: 8.552804e-04 +Nonblocking Hierarchical: 8.343292e-04 +Nonblocking Multileader: 8.741779e-04 +Nonblocking Node Aware: 8.688276e-04 +Nonblocking Locality Aware: 8.356230e-04 +Nonblocking Multileader Locality: 8.646986e-04 +Size 128 +PMPI: 8.616838e-04 +Pairwise: 9.567471e-04 +NonBlocking: 9.765074e-04 +Pairwise Hierarchical: 1.019692e-03 +Pairwise Multileader: 9.539580e-04 +Pairwise Node Aware: 9.384807e-04 +Pairwise Locality Aware: 1.030775e-03 +Pairwise Multileader Locality: 9.804279e-04 +Nonblocking Hierarchical: 9.524498e-04 +Nonblocking Multileader: 9.829334e-04 +Nonblocking Node Aware: 1.012554e-03 +Nonblocking Locality Aware: 9.541409e-04 +Nonblocking Multileader Locality: 9.582342e-04 +Size 256 +PMPI: 1.566533e-03 +Pairwise: 1.364744e-03 +NonBlocking: 1.355045e-03 +Pairwise Hierarchical: 1.434963e-03 +Pairwise Multileader: 1.384403e-03 +Pairwise Node Aware: 1.346539e-03 +Pairwise Locality Aware: 1.414587e-03 +Pairwise Multileader Locality: 1.400138e-03 +Nonblocking Hierarchical: 1.339569e-03 +Nonblocking Multileader: 1.355988e-03 +Nonblocking Node Aware: 1.409037e-03 +Nonblocking Locality Aware: 1.368606e-03 +Nonblocking Multileader Locality: 1.398254e-03 +Size 512 +PMPI: 2.299895e-03 +Pairwise: 2.313026e-03 +NonBlocking: 2.327277e-03 +Pairwise Hierarchical: 2.402696e-03 +Pairwise Multileader: 2.284182e-03 +Pairwise Node Aware: 2.328215e-03 +Pairwise Locality Aware: 2.357838e-03 +Pairwise Multileader Locality: 2.277479e-03 +Nonblocking Hierarchical: 2.299336e-03 +Nonblocking Multileader: 2.392406e-03 +Nonblocking Node Aware: 2.440418e-03 +Nonblocking Locality Aware: 2.324581e-03 +Nonblocking Multileader Locality: 2.349349e-03 +Size 1024 +PMPI: 4.432388e-03 +Pairwise: 4.298112e-03 +NonBlocking: 4.515521e-03 +Pairwise Hierarchical: 4.535741e-03 +Pairwise Multileader: 4.282647e-03 +Pairwise Node Aware: 4.216723e-03 +Pairwise Locality Aware: 4.334202e-03 +Pairwise Multileader Locality: 4.218851e-03 +Nonblocking Hierarchical: 4.220456e-03 +Nonblocking Multileader: 4.412715e-03 +Nonblocking Node Aware: 4.287016e-03 +Nonblocking Locality Aware: 4.338693e-03 +Nonblocking Multileader Locality: 4.406090e-03 +Size 2048 +PMPI: 8.731089e-03 +Pairwise: 8.669969e-03 +NonBlocking: 8.785984e-03 +Pairwise Hierarchical: 8.647612e-03 +Pairwise Multileader: 8.988245e-03 +Pairwise Node Aware: 9.039261e-03 +Pairwise Locality Aware: 8.743904e-03 +Pairwise Multileader Locality: 8.673610e-03 +Nonblocking Hierarchical: 8.800583e-03 +Nonblocking Multileader: 8.925056e-03 +Nonblocking Node Aware: 8.637178e-03 +Nonblocking Locality Aware: 8.887661e-03 +Nonblocking Multileader Locality: 8.860491e-03 +Size 4096 +PMPI: 1.858440e-02 +Pairwise: 1.687651e-02 +NonBlocking: 1.730575e-02 +Pairwise Hierarchical: 1.730055e-02 +Pairwise Multileader: 1.745685e-02 +Pairwise Node Aware: 1.675935e-02 +Pairwise Locality Aware: 1.658179e-02 +Pairwise Multileader Locality: 1.689124e-02 +Nonblocking Hierarchical: 1.670812e-02 +Nonblocking Multileader: 1.683216e-02 +Nonblocking Node Aware: 1.677003e-02 +Nonblocking Locality Aware: 1.714275e-02 +Nonblocking Multileader Locality: 1.709610e-02 +Size 8192 +PMPI: 3.294048e-02 +Pairwise: 3.090418e-02 +NonBlocking: 3.230991e-02 +Pairwise Hierarchical: 3.224829e-02 +Pairwise Multileader: 3.327129e-02 +Pairwise Node Aware: 3.288746e-02 +Pairwise Locality Aware: 3.215076e-02 +Pairwise Multileader Locality: 3.231077e-02 +Nonblocking Hierarchical: 3.184305e-02 +Nonblocking Multileader: 3.348588e-02 +Nonblocking Node Aware: 3.258574e-02 +Nonblocking Locality Aware: 3.419963e-02 +Nonblocking Multileader Locality: 3.531614e-02 +Size 16384 +PMPI: 5.798288e-02 +Pairwise: 5.895465e-02 +NonBlocking: 6.071834e-02 +Pairwise Hierarchical: 5.794330e-02 +Pairwise Multileader: 5.690736e-02 +Pairwise Node Aware: 5.766690e-02 +Pairwise Locality Aware: 5.745253e-02 +Pairwise Multileader Locality: 5.749800e-02 +Nonblocking Hierarchical: 5.705705e-02 +Nonblocking Multileader: 5.764507e-02 +Nonblocking Node Aware: 5.736653e-02 +Nonblocking Locality Aware: 5.716124e-02 +Nonblocking Multileader Locality: 5.760747e-02 diff --git a/runscripts/large_variation/alltoall_N32.2360547.err b/runscripts/large_variation/alltoall_N32.2360547.err new file mode 100644 index 000000000..3db1c4f32 --- /dev/null +++ b/runscripts/large_variation/alltoall_N32.2360547.err @@ -0,0 +1,810 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2360547.0 ON dane98 CANCELLED AT 2025-04-28T15:21:02 *** +slurmstepd: error: *** STEP 2360547.0 ON dane98 CANCELLED AT 2025-04-28T15:21:02 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane840: task 1807: Killed +srun: error: dane1413: task 3005: Killed +srun: error: dane815: task 1505: Killed +srun: error: dane824: tasks 1705,1709: Killed +srun: error: dane1118: tasks 2705,2709: Killed +srun: error: dane1222: tasks 2805,2809,2813: Killed +srun: error: dane1108: tasks 2605,2609,2693: Killed +srun: error: dane1094: tasks 2505,2509,2513: Killed +srun: error: dane1087: tasks 2405,2409: Killed +srun: error: dane317: task 305: Killed +srun: error: dane488: tasks 1005,1009,1013: Killed +srun: error: dane663: tasks 1205,1209,1213: Killed +srun: error: dane474: tasks 805,809,813: Killed +srun: error: dane481: tasks 905,909,913,917: Killed +srun: error: dane1072: tasks 2205,2209,2213,2217: Killed +srun: error: dane1413: tasks 3000-3004,3006-3099: Killed +srun: error: dane815: tasks 1500-1504,1506-1599: Killed +srun: error: dane1084: tasks 2300-2399: Killed +srun: error: dane1222: tasks 2800-2804,2806-2808,2810-2812,2814-2899: Killed +srun: error: dane1108: tasks 2600-2604,2606-2608,2610-2692,2694-2699: Killed +srun: error: dane824: tasks 1700-1704,1706-1708,1710-1799: Killed +srun: error: dane481: tasks 900-904,906-908,910-912,914-916,918-999: Killed +srun: error: dane488: tasks 1000-1004,1006-1008,1010-1012,1014-1099: Killed +srun: error: dane663: tasks 1200-1204,1206-1208,1210-1212,1214-1299: Killed +srun: error: dane840: tasks 1800-1806,1808-1899: Killed +srun: error: dane1094: tasks 2500-2504,2506-2508,2510-2512,2514-2599: Killed +srun: error: dane474: tasks 800-804,806-808,810-812,814-899: Killed +srun: error: dane356: tasks 400-499: Killed +srun: error: dane1040: tasks 2100-2199: Killed +srun: error: dane1118: tasks 2700-2704,2706-2708,2710-2799: Killed +srun: error: dane847: tasks 1900-1999: Killed +srun: error: dane518: tasks 1100-1199: Killed +srun: error: dane1453: tasks 3100-3199: Killed +srun: error: dane407: tasks 500-599: Killed +srun: error: dane1087: tasks 2400-2404,2406-2408,2410-2499: Killed +srun: error: dane309: tasks 200-299: Killed +srun: error: dane317: tasks 300-304,306-399: Killed +srun: error: dane433: tasks 600-699: Killed +srun: error: dane1072: tasks 2200-2204,2206-2208,2210-2212,2214-2216,2218-2299: Killed +srun: error: dane703: tasks 1300-1399: Killed +srun: error: dane818: tasks 1600-1699: Killed +srun: error: dane723: tasks 1400-1499: Killed +srun: error: dane450: tasks 700-799: Killed +srun: error: dane101: tasks 100-199: Killed +srun: error: dane98: tasks 0-99: Killed +srun: error: dane885: tasks 2000-2099: Killed +srun: error: dane1318: tasks 2900-2999: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2360547.1 ON dane98 CANCELLED AT 2025-04-28T15:25:00 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane847: task 1903: Killed +srun: error: dane488: task 1005: Killed +srun: error: dane433: task 605: Killed +srun: error: dane703: task 1305: Killed +srun: error: dane1087: task 2405: Killed +srun: error: dane824: tasks 1705,1709: Killed +srun: error: dane1108: tasks 2605,2609: Killed +srun: error: dane663: tasks 1205,1209: Killed +srun: error: dane1413: tasks 3005,3009,3013: Killed +srun: error: dane1084: tasks 2305,2309,2313: Killed +srun: error: dane356: tasks 405,409,413: Killed +srun: error: dane1222: tasks 2805,2807,2809,2813,2817: Killed +srun: error: dane474: tasks 805,809,813,817: Killed +srun: error: dane407: tasks 505,509,513: Killed +srun: error: dane1413: tasks 3000-3004,3006-3008,3010-3012,3014-3099: Killed +srun: error: dane1222: tasks 2800-2804,2806,2808,2810-2812,2814-2816,2818-2899: Killed +srun: error: dane815: tasks 1500-1599: Killed +srun: error: dane1084: tasks 2300-2304,2306-2308,2310-2312,2314-2399: Killed +srun: error: dane1108: tasks 2600-2604,2606-2608,2610-2699: Killed +srun: error: dane824: tasks 1700-1704,1706-1708,1710-1799: Killed +srun: error: dane481: tasks 900-999: Killed +srun: error: dane488: tasks 1000-1004,1006-1099: Killed +srun: error: dane840: tasks 1800-1899: Killed +srun: error: dane663: tasks 1200-1204,1206-1208,1210-1299: Killed +srun: error: dane1094: tasks 2500-2599: Killed +srun: error: dane474: tasks 800-804,806-808,810-812,814-816,818-899: Killed +srun: error: dane356: tasks 400-404,406-408,410-412,414-499: Killed +srun: error: dane1040: tasks 2100-2199: Killed +srun: error: dane1118: tasks 2700-2799: Killed +srun: error: dane847: tasks 1900-1902,1904-1999: Killed +srun: error: dane1453: tasks 3100-3199: Killed +srun: error: dane518: tasks 1100-1199: Killed +srun: error: dane1087: tasks 2400-2404,2406-2499: Killed +srun: error: dane407: tasks 500-504,506-508,510-512,514-599: Killed +srun: error: dane309: tasks 200-299: Killed +srun: error: dane1072: tasks 2200-2299: Killed +srun: error: dane317: tasks 300-399: Killed +srun: error: dane433: tasks 600-604,606-699: Killed +srun: error: dane703: tasks 1300-1304,1306-1399: Killed +srun: error: dane818: tasks 1600-1699: Killed +srun: error: dane885: tasks 2000-2099: Killed +srun: error: dane450: tasks 700-799: Killed +srun: error: dane98: tasks 0-99: Killed +srun: error: dane101: tasks 100-199: Killed +srun: error: dane723: tasks 1400-1499: Killed +srun: error: dane1318: tasks 2900-2999: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa866010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2360547.2 ON dane98 CANCELLED AT 2025-04-28T15:29:05 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1094: task 2597: Killed +srun: error: dane815: task 1505: Killed +srun: error: dane356: tasks 405,409,413: Killed +srun: error: dane1072: tasks 2209,2213: Killed +srun: error: dane450: tasks 705,709: Killed +srun: error: dane518: tasks 1105,1109,1113: Killed +srun: error: dane1413: tasks 3000-3099: Killed +srun: error: dane1222: tasks 2800-2899: Killed +srun: error: dane1084: tasks 2300-2399: Killed +srun: error: dane824: tasks 1700-1799: Killed +srun: error: dane1108: tasks 2600-2699: Killed +srun: error: dane815: tasks 1500-1504,1506-1599: Killed +srun: error: dane481: tasks 900-999: Killed +srun: error: dane488: tasks 1000-1099: Killed +srun: error: dane663: tasks 1200-1299: Killed +srun: error: dane840: tasks 1800-1899: Killed +srun: error: dane1094: tasks 2500-2596,2598-2599: Killed +srun: error: dane474: tasks 800-899: Killed +srun: error: dane356: tasks 400-404,406-408,410-412,414-499: Killed +srun: error: dane1040: tasks 2100-2199: Killed +srun: error: dane1118: tasks 2700-2799: Killed +srun: error: dane847: tasks 1900-1999: Killed +srun: error: dane1453: tasks 3100-3199: Killed +srun: error: dane518: tasks 1100-1104,1106-1108,1110-1112,1114-1199: Killed +srun: error: dane1087: tasks 2400-2499: Killed +srun: error: dane407: tasks 500-599: Killed +srun: error: dane309: tasks 200-299: Killed +srun: error: dane1072: tasks 2200-2208,2210-2212,2214-2299: Killed +srun: error: dane317: tasks 300-399: Killed +srun: error: dane433: tasks 600-699: Killed +srun: error: dane703: tasks 1300-1399: Killed +srun: error: dane818: tasks 1600-1699: Killed +srun: error: dane885: tasks 2000-2099: Killed +srun: error: dane1318: tasks 2900-2999: Killed +srun: error: dane723: tasks 1400-1499: Killed +srun: error: dane101: tasks 100-199: Killed +srun: error: dane98: tasks 0-99: Killed +srun: error: dane450: tasks 700-704,706-708,710-799: Killed diff --git a/runscripts/large_variation/alltoall_N32.2360547.out b/runscripts/large_variation/alltoall_N32.2360547.out new file mode 100644 index 000000000..ebc37f7a5 --- /dev/null +++ b/runscripts/large_variation/alltoall_N32.2360547.out @@ -0,0 +1,477 @@ +Test 1 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 9.826041e-04 +Pairwise: 2.414388e-02 +NonBlocking: 2.723757e-02 +Pairwise Hierarchical: 3.926939e-02 +Pairwise Multileader: 2.680866e-02 +Pairwise Node Aware: 4.690836e-02 +Pairwise Locality Aware: 1.497779e-01 +Pairwise Multileader Locality: 1.541038e-01 +Nonblocking Hierarchical: 1.745612e-01 +Nonblocking Multileader: 2.241377e-01 +Nonblocking Node Aware: 5.775762e-01 +Nonblocking Locality Aware: 6.357970e-02 +Nonblocking Multileader Locality: 4.596975e-02 +Size 2 +PMPI: 2.386859e-03 +Pairwise: 2.473863e-02 +NonBlocking: 3.942214e-02 +Pairwise Hierarchical: 3.224837e-02 +Pairwise Multileader: 2.740958e-02 +Pairwise Node Aware: 2.437895e-02 +Pairwise Locality Aware: 2.904568e-02 +Pairwise Multileader Locality: 2.498507e-02 +Nonblocking Hierarchical: 2.560349e-02 +Nonblocking Multileader: 2.254406e-02 +Nonblocking Node Aware: 2.498268e-02 +Nonblocking Locality Aware: 2.580421e-02 +Nonblocking Multileader Locality: 2.835858e-02 +Size 4 +PMPI: 3.797619e-03 +Pairwise: 5.408757e-02 +NonBlocking: 5.367407e-02 +Pairwise Hierarchical: 6.041254e-02 +Pairwise Multileader: 4.612055e-02 +Pairwise Node Aware: 7.011288e-02 +Pairwise Locality Aware: 4.581941e-02 +Pairwise Multileader Locality: 5.362064e-02 +Nonblocking Hierarchical: 5.065049e-02 +Nonblocking Multileader: 6.148403e-02 +Nonblocking Node Aware: 5.041593e-02 +Nonblocking Locality Aware: 6.000550e-02 +Nonblocking Multileader Locality: 5.523331e-02 +Size 8 +PMPI: 1.747865e-02 +Pairwise: 6.115501e-02 +NonBlocking: 6.632163e-02 +Pairwise Hierarchical: 6.952543e-02 +Pairwise Multileader: 4.811834e-02 +Pairwise Node Aware: 6.977113e-02 +Pairwise Locality Aware: 4.828028e-02 +Pairwise Multileader Locality: 7.820881e-02 +Nonblocking Hierarchical: 4.922544e-02 +Nonblocking Multileader: 5.151344e-02 +Nonblocking Node Aware: 7.245359e-02 +Nonblocking Locality Aware: 5.452863e-02 +Nonblocking Multileader Locality: 7.127639e-02 +Size 16 +PMPI: 7.304290e-03 +Pairwise: 6.261758e-02 +NonBlocking: 5.604258e-02 +Pairwise Hierarchical: 5.590934e-02 +Pairwise Multileader: 6.718659e-02 +Pairwise Node Aware: 5.298605e-02 +Pairwise Locality Aware: 5.014793e-02 +Pairwise Multileader Locality: 5.085027e-02 +Nonblocking Hierarchical: 4.837573e-02 +Nonblocking Multileader: 6.463511e-02 +Nonblocking Node Aware: 4.828327e-02 +Nonblocking Locality Aware: 4.801850e-02 +Nonblocking Multileader Locality: 5.979480e-02 +Size 32 +PMPI: 4.161875e-02 +Pairwise: 5.614636e-02 +NonBlocking: 3.937927e-02 +Pairwise Hierarchical: 5.554407e-02 +Pairwise Multileader: 5.190624e-02 +Pairwise Node Aware: 5.292180e-02 +Pairwise Locality Aware: 7.571773e-02 +Pairwise Multileader Locality: 5.160992e-02 +Nonblocking Hierarchical: 6.711760e-02 +Nonblocking Multileader: 6.184501e-02 +Nonblocking Node Aware: 4.205091e-02 +Nonblocking Locality Aware: 5.877965e-02 +Nonblocking Multileader Locality: 1.026081e-01 +Size 64 +PMPI: 2.287245e-02 +Pairwise: 6.329885e-02 +NonBlocking: 6.502753e-02 +Pairwise Hierarchical: 7.669126e-02 +Pairwise Multileader: 4.914171e-02 +Pairwise Node Aware: 5.602374e-02 +Pairwise Locality Aware: 5.311069e-02 +Pairwise Multileader Locality: 4.959577e-02 +Nonblocking Hierarchical: 4.885283e-02 +Nonblocking Multileader: 5.918007e-02 +Nonblocking Node Aware: 4.263151e-02 +Nonblocking Locality Aware: 7.812885e-02 +Nonblocking Multileader Locality: 5.007789e-02 +Size 128 +PMPI: 4.716874e-02 +Pairwise: 5.890722e-02 +NonBlocking: 5.876782e-02 +Pairwise Hierarchical: 7.241912e-02 +Pairwise Multileader: 6.540374e-02 +Pairwise Node Aware: 6.428255e-02 +Pairwise Locality Aware: 6.367488e-02 +Pairwise Multileader Locality: 6.106242e-02 +Nonblocking Hierarchical: 6.123403e-02 +Nonblocking Multileader: 7.170348e-02 +Nonblocking Node Aware: 8.747718e-02 +Nonblocking Locality Aware: 6.818051e-02 +Nonblocking Multileader Locality: 6.354163e-02 +Size 256 +PMPI: 1.057859e-01 +Pairwise: 7.744675e-02 +NonBlocking: 9.543256e-02 +Pairwise Hierarchical: 8.367178e-02 +Pairwise Multileader: 9.013147e-02 +Pairwise Node Aware: 8.405228e-02 +Pairwise Locality Aware: 8.454323e-02 +Pairwise Multileader Locality: 8.235178e-02 +Nonblocking Hierarchical: 9.603320e-02 +Nonblocking Multileader: 7.742093e-02 +Nonblocking Node Aware: 8.927167e-02 +Nonblocking Locality Aware: 9.609489e-02 +Nonblocking Multileader Locality: 1.046822e-01 +Size 512 +PMPI: 8.943102e-01 +Pairwise: 1.448317e-01 +NonBlocking: 1.396758e-01 +Pairwise Hierarchical: 1.618009e-01 +Pairwise Multileader: 1.395706e-01 +Pairwise Node Aware: 1.271946e-01 +Pairwise Locality Aware: 1.220732e-01 +Pairwise Multileader Locality: 1.457316e-01 +Nonblocking Hierarchical: 1.248812e-01 +Nonblocking Multileader: 1.208996e-01 +Nonblocking Node Aware: 1.474722e-01 +Nonblocking Locality Aware: 1.230448e-01 +Nonblocking Multileader Locality: 1.465560e-01 +Size 1024 +PMPI: 4.138842e-01 +Pairwise: 2.294786e-01 +NonBlocking: 2.206905e-01 +Pairwise Hierarchical: 2.256253e-01 +Pairwise Multileader: 2.225753e-01 +Pairwise Node Aware: 2.221351e-01 +Pairwise Locality Aware: 2.843145e-01 +Pairwise Multileader Locality: 3.084168e-01 +Nonblocking Hierarchical: 3.005020e-01 +Nonblocking Multileader: 2.478220e-01 +Nonblocking Node Aware: 2.493874e-01 +Nonblocking Locality Aware: 2.307520e-01 +Nonblocking Multileader Locality: 3.174632e-01 +Size 2048 +PMPI: 4.621330e-01 +Pairwise: 4.538689e-01 +NonBlocking: 5.578330e-01 +Test 2 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.129996e-03 +Pairwise: 1.753265e-02 +NonBlocking: 2.673638e-02 +Pairwise Hierarchical: 2.490820e-02 +Pairwise Multileader: 1.948255e-02 +Pairwise Node Aware: 2.942306e-02 +Pairwise Locality Aware: 2.307223e-02 +Pairwise Multileader Locality: 2.952683e-02 +Nonblocking Hierarchical: 3.109303e-02 +Nonblocking Multileader: 2.593822e-02 +Nonblocking Node Aware: 2.329538e-02 +Nonblocking Locality Aware: 3.573372e-02 +Nonblocking Multileader Locality: 2.736175e-02 +Size 2 +PMPI: 1.753983e-03 +Pairwise: 3.289704e-02 +NonBlocking: 3.874315e-02 +Pairwise Hierarchical: 4.907035e-02 +Pairwise Multileader: 2.347907e-02 +Pairwise Node Aware: 2.400862e-02 +Pairwise Locality Aware: 3.143521e-02 +Pairwise Multileader Locality: 4.103145e-02 +Nonblocking Hierarchical: 4.963737e-02 +Nonblocking Multileader: 3.919151e-02 +Nonblocking Node Aware: 2.484421e-02 +Nonblocking Locality Aware: 4.305850e-02 +Nonblocking Multileader Locality: 2.802266e-02 +Size 4 +PMPI: 3.389509e-03 +Pairwise: 5.878613e-02 +NonBlocking: 9.029649e-02 +Pairwise Hierarchical: 5.356805e-02 +Pairwise Multileader: 4.649414e-02 +Pairwise Node Aware: 4.660718e-02 +Pairwise Locality Aware: 5.640725e-02 +Pairwise Multileader Locality: 6.670050e-02 +Nonblocking Hierarchical: 5.746467e-02 +Nonblocking Multileader: 4.944313e-02 +Nonblocking Node Aware: 4.720643e-02 +Nonblocking Locality Aware: 4.509149e-02 +Nonblocking Multileader Locality: 5.058613e-02 +Size 8 +PMPI: 4.223746e-03 +Pairwise: 6.299449e-02 +NonBlocking: 4.170695e-02 +Pairwise Hierarchical: 5.214472e-02 +Pairwise Multileader: 4.934200e-02 +Pairwise Node Aware: 4.869562e-02 +Pairwise Locality Aware: 6.271043e-02 +Pairwise Multileader Locality: 4.903911e-02 +Nonblocking Hierarchical: 3.976587e-02 +Nonblocking Multileader: 5.558308e-02 +Nonblocking Node Aware: 4.917173e-02 +Nonblocking Locality Aware: 4.688437e-02 +Nonblocking Multileader Locality: 5.014251e-02 +Size 16 +PMPI: 1.436249e-02 +Pairwise: 5.813710e-02 +NonBlocking: 3.299346e-02 +Pairwise Hierarchical: 5.283140e-02 +Pairwise Multileader: 6.611614e-02 +Pairwise Node Aware: 6.981683e-02 +Pairwise Locality Aware: 4.669792e-02 +Pairwise Multileader Locality: 5.536172e-02 +Nonblocking Hierarchical: 6.548953e-02 +Nonblocking Multileader: 5.324015e-02 +Nonblocking Node Aware: 5.254907e-02 +Nonblocking Locality Aware: 4.881280e-02 +Nonblocking Multileader Locality: 6.273303e-02 +Size 32 +PMPI: 1.285187e-02 +Pairwise: 6.744429e-02 +NonBlocking: 5.704727e-02 +Pairwise Hierarchical: 8.475982e-02 +Pairwise Multileader: 5.373635e-02 +Pairwise Node Aware: 5.428770e-02 +Pairwise Locality Aware: 6.649026e-02 +Pairwise Multileader Locality: 5.756476e-02 +Nonblocking Hierarchical: 5.704761e-02 +Nonblocking Multileader: 4.646498e-02 +Nonblocking Node Aware: 5.333859e-02 +Nonblocking Locality Aware: 5.761107e-02 +Nonblocking Multileader Locality: 4.500670e-02 +Size 64 +PMPI: 2.233129e-02 +Pairwise: 6.586038e-02 +NonBlocking: 5.343228e-02 +Pairwise Hierarchical: 6.650329e-02 +Pairwise Multileader: 4.763208e-02 +Pairwise Node Aware: 5.733049e-02 +Pairwise Locality Aware: 6.391403e-02 +Pairwise Multileader Locality: 4.421960e-02 +Nonblocking Hierarchical: 6.395931e-02 +Nonblocking Multileader: 5.438600e-02 +Nonblocking Node Aware: 4.550185e-02 +Nonblocking Locality Aware: 6.036367e-02 +Nonblocking Multileader Locality: 8.235745e-02 +Size 128 +PMPI: 4.862026e-02 +Pairwise: 6.849886e-02 +NonBlocking: 6.637700e-02 +Pairwise Hierarchical: 6.470364e-02 +Pairwise Multileader: 9.218680e-02 +Pairwise Node Aware: 8.372026e-02 +Pairwise Locality Aware: 9.291288e-02 +Pairwise Multileader Locality: 7.516849e-02 +Nonblocking Hierarchical: 8.696720e-02 +Nonblocking Multileader: 8.000311e-02 +Nonblocking Node Aware: 8.804192e-02 +Nonblocking Locality Aware: 6.993111e-02 +Nonblocking Multileader Locality: 6.716295e-02 +Size 256 +PMPI: 9.942737e-02 +Pairwise: 8.473659e-02 +NonBlocking: 8.173339e-02 +Pairwise Hierarchical: 1.015124e-01 +Pairwise Multileader: 8.002832e-02 +Pairwise Node Aware: 7.870171e-02 +Pairwise Locality Aware: 1.063262e-01 +Pairwise Multileader Locality: 8.270629e-02 +Nonblocking Hierarchical: 9.810297e-02 +Nonblocking Multileader: 8.128847e-02 +Nonblocking Node Aware: 9.173861e-02 +Nonblocking Locality Aware: 9.828642e-02 +Nonblocking Multileader Locality: 8.742304e-02 +Size 512 +PMPI: 5.812601e-01 +Pairwise: 1.263482e-01 +NonBlocking: 1.374612e-01 +Pairwise Hierarchical: 1.453332e-01 +Pairwise Multileader: 1.282404e-01 +Pairwise Node Aware: 1.670395e-01 +Pairwise Locality Aware: 1.223810e-01 +Pairwise Multileader Locality: 1.282587e-01 +Nonblocking Hierarchical: 1.329585e-01 +Nonblocking Multileader: 1.293222e-01 +Nonblocking Node Aware: 1.629168e-01 +Nonblocking Locality Aware: 1.322454e-01 +Nonblocking Multileader Locality: 1.442056e-01 +Size 1024 +PMPI: 2.658254e-01 +Pairwise: 2.319546e-01 +NonBlocking: 2.488415e-01 +Pairwise Hierarchical: 2.498900e-01 +Pairwise Multileader: 2.471419e-01 +Pairwise Node Aware: 2.313096e-01 +Pairwise Locality Aware: 2.196701e-01 +Pairwise Multileader Locality: 2.200911e-01 +Nonblocking Hierarchical: 2.642157e-01 +Nonblocking Multileader: 2.671126e-01 +Nonblocking Node Aware: 2.617167e-01 +Nonblocking Locality Aware: 2.118310e-01 +Nonblocking Multileader Locality: 2.164828e-01 +Size 2048 +PMPI: 4.484462e-01 +Pairwise: 4.881543e-01 +NonBlocking: 4.380982e-01 +Test 3 Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.246205e-03 +Pairwise: 2.771837e-02 +NonBlocking: 2.939384e-02 +Pairwise Hierarchical: 3.190094e-02 +Pairwise Multileader: 2.920927e-02 +Pairwise Node Aware: 2.155771e-02 +Pairwise Locality Aware: 3.322819e-02 +Pairwise Multileader Locality: 2.097959e-02 +Nonblocking Hierarchical: 2.743776e-02 +Nonblocking Multileader: 2.855985e-02 +Nonblocking Node Aware: 2.083218e-02 +Nonblocking Locality Aware: 2.787320e-02 +Nonblocking Multileader Locality: 2.321148e-02 +Size 2 +PMPI: 1.642154e-03 +Pairwise: 3.858737e-02 +NonBlocking: 2.324719e-02 +Pairwise Hierarchical: 3.091423e-02 +Pairwise Multileader: 2.586341e-02 +Pairwise Node Aware: 2.555411e-02 +Pairwise Locality Aware: 2.497034e-02 +Pairwise Multileader Locality: 2.386566e-02 +Nonblocking Hierarchical: 2.777198e-02 +Nonblocking Multileader: 3.135462e-02 +Nonblocking Node Aware: 1.967427e-02 +Nonblocking Locality Aware: 2.787608e-02 +Nonblocking Multileader Locality: 2.397105e-02 +Size 4 +PMPI: 4.592345e-03 +Pairwise: 4.579277e-02 +NonBlocking: 5.780983e-02 +Pairwise Hierarchical: 6.162173e-02 +Pairwise Multileader: 5.481073e-02 +Pairwise Node Aware: 4.687905e-02 +Pairwise Locality Aware: 8.312224e-02 +Pairwise Multileader Locality: 5.374200e-02 +Nonblocking Hierarchical: 6.251643e-02 +Nonblocking Multileader: 5.312571e-02 +Nonblocking Node Aware: 4.669301e-02 +Nonblocking Locality Aware: 4.414793e-02 +Nonblocking Multileader Locality: 5.149313e-02 +Size 8 +PMPI: 8.035749e-03 +Pairwise: 6.966177e-02 +NonBlocking: 5.198357e-02 +Pairwise Hierarchical: 5.603069e-02 +Pairwise Multileader: 4.730235e-02 +Pairwise Node Aware: 5.167964e-02 +Pairwise Locality Aware: 6.695110e-02 +Pairwise Multileader Locality: 5.401915e-02 +Nonblocking Hierarchical: 5.046341e-02 +Nonblocking Multileader: 4.225679e-02 +Nonblocking Node Aware: 5.801305e-02 +Nonblocking Locality Aware: 4.602839e-02 +Nonblocking Multileader Locality: 4.780532e-02 +Size 16 +PMPI: 7.340360e-03 +Pairwise: 4.703347e-02 +NonBlocking: 5.347698e-02 +Pairwise Hierarchical: 4.652665e-02 +Pairwise Multileader: 4.321476e-02 +Pairwise Node Aware: 6.210484e-02 +Pairwise Locality Aware: 7.104199e-02 +Pairwise Multileader Locality: 4.461489e-02 +Nonblocking Hierarchical: 5.746926e-02 +Nonblocking Multileader: 4.803938e-02 +Nonblocking Node Aware: 4.323668e-02 +Nonblocking Locality Aware: 6.385954e-02 +Nonblocking Multileader Locality: 4.832358e-02 +Size 32 +PMPI: 1.360981e-02 +Pairwise: 5.702171e-02 +NonBlocking: 4.991930e-02 +Pairwise Hierarchical: 7.033996e-02 +Pairwise Multileader: 5.338687e-02 +Pairwise Node Aware: 4.976314e-02 +Pairwise Locality Aware: 7.658075e-02 +Pairwise Multileader Locality: 7.914961e-02 +Nonblocking Hierarchical: 5.062429e-02 +Nonblocking Multileader: 5.545705e-02 +Nonblocking Node Aware: 9.218425e-02 +Nonblocking Locality Aware: 5.664334e-02 +Nonblocking Multileader Locality: 5.494613e-02 +Size 64 +PMPI: 2.229373e-02 +Pairwise: 5.484528e-02 +NonBlocking: 6.280481e-02 +Pairwise Hierarchical: 5.065947e-02 +Pairwise Multileader: 5.280549e-02 +Pairwise Node Aware: 5.852343e-02 +Pairwise Locality Aware: 8.188483e-02 +Pairwise Multileader Locality: 5.720282e-02 +Nonblocking Hierarchical: 6.308517e-02 +Nonblocking Multileader: 7.061371e-02 +Nonblocking Node Aware: 6.603695e-02 +Nonblocking Locality Aware: 6.195343e-02 +Nonblocking Multileader Locality: 4.951889e-02 +Size 128 +PMPI: 5.219849e-02 +Pairwise: 6.234890e-02 +NonBlocking: 6.580357e-02 +Pairwise Hierarchical: 5.785998e-02 +Pairwise Multileader: 5.830584e-02 +Pairwise Node Aware: 6.539454e-02 +Pairwise Locality Aware: 6.035747e-02 +Pairwise Multileader Locality: 6.233491e-02 +Nonblocking Hierarchical: 6.333394e-02 +Nonblocking Multileader: 5.966596e-02 +Nonblocking Node Aware: 6.169930e-02 +Nonblocking Locality Aware: 5.998808e-02 +Nonblocking Multileader Locality: 6.518273e-02 +Size 256 +PMPI: 1.018768e-01 +Pairwise: 8.439122e-02 +NonBlocking: 8.410699e-02 +Pairwise Hierarchical: 8.731635e-02 +Pairwise Multileader: 8.027766e-02 +Pairwise Node Aware: 1.117799e-01 +Pairwise Locality Aware: 8.858046e-02 +Pairwise Multileader Locality: 7.675181e-02 +Nonblocking Hierarchical: 9.659376e-02 +Nonblocking Multileader: 1.129712e-01 +Nonblocking Node Aware: 7.407913e-02 +Nonblocking Locality Aware: 8.638916e-02 +Nonblocking Multileader Locality: 7.755937e-02 +Size 512 +PMPI: 5.876054e-01 +Pairwise: 1.305945e-01 +NonBlocking: 1.426622e-01 +Pairwise Hierarchical: 1.399910e-01 +Pairwise Multileader: 1.279068e-01 +Pairwise Node Aware: 1.166689e-01 +Pairwise Locality Aware: 1.250507e-01 +Pairwise Multileader Locality: 1.234933e-01 +Nonblocking Hierarchical: 1.194385e-01 +Nonblocking Multileader: 1.156871e-01 +Nonblocking Node Aware: 1.289303e-01 +Nonblocking Locality Aware: 1.228299e-01 +Nonblocking Multileader Locality: 1.218491e-01 +Size 1024 +PMPI: 2.476676e-01 +Pairwise: 2.176899e-01 +NonBlocking: 2.082872e-01 +Pairwise Hierarchical: 2.286725e-01 +Pairwise Multileader: 2.452643e-01 +Pairwise Node Aware: 2.400099e-01 +Pairwise Locality Aware: 4.621774e-01 +Pairwise Multileader Locality: 7.375596e-01 +Nonblocking Hierarchical: 3.958770e-01 +Nonblocking Multileader: 2.281266e-01 +Nonblocking Node Aware: 2.186069e-01 +Nonblocking Locality Aware: 2.188423e-01 +Nonblocking Multileader Locality: 2.227004e-01 +Size 2048 +PMPI: 4.289845e-01 +Pairwise: 4.221499e-01 +NonBlocking: 4.440187e-01 diff --git a/runscripts/large_variation/alltoall_N4.2360537.err b/runscripts/large_variation/alltoall_N4.2360537.err new file mode 100644 index 000000000..5297dc52a --- /dev/null +++ b/runscripts/large_variation/alltoall_N4.2360537.err @@ -0,0 +1,103 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2360537.0 ON dane132 CANCELLED AT 2025-04-28T15:04:43 *** +slurmstepd: error: *** STEP 2360537.0 ON dane132 CANCELLED AT 2025-04-28T15:04:43 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1540: tasks 300-399: Killed +srun: error: dane704: tasks 200-299: Killed +srun: error: dane518: tasks 100-199: Killed +srun: error: dane132: tasks 0-99: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2360537.1 ON dane132 CANCELLED AT 2025-04-28T15:08:30 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1540: tasks 300-399: Killed +srun: error: dane704: tasks 200-299: Killed +srun: error: dane518: tasks 100-199: Killed +srun: error: dane132: tasks 0-99: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2360537.2 ON dane132 CANCELLED AT 2025-04-28T15:12:21 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1540: tasks 300-399: Killed +srun: error: dane704: tasks 200-299: Killed +srun: error: dane518: tasks 100-199: Killed +srun: error: dane132: tasks 0-99: Killed diff --git a/runscripts/large_variation/alltoall_N4.2360537.out b/runscripts/large_variation/alltoall_N4.2360537.out new file mode 100644 index 000000000..3fed3af5d --- /dev/null +++ b/runscripts/large_variation/alltoall_N4.2360537.out @@ -0,0 +1,603 @@ +Test 1 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 9.629160e-05 +Pairwise: 1.230724e-03 +NonBlocking: 1.081387e-03 +Pairwise Hierarchical: 1.263463e-03 +Pairwise Multileader: 1.031409e-03 +Pairwise Node Aware: 1.129176e-03 +Pairwise Locality Aware: 1.313658e-03 +Pairwise Multileader Locality: 1.143205e-03 +Nonblocking Hierarchical: 1.105926e-03 +Nonblocking Multileader: 1.245459e-03 +Nonblocking Node Aware: 1.104112e-03 +Nonblocking Locality Aware: 1.233362e-03 +Nonblocking Multileader Locality: 1.197630e-03 +Size 2 +PMPI: 1.008519e-04 +Pairwise: 1.094823e-03 +NonBlocking: 1.620274e-03 +Pairwise Hierarchical: 1.090499e-03 +Pairwise Multileader: 1.099432e-03 +Pairwise Node Aware: 1.215572e-03 +Pairwise Locality Aware: 1.075930e-03 +Pairwise Multileader Locality: 1.232975e-03 +Nonblocking Hierarchical: 1.311289e-03 +Nonblocking Multileader: 1.202097e-03 +Nonblocking Node Aware: 1.030429e-03 +Nonblocking Locality Aware: 1.320324e-03 +Nonblocking Multileader Locality: 1.079196e-03 +Size 4 +PMPI: 1.652487e-04 +Pairwise: 2.671986e-03 +NonBlocking: 2.347732e-03 +Pairwise Hierarchical: 2.777450e-03 +Pairwise Multileader: 3.179439e-03 +Pairwise Node Aware: 2.254445e-03 +Pairwise Locality Aware: 2.607066e-03 +Pairwise Multileader Locality: 2.352774e-03 +Nonblocking Hierarchical: 2.156382e-03 +Nonblocking Multileader: 3.232190e-03 +Nonblocking Node Aware: 2.676535e-03 +Nonblocking Locality Aware: 2.360041e-03 +Nonblocking Multileader Locality: 2.270735e-03 +Size 8 +PMPI: 2.606551e-04 +Pairwise: 2.381865e-03 +NonBlocking: 2.779533e-03 +Pairwise Hierarchical: 3.245795e-03 +Pairwise Multileader: 2.163731e-03 +Pairwise Node Aware: 3.349113e-03 +Pairwise Locality Aware: 2.235471e-03 +Pairwise Multileader Locality: 2.423178e-03 +Nonblocking Hierarchical: 2.491001e-03 +Nonblocking Multileader: 2.561747e-03 +Nonblocking Node Aware: 2.439878e-03 +Nonblocking Locality Aware: 2.106897e-03 +Nonblocking Multileader Locality: 2.810432e-03 +Size 16 +PMPI: 4.287498e-04 +Pairwise: 2.314044e-03 +NonBlocking: 2.591416e-03 +Pairwise Hierarchical: 2.465972e-03 +Pairwise Multileader: 2.315147e-03 +Pairwise Node Aware: 2.339446e-03 +Pairwise Locality Aware: 2.880989e-03 +Pairwise Multileader Locality: 2.272282e-03 +Nonblocking Hierarchical: 2.122349e-03 +Nonblocking Multileader: 2.661041e-03 +Nonblocking Node Aware: 2.784221e-03 +Nonblocking Locality Aware: 2.371002e-03 +Nonblocking Multileader Locality: 2.405980e-03 +Size 32 +PMPI: 7.570054e-04 +Pairwise: 2.304190e-03 +NonBlocking: 2.300413e-03 +Pairwise Hierarchical: 2.849830e-03 +Pairwise Multileader: 2.458815e-03 +Pairwise Node Aware: 2.366046e-03 +Pairwise Locality Aware: 2.618913e-03 +Pairwise Multileader Locality: 2.804838e-03 +Nonblocking Hierarchical: 2.312021e-03 +Nonblocking Multileader: 2.644027e-03 +Nonblocking Node Aware: 2.505395e-03 +Nonblocking Locality Aware: 2.280749e-03 +Nonblocking Multileader Locality: 2.851871e-03 +Size 64 +PMPI: 1.439005e-03 +Pairwise: 2.795872e-03 +NonBlocking: 2.938994e-03 +Pairwise Hierarchical: 2.648971e-03 +Pairwise Multileader: 2.532548e-03 +Pairwise Node Aware: 2.987062e-03 +Pairwise Locality Aware: 2.678438e-03 +Pairwise Multileader Locality: 2.752154e-03 +Nonblocking Hierarchical: 2.472623e-03 +Nonblocking Multileader: 3.165863e-03 +Nonblocking Node Aware: 2.400467e-03 +Nonblocking Locality Aware: 2.883960e-03 +Nonblocking Multileader Locality: 3.539539e-03 +Size 128 +PMPI: 2.349531e-03 +Pairwise: 3.175064e-03 +NonBlocking: 3.491498e-03 +Pairwise Hierarchical: 3.513779e-03 +Pairwise Multileader: 3.226952e-03 +Pairwise Node Aware: 3.479589e-03 +Pairwise Locality Aware: 3.469407e-03 +Pairwise Multileader Locality: 2.865622e-03 +Nonblocking Hierarchical: 3.044478e-03 +Nonblocking Multileader: 3.664037e-03 +Nonblocking Node Aware: 3.086492e-03 +Nonblocking Locality Aware: 3.042251e-03 +Nonblocking Multileader Locality: 2.970777e-03 +Size 256 +PMPI: 4.781352e-03 +Pairwise: 4.503023e-03 +NonBlocking: 5.043706e-03 +Pairwise Hierarchical: 5.574857e-03 +Pairwise Multileader: 4.335043e-03 +Pairwise Node Aware: 4.689665e-03 +Pairwise Locality Aware: 5.201884e-03 +Pairwise Multileader Locality: 4.527238e-03 +Nonblocking Hierarchical: 4.992861e-03 +Nonblocking Multileader: 4.999473e-03 +Nonblocking Node Aware: 4.850882e-03 +Nonblocking Locality Aware: 4.718679e-03 +Nonblocking Multileader Locality: 5.522038e-03 +Size 512 +PMPI: 7.575657e-03 +Pairwise: 7.816517e-03 +NonBlocking: 8.112280e-03 +Pairwise Hierarchical: 7.627304e-03 +Pairwise Multileader: 8.443543e-03 +Pairwise Node Aware: 8.317672e-03 +Pairwise Locality Aware: 7.456648e-03 +Pairwise Multileader Locality: 7.659914e-03 +Nonblocking Hierarchical: 8.375695e-03 +Nonblocking Multileader: 7.605350e-03 +Nonblocking Node Aware: 8.045429e-03 +Nonblocking Locality Aware: 8.536798e-03 +Nonblocking Multileader Locality: 7.783621e-03 +Size 1024 +PMPI: 1.474983e-02 +Pairwise: 1.569993e-02 +NonBlocking: 1.388627e-02 +Pairwise Hierarchical: 1.446091e-02 +Pairwise Multileader: 1.498372e-02 +Pairwise Node Aware: 1.425112e-02 +Pairwise Locality Aware: 1.427026e-02 +Pairwise Multileader Locality: 1.491021e-02 +Nonblocking Hierarchical: 1.377211e-02 +Nonblocking Multileader: 1.429367e-02 +Nonblocking Node Aware: 1.450767e-02 +Nonblocking Locality Aware: 1.459682e-02 +Nonblocking Multileader Locality: 1.402632e-02 +Size 2048 +PMPI: 2.981008e-02 +Pairwise: 2.680657e-02 +NonBlocking: 2.841213e-02 +Pairwise Hierarchical: 2.785661e-02 +Pairwise Multileader: 2.743801e-02 +Pairwise Node Aware: 2.767473e-02 +Pairwise Locality Aware: 2.776659e-02 +Pairwise Multileader Locality: 2.878740e-02 +Nonblocking Hierarchical: 2.940561e-02 +Nonblocking Multileader: 2.725285e-02 +Nonblocking Node Aware: 2.788249e-02 +Nonblocking Locality Aware: 2.819197e-02 +Nonblocking Multileader Locality: 2.722630e-02 +Size 4096 +PMPI: 5.974116e-02 +Pairwise: 5.427857e-02 +NonBlocking: 5.427595e-02 +Pairwise Hierarchical: 5.305871e-02 +Pairwise Multileader: 5.478593e-02 +Pairwise Node Aware: 5.466564e-02 +Pairwise Locality Aware: 5.520663e-02 +Pairwise Multileader Locality: 5.437924e-02 +Nonblocking Hierarchical: 5.206522e-02 +Nonblocking Multileader: 5.461136e-02 +Nonblocking Node Aware: 5.057303e-02 +Nonblocking Locality Aware: 5.487561e-02 +Nonblocking Multileader Locality: 5.676738e-02 +Size 8192 +PMPI: 1.048882e-01 +Pairwise: 1.010785e-01 +NonBlocking: 1.033546e-01 +Pairwise Hierarchical: 9.930656e-02 +Pairwise Multileader: 1.056455e-01 +Pairwise Node Aware: 9.973407e-02 +Pairwise Locality Aware: 1.122174e-01 +Pairwise Multileader Locality: 1.062733e-01 +Nonblocking Hierarchical: 9.923313e-02 +Nonblocking Multileader: 1.043754e-01 +Nonblocking Node Aware: 1.034866e-01 +Nonblocking Locality Aware: 1.022285e-01 +Nonblocking Multileader Locality: 9.926860e-02 +Size 16384 +PMPI: 1.822942e-01 +Pairwise: 1.750685e-01 +NonBlocking: 1.632502e-01 +Test 2 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 9.860826e-05 +Pairwise: 1.219649e-03 +NonBlocking: 1.108600e-03 +Pairwise Hierarchical: 1.147481e-03 +Pairwise Multileader: 1.523649e-03 +Pairwise Node Aware: 1.026202e-03 +Pairwise Locality Aware: 1.170373e-03 +Pairwise Multileader Locality: 1.485228e-03 +Nonblocking Hierarchical: 1.124501e-03 +Nonblocking Multileader: 1.119652e-03 +Nonblocking Node Aware: 1.337072e-03 +Nonblocking Locality Aware: 1.246053e-03 +Nonblocking Multileader Locality: 1.114369e-03 +Size 2 +PMPI: 1.107899e-04 +Pairwise: 1.466296e-03 +NonBlocking: 1.073725e-03 +Pairwise Hierarchical: 1.026835e-03 +Pairwise Multileader: 1.241449e-03 +Pairwise Node Aware: 1.248419e-03 +Pairwise Locality Aware: 1.042245e-03 +Pairwise Multileader Locality: 1.125495e-03 +Nonblocking Hierarchical: 1.248313e-03 +Nonblocking Multileader: 1.345892e-03 +Nonblocking Node Aware: 1.179216e-03 +Nonblocking Locality Aware: 1.189221e-03 +Nonblocking Multileader Locality: 1.401873e-03 +Size 4 +PMPI: 1.356662e-04 +Pairwise: 2.278854e-03 +NonBlocking: 3.416848e-03 +Pairwise Hierarchical: 3.004841e-03 +Pairwise Multileader: 2.189405e-03 +Pairwise Node Aware: 2.061554e-03 +Pairwise Locality Aware: 2.311525e-03 +Pairwise Multileader Locality: 2.243352e-03 +Nonblocking Hierarchical: 2.087596e-03 +Nonblocking Multileader: 2.222907e-03 +Nonblocking Node Aware: 2.258783e-03 +Nonblocking Locality Aware: 1.988350e-03 +Nonblocking Multileader Locality: 2.116602e-03 +Size 8 +PMPI: 2.467012e-04 +Pairwise: 2.218034e-03 +NonBlocking: 2.513448e-03 +Pairwise Hierarchical: 2.496292e-03 +Pairwise Multileader: 2.360557e-03 +Pairwise Node Aware: 2.641063e-03 +Pairwise Locality Aware: 2.704117e-03 +Pairwise Multileader Locality: 2.128073e-03 +Nonblocking Hierarchical: 2.695476e-03 +Nonblocking Multileader: 2.167096e-03 +Nonblocking Node Aware: 2.438187e-03 +Nonblocking Locality Aware: 2.366174e-03 +Nonblocking Multileader Locality: 2.422682e-03 +Size 16 +PMPI: 4.340086e-04 +Pairwise: 2.297528e-03 +NonBlocking: 2.771495e-03 +Pairwise Hierarchical: 2.330633e-03 +Pairwise Multileader: 2.765541e-03 +Pairwise Node Aware: 2.459886e-03 +Pairwise Locality Aware: 2.431671e-03 +Pairwise Multileader Locality: 3.097319e-03 +Nonblocking Hierarchical: 2.278907e-03 +Nonblocking Multileader: 2.329111e-03 +Nonblocking Node Aware: 2.784853e-03 +Nonblocking Locality Aware: 2.500376e-03 +Nonblocking Multileader Locality: 2.614911e-03 +Size 32 +PMPI: 7.622357e-04 +Pairwise: 2.781305e-03 +NonBlocking: 2.203494e-03 +Pairwise Hierarchical: 2.985038e-03 +Pairwise Multileader: 2.237905e-03 +Pairwise Node Aware: 2.328233e-03 +Pairwise Locality Aware: 3.078220e-03 +Pairwise Multileader Locality: 2.552968e-03 +Nonblocking Hierarchical: 2.380247e-03 +Nonblocking Multileader: 2.587044e-03 +Nonblocking Node Aware: 3.033633e-03 +Nonblocking Locality Aware: 2.263874e-03 +Nonblocking Multileader Locality: 2.252197e-03 +Size 64 +PMPI: 1.530884e-03 +Pairwise: 2.611426e-03 +NonBlocking: 2.791849e-03 +Pairwise Hierarchical: 2.699954e-03 +Pairwise Multileader: 2.359686e-03 +Pairwise Node Aware: 2.973872e-03 +Pairwise Locality Aware: 2.691472e-03 +Pairwise Multileader Locality: 2.743138e-03 +Nonblocking Hierarchical: 2.971578e-03 +Nonblocking Multileader: 2.528403e-03 +Nonblocking Node Aware: 2.687324e-03 +Nonblocking Locality Aware: 2.752782e-03 +Nonblocking Multileader Locality: 2.583080e-03 +Size 128 +PMPI: 2.543441e-03 +Pairwise: 2.758633e-03 +NonBlocking: 3.392278e-03 +Pairwise Hierarchical: 3.359388e-03 +Pairwise Multileader: 3.096835e-03 +Pairwise Node Aware: 3.528358e-03 +Pairwise Locality Aware: 3.601793e-03 +Pairwise Multileader Locality: 3.072388e-03 +Nonblocking Hierarchical: 3.111663e-03 +Nonblocking Multileader: 4.117345e-03 +Nonblocking Node Aware: 2.964938e-03 +Nonblocking Locality Aware: 3.028005e-03 +Nonblocking Multileader Locality: 3.006039e-03 +Size 256 +PMPI: 4.705182e-03 +Pairwise: 4.646406e-03 +NonBlocking: 5.360149e-03 +Pairwise Hierarchical: 4.881591e-03 +Pairwise Multileader: 6.079306e-03 +Pairwise Node Aware: 6.182059e-03 +Pairwise Locality Aware: 5.200727e-03 +Pairwise Multileader Locality: 4.932190e-03 +Nonblocking Hierarchical: 4.995234e-03 +Nonblocking Multileader: 5.961313e-03 +Nonblocking Node Aware: 4.639222e-03 +Nonblocking Locality Aware: 4.689730e-03 +Nonblocking Multileader Locality: 5.347076e-03 +Size 512 +PMPI: 9.152151e-03 +Pairwise: 7.738878e-03 +NonBlocking: 8.548002e-03 +Pairwise Hierarchical: 7.874365e-03 +Pairwise Multileader: 7.852275e-03 +Pairwise Node Aware: 7.935804e-03 +Pairwise Locality Aware: 8.337242e-03 +Pairwise Multileader Locality: 7.625298e-03 +Nonblocking Hierarchical: 8.083670e-03 +Nonblocking Multileader: 8.040416e-03 +Nonblocking Node Aware: 7.565682e-03 +Nonblocking Locality Aware: 8.122535e-03 +Nonblocking Multileader Locality: 8.657332e-03 +Size 1024 +PMPI: 1.491441e-02 +Pairwise: 1.439425e-02 +NonBlocking: 1.520810e-02 +Pairwise Hierarchical: 1.430123e-02 +Pairwise Multileader: 1.462796e-02 +Pairwise Node Aware: 1.503053e-02 +Pairwise Locality Aware: 1.365149e-02 +Pairwise Multileader Locality: 1.514627e-02 +Nonblocking Hierarchical: 1.433250e-02 +Nonblocking Multileader: 1.382261e-02 +Nonblocking Node Aware: 1.488482e-02 +Nonblocking Locality Aware: 1.464068e-02 +Nonblocking Multileader Locality: 1.450480e-02 +Size 2048 +PMPI: 2.903380e-02 +Pairwise: 2.816611e-02 +NonBlocking: 2.688742e-02 +Pairwise Hierarchical: 2.887485e-02 +Pairwise Multileader: 2.871881e-02 +Pairwise Node Aware: 2.834821e-02 +Pairwise Locality Aware: 2.842148e-02 +Pairwise Multileader Locality: 2.820440e-02 +Nonblocking Hierarchical: 2.874692e-02 +Nonblocking Multileader: 2.850856e-02 +Nonblocking Node Aware: 2.754389e-02 +Nonblocking Locality Aware: 2.880991e-02 +Nonblocking Multileader Locality: 2.751582e-02 +Size 4096 +PMPI: 6.011763e-02 +Pairwise: 5.598192e-02 +NonBlocking: 5.296732e-02 +Pairwise Hierarchical: 5.683577e-02 +Pairwise Multileader: 5.617113e-02 +Pairwise Node Aware: 5.494982e-02 +Pairwise Locality Aware: 5.380007e-02 +Pairwise Multileader Locality: 5.211527e-02 +Nonblocking Hierarchical: 5.709453e-02 +Nonblocking Multileader: 6.113136e-02 +Nonblocking Node Aware: 6.101218e-02 +Nonblocking Locality Aware: 5.813805e-02 +Nonblocking Multileader Locality: 5.896305e-02 +Size 8192 +PMPI: 1.184387e-01 +Pairwise: 1.058936e-01 +NonBlocking: 9.920959e-02 +Pairwise Hierarchical: 1.010078e-01 +Pairwise Multileader: 1.040025e-01 +Pairwise Node Aware: 1.048925e-01 +Pairwise Locality Aware: 9.876482e-02 +Pairwise Multileader Locality: 1.046464e-01 +Nonblocking Hierarchical: 9.977140e-02 +Nonblocking Multileader: 1.028430e-01 +Nonblocking Node Aware: 9.982207e-02 +Nonblocking Locality Aware: 1.096041e-01 +Nonblocking Multileader Locality: 1.002241e-01 +Size 16384 +PMPI: 1.780843e-01 +Pairwise: 1.774439e-01 +NonBlocking: 1.648443e-01 +Test 3 Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 1.294340e-04 +Pairwise: 1.105743e-03 +NonBlocking: 1.058611e-03 +Pairwise Hierarchical: 1.180831e-03 +Pairwise Multileader: 1.205149e-03 +Pairwise Node Aware: 1.076317e-03 +Pairwise Locality Aware: 1.681534e-03 +Pairwise Multileader Locality: 1.396770e-03 +Nonblocking Hierarchical: 1.342352e-03 +Nonblocking Multileader: 1.162957e-03 +Nonblocking Node Aware: 1.531131e-03 +Nonblocking Locality Aware: 1.052140e-03 +Nonblocking Multileader Locality: 1.387062e-03 +Size 2 +PMPI: 9.789916e-05 +Pairwise: 1.182371e-03 +NonBlocking: 1.300506e-03 +Pairwise Hierarchical: 1.057764e-03 +Pairwise Multileader: 1.203302e-03 +Pairwise Node Aware: 1.216811e-03 +Pairwise Locality Aware: 1.404742e-03 +Pairwise Multileader Locality: 1.116686e-03 +Nonblocking Hierarchical: 1.356526e-03 +Nonblocking Multileader: 1.233562e-03 +Nonblocking Node Aware: 1.068525e-03 +Nonblocking Locality Aware: 1.206900e-03 +Nonblocking Multileader Locality: 1.334836e-03 +Size 4 +PMPI: 1.375581e-04 +Pairwise: 2.133748e-03 +NonBlocking: 2.856330e-03 +Pairwise Hierarchical: 2.142567e-03 +Pairwise Multileader: 2.381836e-03 +Pairwise Node Aware: 2.665437e-03 +Pairwise Locality Aware: 2.317456e-03 +Pairwise Multileader Locality: 2.539615e-03 +Nonblocking Hierarchical: 2.785476e-03 +Nonblocking Multileader: 2.377035e-03 +Nonblocking Node Aware: 2.122288e-03 +Nonblocking Locality Aware: 2.782067e-03 +Nonblocking Multileader Locality: 2.270037e-03 +Size 8 +PMPI: 2.567130e-04 +Pairwise: 2.370842e-03 +NonBlocking: 2.895701e-03 +Pairwise Hierarchical: 2.338355e-03 +Pairwise Multileader: 2.412992e-03 +Pairwise Node Aware: 2.992173e-03 +Pairwise Locality Aware: 2.287385e-03 +Pairwise Multileader Locality: 2.503082e-03 +Nonblocking Hierarchical: 2.634246e-03 +Nonblocking Multileader: 2.544626e-03 +Nonblocking Node Aware: 2.336081e-03 +Nonblocking Locality Aware: 2.619533e-03 +Nonblocking Multileader Locality: 2.300828e-03 +Size 16 +PMPI: 4.353045e-04 +Pairwise: 2.759665e-03 +NonBlocking: 2.247321e-03 +Pairwise Hierarchical: 2.923613e-03 +Pairwise Multileader: 2.390810e-03 +Pairwise Node Aware: 2.349137e-03 +Pairwise Locality Aware: 2.809183e-03 +Pairwise Multileader Locality: 2.385692e-03 +Nonblocking Hierarchical: 2.201875e-03 +Nonblocking Multileader: 3.020223e-03 +Nonblocking Node Aware: 2.190225e-03 +Nonblocking Locality Aware: 2.602929e-03 +Nonblocking Multileader Locality: 2.448472e-03 +Size 32 +PMPI: 7.078702e-04 +Pairwise: 2.587260e-03 +NonBlocking: 2.457939e-03 +Pairwise Hierarchical: 2.402497e-03 +Pairwise Multileader: 2.808923e-03 +Pairwise Node Aware: 2.296646e-03 +Pairwise Locality Aware: 2.209220e-03 +Pairwise Multileader Locality: 2.820209e-03 +Nonblocking Hierarchical: 2.549473e-03 +Nonblocking Multileader: 2.329342e-03 +Nonblocking Node Aware: 2.858810e-03 +Nonblocking Locality Aware: 2.413380e-03 +Nonblocking Multileader Locality: 2.278619e-03 +Size 64 +PMPI: 1.438631e-03 +Pairwise: 2.977866e-03 +NonBlocking: 2.543559e-03 +Pairwise Hierarchical: 2.896111e-03 +Pairwise Multileader: 3.233197e-03 +Pairwise Node Aware: 2.569986e-03 +Pairwise Locality Aware: 2.953458e-03 +Pairwise Multileader Locality: 3.520862e-03 +Nonblocking Hierarchical: 2.763175e-03 +Nonblocking Multileader: 2.759791e-03 +Nonblocking Node Aware: 2.724931e-03 +Nonblocking Locality Aware: 3.073360e-03 +Nonblocking Multileader Locality: 2.849240e-03 +Size 128 +PMPI: 2.554637e-03 +Pairwise: 2.871670e-03 +NonBlocking: 3.882775e-03 +Pairwise Hierarchical: 3.433673e-03 +Pairwise Multileader: 3.088037e-03 +Pairwise Node Aware: 3.276771e-03 +Pairwise Locality Aware: 3.735764e-03 +Pairwise Multileader Locality: 3.033474e-03 +Nonblocking Hierarchical: 3.023190e-03 +Nonblocking Multileader: 3.871567e-03 +Nonblocking Node Aware: 3.215543e-03 +Nonblocking Locality Aware: 3.366483e-03 +Nonblocking Multileader Locality: 3.799293e-03 +Size 256 +PMPI: 4.458506e-03 +Pairwise: 4.964308e-03 +NonBlocking: 4.929998e-03 +Pairwise Hierarchical: 4.616517e-03 +Pairwise Multileader: 5.447465e-03 +Pairwise Node Aware: 4.776841e-03 +Pairwise Locality Aware: 4.547929e-03 +Pairwise Multileader Locality: 4.934618e-03 +Nonblocking Hierarchical: 5.192867e-03 +Nonblocking Multileader: 4.656489e-03 +Nonblocking Node Aware: 4.745737e-03 +Nonblocking Locality Aware: 5.201543e-03 +Nonblocking Multileader Locality: 4.753158e-03 +Size 512 +PMPI: 7.794112e-03 +Pairwise: 8.333074e-03 +NonBlocking: 7.635762e-03 +Pairwise Hierarchical: 8.353788e-03 +Pairwise Multileader: 8.832120e-03 +Pairwise Node Aware: 7.764465e-03 +Pairwise Locality Aware: 7.979630e-03 +Pairwise Multileader Locality: 8.558465e-03 +Nonblocking Hierarchical: 7.902053e-03 +Nonblocking Multileader: 7.993798e-03 +Nonblocking Node Aware: 8.753751e-03 +Nonblocking Locality Aware: 8.349315e-03 +Nonblocking Multileader Locality: 8.008212e-03 +Size 1024 +PMPI: 1.547119e-02 +Pairwise: 1.460059e-02 +NonBlocking: 1.450753e-02 +Pairwise Hierarchical: 1.495964e-02 +Pairwise Multileader: 1.532782e-02 +Pairwise Node Aware: 1.596198e-02 +Pairwise Locality Aware: 1.525062e-02 +Pairwise Multileader Locality: 1.464991e-02 +Nonblocking Hierarchical: 1.555019e-02 +Nonblocking Multileader: 1.513085e-02 +Nonblocking Node Aware: 1.488789e-02 +Nonblocking Locality Aware: 1.411558e-02 +Nonblocking Multileader Locality: 1.375323e-02 +Size 2048 +PMPI: 2.904060e-02 +Pairwise: 2.812144e-02 +NonBlocking: 2.861141e-02 +Pairwise Hierarchical: 2.946434e-02 +Pairwise Multileader: 2.791669e-02 +Pairwise Node Aware: 2.871084e-02 +Pairwise Locality Aware: 2.931808e-02 +Pairwise Multileader Locality: 2.883570e-02 +Nonblocking Hierarchical: 2.803756e-02 +Nonblocking Multileader: 2.931449e-02 +Nonblocking Node Aware: 2.782990e-02 +Nonblocking Locality Aware: 2.867790e-02 +Nonblocking Multileader Locality: 2.955064e-02 +Size 4096 +PMPI: 5.921883e-02 +Pairwise: 5.820843e-02 +NonBlocking: 5.540468e-02 +Pairwise Hierarchical: 5.697699e-02 +Pairwise Multileader: 5.500245e-02 +Pairwise Node Aware: 5.377045e-02 +Pairwise Locality Aware: 5.428526e-02 +Pairwise Multileader Locality: 5.169337e-02 +Nonblocking Hierarchical: 5.405317e-02 +Nonblocking Multileader: 5.429125e-02 +Nonblocking Node Aware: 5.101792e-02 +Nonblocking Locality Aware: 5.536911e-02 +Nonblocking Multileader Locality: 5.444898e-02 +Size 8192 +PMPI: 1.035385e-01 +Pairwise: 1.043112e-01 +NonBlocking: 1.050093e-01 +Pairwise Hierarchical: 1.131411e-01 +Pairwise Multileader: 1.070837e-01 +Pairwise Node Aware: 1.067361e-01 +Pairwise Locality Aware: 1.035765e-01 +Pairwise Multileader Locality: 1.081378e-01 +Nonblocking Hierarchical: 1.094581e-01 +Nonblocking Multileader: 9.895391e-02 +Nonblocking Node Aware: 1.079249e-01 +Nonblocking Locality Aware: 9.974325e-02 +Nonblocking Multileader Locality: 1.063928e-01 +Size 16384 +PMPI: 1.793390e-01 +Pairwise: 1.787173e-01 +NonBlocking: 1.635528e-01 diff --git a/runscripts/large_variation/alltoall_N8.2360539.err b/runscripts/large_variation/alltoall_N8.2360539.err new file mode 100644 index 000000000..0bfc757b2 --- /dev/null +++ b/runscripts/large_variation/alltoall_N8.2360539.err @@ -0,0 +1,200 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2360539.0 ON dane441 CANCELLED AT 2025-04-28T15:11:32 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1194: tasks 400-499: Killed +srun: error: dane1523: tasks 700-799: Killed +srun: error: dane1474: tasks 600-699: Killed +srun: error: dane1473: tasks 500-599: Killed +srun: error: dane441: tasks 0-99: Killed +srun: error: dane501: tasks 100-199: Killed +srun: error: dane1016: tasks 300-399: Killed +srun: error: dane591: tasks 200-299: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2360539.1 ON dane441 CANCELLED AT 2025-04-28T15:15:14 *** +slurmstepd: error: *** STEP 2360539.1 ON dane441 CANCELLED AT 2025-04-28T15:15:14 *** +slurmstepd: error: *** STEP 2360539.1 ON dane441 CANCELLED AT 2025-04-28T15:15:14 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1194: tasks 400-499: Killed +srun: error: dane1523: tasks 700-799: Killed +srun: error: dane1473: tasks 500-599: Killed +srun: error: dane1474: tasks 600-699: Killed +srun: error: dane441: tasks 0-99: Killed +srun: error: dane501: tasks 100-199: Killed +srun: error: dane1016: tasks 300-399: Killed +srun: error: dane591: tasks 200-299: Killed +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2360539.2 ON dane441 CANCELLED AT 2025-04-28T15:18:59 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1194: tasks 400-499: Killed +srun: error: dane1523: tasks 700-799: Killed +srun: error: dane1474: tasks 600-699: Killed +srun: error: dane1473: tasks 500-599: Killed +srun: error: dane441: tasks 0-99: Killed +srun: error: dane501: tasks 100-199: Killed +srun: error: dane1016: tasks 300-399: Killed +srun: error: dane591: tasks 200-299: Killed diff --git a/runscripts/large_variation/alltoall_N8.2360539.out b/runscripts/large_variation/alltoall_N8.2360539.out new file mode 100644 index 000000000..d3654ed63 --- /dev/null +++ b/runscripts/large_variation/alltoall_N8.2360539.out @@ -0,0 +1,561 @@ +Test 1 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.811200e-04 +Pairwise: 3.521407e-03 +NonBlocking: 2.893949e-03 +Pairwise Hierarchical: 2.850623e-03 +Pairwise Multileader: 2.649692e-03 +Pairwise Node Aware: 3.295950e-03 +Pairwise Locality Aware: 2.781830e-03 +Pairwise Multileader Locality: 2.948131e-03 +Nonblocking Hierarchical: 2.721707e-03 +Nonblocking Multileader: 2.975260e-03 +Nonblocking Node Aware: 2.647479e-03 +Nonblocking Locality Aware: 3.230657e-03 +Nonblocking Multileader Locality: 3.003800e-03 +Size 2 +PMPI: 2.848469e-04 +Pairwise: 2.764648e-03 +NonBlocking: 3.412815e-03 +Pairwise Hierarchical: 2.588714e-03 +Pairwise Multileader: 3.078236e-03 +Pairwise Node Aware: 3.332137e-03 +Pairwise Locality Aware: 2.884582e-03 +Pairwise Multileader Locality: 2.490132e-03 +Nonblocking Hierarchical: 3.240283e-03 +Nonblocking Multileader: 2.613924e-03 +Nonblocking Node Aware: 2.530481e-03 +Nonblocking Locality Aware: 3.357627e-03 +Nonblocking Multileader Locality: 2.715237e-03 +Size 4 +PMPI: 4.319409e-04 +Pairwise: 6.628296e-03 +NonBlocking: 5.283093e-03 +Pairwise Hierarchical: 5.406963e-03 +Pairwise Multileader: 7.742311e-03 +Pairwise Node Aware: 5.558699e-03 +Pairwise Locality Aware: 5.072274e-03 +Pairwise Multileader Locality: 8.304914e-03 +Nonblocking Hierarchical: 6.437108e-03 +Nonblocking Multileader: 5.095819e-03 +Nonblocking Node Aware: 6.741505e-03 +Nonblocking Locality Aware: 5.504057e-03 +Nonblocking Multileader Locality: 6.639232e-03 +Size 8 +PMPI: 6.421274e-04 +Pairwise: 6.903866e-03 +NonBlocking: 5.549642e-03 +Pairwise Hierarchical: 7.085817e-03 +Pairwise Multileader: 5.281680e-03 +Pairwise Node Aware: 6.128135e-03 +Pairwise Locality Aware: 7.341169e-03 +Pairwise Multileader Locality: 5.503048e-03 +Nonblocking Hierarchical: 6.873990e-03 +Nonblocking Multileader: 5.306108e-03 +Nonblocking Node Aware: 7.122683e-03 +Nonblocking Locality Aware: 5.685600e-03 +Nonblocking Multileader Locality: 6.634251e-03 +Size 16 +PMPI: 1.123336e-03 +Pairwise: 7.246513e-03 +NonBlocking: 7.609859e-03 +Pairwise Hierarchical: 5.545381e-03 +Pairwise Multileader: 6.278111e-03 +Pairwise Node Aware: 5.686899e-03 +Pairwise Locality Aware: 8.107891e-03 +Pairwise Multileader Locality: 5.618498e-03 +Nonblocking Hierarchical: 7.273391e-03 +Nonblocking Multileader: 5.886780e-03 +Nonblocking Node Aware: 6.226604e-03 +Nonblocking Locality Aware: 6.708192e-03 +Nonblocking Multileader Locality: 7.687489e-03 +Size 32 +PMPI: 2.366367e-03 +Pairwise: 7.209700e-03 +NonBlocking: 5.693272e-03 +Pairwise Hierarchical: 6.528468e-03 +Pairwise Multileader: 8.215901e-03 +Pairwise Node Aware: 6.300336e-03 +Pairwise Locality Aware: 6.192620e-03 +Pairwise Multileader Locality: 7.562225e-03 +Nonblocking Hierarchical: 6.689360e-03 +Nonblocking Multileader: 5.752759e-03 +Nonblocking Node Aware: 7.594841e-03 +Nonblocking Locality Aware: 5.816568e-03 +Nonblocking Multileader Locality: 5.608625e-03 +Size 64 +PMPI: 4.631845e-03 +Pairwise: 7.340156e-03 +NonBlocking: 6.779959e-03 +Pairwise Hierarchical: 6.746740e-03 +Pairwise Multileader: 7.326655e-03 +Pairwise Node Aware: 7.302481e-03 +Pairwise Locality Aware: 5.912310e-03 +Pairwise Multileader Locality: 9.449649e-03 +Nonblocking Hierarchical: 7.480619e-03 +Nonblocking Multileader: 6.645541e-03 +Nonblocking Node Aware: 9.434145e-03 +Nonblocking Locality Aware: 6.185879e-03 +Nonblocking Multileader Locality: 7.363020e-03 +Size 128 +PMPI: 7.445735e-03 +Pairwise: 8.190747e-03 +NonBlocking: 8.435292e-03 +Pairwise Hierarchical: 1.049669e-02 +Pairwise Multileader: 9.214438e-03 +Pairwise Node Aware: 8.343832e-03 +Pairwise Locality Aware: 7.727410e-03 +Pairwise Multileader Locality: 1.107328e-02 +Nonblocking Hierarchical: 7.786075e-03 +Nonblocking Multileader: 1.159779e-02 +Nonblocking Node Aware: 8.168058e-03 +Nonblocking Locality Aware: 8.959125e-03 +Nonblocking Multileader Locality: 8.846739e-03 +Size 256 +PMPI: 1.525572e-02 +Pairwise: 1.397910e-02 +NonBlocking: 1.212778e-02 +Pairwise Hierarchical: 1.438073e-02 +Pairwise Multileader: 1.236922e-02 +Pairwise Node Aware: 1.401713e-02 +Pairwise Locality Aware: 1.304820e-02 +Pairwise Multileader Locality: 1.273262e-02 +Nonblocking Hierarchical: 1.327901e-02 +Nonblocking Multileader: 1.258120e-02 +Nonblocking Node Aware: 1.573474e-02 +Nonblocking Locality Aware: 1.251284e-02 +Nonblocking Multileader Locality: 1.347404e-02 +Size 512 +PMPI: 2.735067e-02 +Pairwise: 1.962917e-02 +NonBlocking: 2.259610e-02 +Pairwise Hierarchical: 2.225722e-02 +Pairwise Multileader: 2.118847e-02 +Pairwise Node Aware: 2.039104e-02 +Pairwise Locality Aware: 2.136252e-02 +Pairwise Multileader Locality: 1.897273e-02 +Nonblocking Hierarchical: 2.344867e-02 +Nonblocking Multileader: 1.915063e-02 +Nonblocking Node Aware: 1.993923e-02 +Nonblocking Locality Aware: 2.162658e-02 +Nonblocking Multileader Locality: 2.056860e-02 +Size 1024 +PMPI: 3.939826e-02 +Pairwise: 3.571359e-02 +NonBlocking: 4.039657e-02 +Pairwise Hierarchical: 3.577696e-02 +Pairwise Multileader: 4.093503e-02 +Pairwise Node Aware: 3.338122e-02 +Pairwise Locality Aware: 3.956468e-02 +Pairwise Multileader Locality: 3.496737e-02 +Nonblocking Hierarchical: 3.800526e-02 +Nonblocking Multileader: 3.724218e-02 +Nonblocking Node Aware: 3.805352e-02 +Nonblocking Locality Aware: 3.403266e-02 +Nonblocking Multileader Locality: 4.064833e-02 +Size 2048 +PMPI: 6.549559e-02 +Pairwise: 7.715260e-02 +NonBlocking: 6.432197e-02 +Pairwise Hierarchical: 6.685637e-02 +Pairwise Multileader: 7.496962e-02 +Pairwise Node Aware: 6.642739e-02 +Pairwise Locality Aware: 7.587650e-02 +Pairwise Multileader Locality: 6.297952e-02 +Nonblocking Hierarchical: 7.086930e-02 +Nonblocking Multileader: 7.400993e-02 +Nonblocking Node Aware: 7.301233e-02 +Nonblocking Locality Aware: 7.373612e-02 +Nonblocking Multileader Locality: 6.644344e-02 +Size 4096 +PMPI: 1.575963e-01 +Pairwise: 1.383478e-01 +NonBlocking: 1.328953e-01 +Pairwise Hierarchical: 1.433117e-01 +Pairwise Multileader: 1.369740e-01 +Pairwise Node Aware: 1.432697e-01 +Pairwise Locality Aware: 1.419812e-01 +Pairwise Multileader Locality: 1.587157e-01 +Nonblocking Hierarchical: 1.396812e-01 +Nonblocking Multileader: 1.561847e-01 +Nonblocking Node Aware: 1.279995e-01 +Nonblocking Locality Aware: 1.453339e-01 +Nonblocking Multileader Locality: 1.558796e-01 +Size 8192 +PMPI: 2.724658e-01 +Pairwise: 2.496833e-01 +NonBlocking: 2.517732e-01 +Test 2 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.834646e-04 +Pairwise: 3.475505e-03 +NonBlocking: 2.341288e-03 +Pairwise Hierarchical: 2.856451e-03 +Pairwise Multileader: 3.312976e-03 +Pairwise Node Aware: 2.946095e-03 +Pairwise Locality Aware: 2.891359e-03 +Pairwise Multileader Locality: 2.767704e-03 +Nonblocking Hierarchical: 3.214668e-03 +Nonblocking Multileader: 2.752761e-03 +Nonblocking Node Aware: 3.099998e-03 +Nonblocking Locality Aware: 3.197878e-03 +Nonblocking Multileader Locality: 2.487948e-03 +Size 2 +PMPI: 2.581658e-04 +Pairwise: 2.835552e-03 +NonBlocking: 2.825916e-03 +Pairwise Hierarchical: 2.576495e-03 +Pairwise Multileader: 3.281475e-03 +Pairwise Node Aware: 2.490161e-03 +Pairwise Locality Aware: 3.047476e-03 +Pairwise Multileader Locality: 2.932853e-03 +Nonblocking Hierarchical: 2.508005e-03 +Nonblocking Multileader: 2.671025e-03 +Nonblocking Node Aware: 3.383377e-03 +Nonblocking Locality Aware: 3.099752e-03 +Nonblocking Multileader Locality: 2.879229e-03 +Size 4 +PMPI: 3.774269e-04 +Pairwise: 7.927888e-03 +NonBlocking: 5.357637e-03 +Pairwise Hierarchical: 7.263378e-03 +Pairwise Multileader: 6.194416e-03 +Pairwise Node Aware: 7.192961e-03 +Pairwise Locality Aware: 6.339057e-03 +Pairwise Multileader Locality: 6.750235e-03 +Nonblocking Hierarchical: 6.778915e-03 +Nonblocking Multileader: 7.407809e-03 +Nonblocking Node Aware: 5.587622e-03 +Nonblocking Locality Aware: 7.471386e-03 +Nonblocking Multileader Locality: 7.282022e-03 +Size 8 +PMPI: 6.378534e-04 +Pairwise: 7.241025e-03 +NonBlocking: 5.626256e-03 +Pairwise Hierarchical: 6.979289e-03 +Pairwise Multileader: 5.825531e-03 +Pairwise Node Aware: 7.594595e-03 +Pairwise Locality Aware: 5.168410e-03 +Pairwise Multileader Locality: 5.392081e-03 +Nonblocking Hierarchical: 8.523823e-03 +Nonblocking Multileader: 6.473009e-03 +Nonblocking Node Aware: 6.672967e-03 +Nonblocking Locality Aware: 6.552373e-03 +Nonblocking Multileader Locality: 6.142854e-03 +Size 16 +PMPI: 1.236416e-03 +Pairwise: 6.366083e-03 +NonBlocking: 6.987824e-03 +Pairwise Hierarchical: 6.045883e-03 +Pairwise Multileader: 7.619239e-03 +Pairwise Node Aware: 6.129803e-03 +Pairwise Locality Aware: 8.123980e-03 +Pairwise Multileader Locality: 5.773397e-03 +Nonblocking Hierarchical: 7.820568e-03 +Nonblocking Multileader: 6.815020e-03 +Nonblocking Node Aware: 5.561677e-03 +Nonblocking Locality Aware: 6.750500e-03 +Nonblocking Multileader Locality: 6.312410e-03 +Size 32 +PMPI: 2.451160e-03 +Pairwise: 6.066868e-03 +NonBlocking: 6.805513e-03 +Pairwise Hierarchical: 6.542262e-03 +Pairwise Multileader: 6.071800e-03 +Pairwise Node Aware: 9.344145e-03 +Pairwise Locality Aware: 6.341815e-03 +Pairwise Multileader Locality: 6.066805e-03 +Nonblocking Hierarchical: 7.808069e-03 +Nonblocking Multileader: 6.146418e-03 +Nonblocking Node Aware: 7.208765e-03 +Nonblocking Locality Aware: 6.987455e-03 +Nonblocking Multileader Locality: 8.200845e-03 +Size 64 +PMPI: 3.864662e-03 +Pairwise: 7.500942e-03 +NonBlocking: 7.702336e-03 +Pairwise Hierarchical: 6.428817e-03 +Pairwise Multileader: 8.009166e-03 +Pairwise Node Aware: 7.686062e-03 +Pairwise Locality Aware: 7.531199e-03 +Pairwise Multileader Locality: 7.785861e-03 +Nonblocking Hierarchical: 6.620483e-03 +Nonblocking Multileader: 9.121063e-03 +Nonblocking Node Aware: 8.327883e-03 +Nonblocking Locality Aware: 5.876366e-03 +Nonblocking Multileader Locality: 7.350318e-03 +Size 128 +PMPI: 8.890054e-03 +Pairwise: 9.147705e-03 +NonBlocking: 1.041058e-02 +Pairwise Hierarchical: 1.085068e-02 +Pairwise Multileader: 9.357139e-03 +Pairwise Node Aware: 9.901974e-03 +Pairwise Locality Aware: 8.727490e-03 +Pairwise Multileader Locality: 1.029510e-02 +Nonblocking Hierarchical: 9.060816e-03 +Nonblocking Multileader: 1.002644e-02 +Nonblocking Node Aware: 1.046902e-02 +Nonblocking Locality Aware: 7.501843e-03 +Nonblocking Multileader Locality: 1.101679e-02 +Size 256 +PMPI: 1.475886e-02 +Pairwise: 1.369642e-02 +NonBlocking: 1.331213e-02 +Pairwise Hierarchical: 1.487171e-02 +Pairwise Multileader: 1.467479e-02 +Pairwise Node Aware: 1.255078e-02 +Pairwise Locality Aware: 1.612877e-02 +Pairwise Multileader Locality: 1.179470e-02 +Nonblocking Hierarchical: 1.258675e-02 +Nonblocking Multileader: 1.663303e-02 +Nonblocking Node Aware: 1.226793e-02 +Nonblocking Locality Aware: 1.444784e-02 +Nonblocking Multileader Locality: 1.446227e-02 +Size 512 +PMPI: 1.898273e-02 +Pairwise: 2.080118e-02 +NonBlocking: 2.116309e-02 +Pairwise Hierarchical: 2.352405e-02 +Pairwise Multileader: 2.208658e-02 +Pairwise Node Aware: 1.979361e-02 +Pairwise Locality Aware: 2.194301e-02 +Pairwise Multileader Locality: 2.189598e-02 +Nonblocking Hierarchical: 2.064864e-02 +Nonblocking Multileader: 2.302101e-02 +Nonblocking Node Aware: 2.024171e-02 +Nonblocking Locality Aware: 2.015676e-02 +Nonblocking Multileader Locality: 2.130077e-02 +Size 1024 +PMPI: 4.465023e-02 +Pairwise: 3.962538e-02 +NonBlocking: 3.857913e-02 +Pairwise Hierarchical: 3.920613e-02 +Pairwise Multileader: 4.078679e-02 +Pairwise Node Aware: 3.890839e-02 +Pairwise Locality Aware: 3.572150e-02 +Pairwise Multileader Locality: 4.380974e-02 +Nonblocking Hierarchical: 3.561561e-02 +Nonblocking Multileader: 4.062995e-02 +Nonblocking Node Aware: 4.055996e-02 +Nonblocking Locality Aware: 3.786851e-02 +Nonblocking Multileader Locality: 3.876701e-02 +Size 2048 +PMPI: 7.524712e-02 +Pairwise: 7.228925e-02 +NonBlocking: 6.943277e-02 +Pairwise Hierarchical: 7.052319e-02 +Pairwise Multileader: 6.999680e-02 +Pairwise Node Aware: 7.093442e-02 +Pairwise Locality Aware: 6.841923e-02 +Pairwise Multileader Locality: 7.622582e-02 +Nonblocking Hierarchical: 7.415313e-02 +Nonblocking Multileader: 7.088169e-02 +Nonblocking Node Aware: 7.226954e-02 +Nonblocking Locality Aware: 7.045615e-02 +Nonblocking Multileader Locality: 7.441819e-02 +Size 4096 +PMPI: 1.512932e-01 +Pairwise: 1.380810e-01 +NonBlocking: 1.416743e-01 +Pairwise Hierarchical: 1.506657e-01 +Pairwise Multileader: 1.427298e-01 +Pairwise Node Aware: 1.406017e-01 +Pairwise Locality Aware: 1.279716e-01 +Pairwise Multileader Locality: 1.392337e-01 +Nonblocking Hierarchical: 1.240772e-01 +Nonblocking Multileader: 1.374259e-01 +Nonblocking Node Aware: 1.328682e-01 +Nonblocking Locality Aware: 1.446104e-01 +Nonblocking Multileader Locality: 1.428914e-01 +Size 8192 +PMPI: 2.678221e-01 +Pairwise: 2.756026e-01 +NonBlocking: 2.456563e-01 +Test 3 Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.825105e-04 +Pairwise: 3.517763e-03 +NonBlocking: 2.547227e-03 +Pairwise Hierarchical: 3.259819e-03 +Pairwise Multileader: 2.474728e-03 +Pairwise Node Aware: 2.855000e-03 +Pairwise Locality Aware: 3.390863e-03 +Pairwise Multileader Locality: 2.915645e-03 +Nonblocking Hierarchical: 2.746114e-03 +Nonblocking Multileader: 3.259458e-03 +Nonblocking Node Aware: 2.619893e-03 +Nonblocking Locality Aware: 2.519823e-03 +Nonblocking Multileader Locality: 3.673156e-03 +Size 2 +PMPI: 2.350378e-04 +Pairwise: 2.321987e-03 +NonBlocking: 3.365513e-03 +Pairwise Hierarchical: 3.066358e-03 +Pairwise Multileader: 2.618541e-03 +Pairwise Node Aware: 3.139503e-03 +Pairwise Locality Aware: 3.101032e-03 +Pairwise Multileader Locality: 2.636624e-03 +Nonblocking Hierarchical: 3.169711e-03 +Nonblocking Multileader: 3.240286e-03 +Nonblocking Node Aware: 3.135845e-03 +Nonblocking Locality Aware: 2.401091e-03 +Nonblocking Multileader Locality: 3.059444e-03 +Size 4 +PMPI: 4.601321e-04 +Pairwise: 4.953928e-03 +NonBlocking: 6.751852e-03 +Pairwise Hierarchical: 6.897826e-03 +Pairwise Multileader: 5.443488e-03 +Pairwise Node Aware: 6.429330e-03 +Pairwise Locality Aware: 6.804957e-03 +Pairwise Multileader Locality: 7.171392e-03 +Nonblocking Hierarchical: 5.682153e-03 +Nonblocking Multileader: 5.586648e-03 +Nonblocking Node Aware: 7.240470e-03 +Nonblocking Locality Aware: 5.527542e-03 +Nonblocking Multileader Locality: 7.668553e-03 +Size 8 +PMPI: 6.314219e-04 +Pairwise: 6.354376e-03 +NonBlocking: 6.168721e-03 +Pairwise Hierarchical: 6.886176e-03 +Pairwise Multileader: 6.254541e-03 +Pairwise Node Aware: 8.060245e-03 +Pairwise Locality Aware: 6.783540e-03 +Pairwise Multileader Locality: 5.705156e-03 +Nonblocking Hierarchical: 8.681483e-03 +Nonblocking Multileader: 4.661451e-03 +Nonblocking Node Aware: 6.063131e-03 +Nonblocking Locality Aware: 8.274765e-03 +Nonblocking Multileader Locality: 5.094924e-03 +Size 16 +PMPI: 1.368796e-03 +Pairwise: 6.614576e-03 +NonBlocking: 6.118007e-03 +Pairwise Hierarchical: 5.702936e-03 +Pairwise Multileader: 6.396730e-03 +Pairwise Node Aware: 7.738806e-03 +Pairwise Locality Aware: 5.631059e-03 +Pairwise Multileader Locality: 5.767715e-03 +Nonblocking Hierarchical: 7.888122e-03 +Nonblocking Multileader: 5.996184e-03 +Nonblocking Node Aware: 6.515171e-03 +Nonblocking Locality Aware: 7.294584e-03 +Nonblocking Multileader Locality: 6.920136e-03 +Size 32 +PMPI: 2.480267e-03 +Pairwise: 6.572079e-03 +NonBlocking: 5.535229e-03 +Pairwise Hierarchical: 8.226685e-03 +Pairwise Multileader: 6.545345e-03 +Pairwise Node Aware: 6.536526e-03 +Pairwise Locality Aware: 7.247858e-03 +Pairwise Multileader Locality: 6.605795e-03 +Nonblocking Hierarchical: 7.008804e-03 +Nonblocking Multileader: 5.951009e-03 +Nonblocking Node Aware: 7.730868e-03 +Nonblocking Locality Aware: 5.301074e-03 +Nonblocking Multileader Locality: 7.376380e-03 +Size 64 +PMPI: 4.122380e-03 +Pairwise: 7.219060e-03 +NonBlocking: 7.023650e-03 +Pairwise Hierarchical: 8.287746e-03 +Pairwise Multileader: 7.876149e-03 +Pairwise Node Aware: 6.731655e-03 +Pairwise Locality Aware: 8.683333e-03 +Pairwise Multileader Locality: 6.622277e-03 +Nonblocking Hierarchical: 9.424667e-03 +Nonblocking Multileader: 7.636897e-03 +Nonblocking Node Aware: 7.503747e-03 +Nonblocking Locality Aware: 8.826024e-03 +Nonblocking Multileader Locality: 7.359851e-03 +Size 128 +PMPI: 8.603857e-03 +Pairwise: 1.197835e-02 +NonBlocking: 7.517713e-03 +Pairwise Hierarchical: 9.870172e-03 +Pairwise Multileader: 9.713111e-03 +Pairwise Node Aware: 8.497057e-03 +Pairwise Locality Aware: 8.761609e-03 +Pairwise Multileader Locality: 1.001243e-02 +Nonblocking Hierarchical: 1.019544e-02 +Nonblocking Multileader: 7.314371e-03 +Nonblocking Node Aware: 6.765727e-03 +Nonblocking Locality Aware: 8.150565e-03 +Nonblocking Multileader Locality: 8.431555e-03 +Size 256 +PMPI: 1.624787e-02 +Pairwise: 9.934758e-03 +NonBlocking: 1.026055e-02 +Pairwise Hierarchical: 1.207986e-02 +Pairwise Multileader: 1.227362e-02 +Pairwise Node Aware: 9.193020e-03 +Pairwise Locality Aware: 1.020543e-02 +Pairwise Multileader Locality: 1.311131e-02 +Nonblocking Hierarchical: 1.368764e-02 +Nonblocking Multileader: 1.416646e-02 +Nonblocking Node Aware: 1.442926e-02 +Nonblocking Locality Aware: 1.243219e-02 +Nonblocking Multileader Locality: 1.682191e-02 +Size 512 +PMPI: 3.391418e-02 +Pairwise: 2.074199e-02 +NonBlocking: 2.353977e-02 +Pairwise Hierarchical: 2.688426e-02 +Pairwise Multileader: 2.066354e-02 +Pairwise Node Aware: 2.223381e-02 +Pairwise Locality Aware: 2.262113e-02 +Pairwise Multileader Locality: 1.859274e-02 +Nonblocking Hierarchical: 2.412895e-02 +Nonblocking Multileader: 2.109360e-02 +Nonblocking Node Aware: 2.271655e-02 +Nonblocking Locality Aware: 2.456613e-02 +Nonblocking Multileader Locality: 2.121852e-02 +Size 1024 +PMPI: 3.858009e-02 +Pairwise: 3.656811e-02 +NonBlocking: 3.927047e-02 +Pairwise Hierarchical: 3.783872e-02 +Pairwise Multileader: 4.029525e-02 +Pairwise Node Aware: 3.616755e-02 +Pairwise Locality Aware: 4.274509e-02 +Pairwise Multileader Locality: 3.480872e-02 +Nonblocking Hierarchical: 4.095036e-02 +Nonblocking Multileader: 3.541178e-02 +Nonblocking Node Aware: 4.001329e-02 +Nonblocking Locality Aware: 4.086282e-02 +Nonblocking Multileader Locality: 4.661294e-02 +Size 2048 +PMPI: 8.487110e-02 +Pairwise: 7.404645e-02 +NonBlocking: 7.492254e-02 +Pairwise Hierarchical: 7.945587e-02 +Pairwise Multileader: 6.544949e-02 +Pairwise Node Aware: 7.784926e-02 +Pairwise Locality Aware: 6.752424e-02 +Pairwise Multileader Locality: 7.373779e-02 +Nonblocking Hierarchical: 7.863946e-02 +Nonblocking Multileader: 6.811988e-02 +Nonblocking Node Aware: 7.379783e-02 +Nonblocking Locality Aware: 6.814850e-02 +Nonblocking Multileader Locality: 7.749559e-02 +Size 4096 +PMPI: 1.383733e-01 +Pairwise: 1.569625e-01 +NonBlocking: 1.396764e-01 +Pairwise Hierarchical: 1.371730e-01 +Pairwise Multileader: 1.437907e-01 +Pairwise Node Aware: 1.387820e-01 +Pairwise Locality Aware: 1.256677e-01 +Pairwise Multileader Locality: 1.347382e-01 +Nonblocking Hierarchical: 1.351157e-01 +Nonblocking Multileader: 1.246995e-01 +Nonblocking Node Aware: 1.480999e-01 +Nonblocking Locality Aware: 1.443929e-01 +Nonblocking Multileader Locality: 1.303272e-01 +Size 8192 +PMPI: 2.339743e-01 +Pairwise: 2.834612e-01 +NonBlocking: 2.790918e-01 diff --git a/runscripts/test_alltoall b/runscripts/test_alltoall new file mode 100644 index 000000000..e7396dfe5 --- /dev/null +++ b/runscripts/test_alltoall @@ -0,0 +1,27 @@ +#!/bin/bash + +#SBATCH --nodes=64 +#SBATCH --ntasks-per-node=100 +#SBATCH --output=alltoall_N64.%j.out +#SBATCH --error=alltoall_N64.%j.err +#SBATCH --account=UNM +#SBATCH --time=00:30:00 +##SBATCH --partition=pdebug +#SBATCH --cpus-per-task=1 +#SBATCH --exclusive + +cd $HOME/AdvanceAlltoall/build/benchmarks +nnodes=64 +ppn=100 +ntasks=$(( ${nnodes} * ${ppn} )) + +echo "Test 1 Running Alltoall Benchmark on ${nnodes} nodes and ${ntasks} tasks" +srun -n ${ntasks} -N ${nnodes} ./alltoall_topology + +echo "Test 2 Running Alltoall Benchmark on ${nnodes} nodes and ${ntasks} tasks" +srun -n ${ntasks} -N ${nnodes} ./alltoall_topology + +echo "Test 3 Running Alltoall Benchmark on ${nnodes} nodes and ${ntasks} tasks" +srun -n ${ntasks} -N ${nnodes} ./alltoall_topology + + diff --git a/runscripts/wrapping_pmpi/alltoall_N16.2357549.err b/runscripts/wrapping_pmpi/alltoall_N16.2357549.err new file mode 100644 index 000000000..bf32cc586 --- /dev/null +++ b/runscripts/wrapping_pmpi/alltoall_N16.2357549.err @@ -0,0 +1,132 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1555007e1010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2357549.0 ON dane96 CANCELLED AT 2025-04-28T12:09:56 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane644: task 605: Killed +srun: error: dane511: tasks 505,509: Killed +srun: error: dane664: tasks 800-899: Killed +srun: error: dane770: tasks 1000-1099: Killed +srun: error: dane1394: tasks 1500-1599: Killed +srun: error: dane1050: tasks 1300-1399: Killed +srun: error: dane511: tasks 500-504,506-508,510-599: Killed +srun: error: dane334: tasks 400-499: Killed +srun: error: dane1096: tasks 1400-1499: Killed +srun: error: dane1033: tasks 1200-1299: Killed +srun: error: dane828: tasks 1100-1199: Killed +srun: error: dane239: tasks 300-399: Killed +srun: error: dane757: tasks 900-999: Killed +srun: error: dane649: tasks 700-799: Killed +srun: error: dane96: tasks 0-99: Killed +srun: error: dane122: tasks 100-199: Killed +srun: error: dane644: tasks 600-604,606-699: Killed +srun: error: dane132: tasks 200-299: Killed diff --git a/runscripts/wrapping_pmpi/alltoall_N16.2357549.out b/runscripts/wrapping_pmpi/alltoall_N16.2357549.out new file mode 100644 index 000000000..6f2feb437 --- /dev/null +++ b/runscripts/wrapping_pmpi/alltoall_N16.2357549.out @@ -0,0 +1,113 @@ +Running Alltoall Benchmark on 16 nodes and 1600 tasks +Size 1 +PMPI: 3.734732e-04 +Pairwise: 6.118291e-03 +NonBlocking: 6.082917e-03 +Hierarchical: 6.635214e-03 +Multileader: 5.845607e-03 +Node Aware: 6.786968e-03 +Locality Aware: 7.324603e-03 +Multileader Locality: 7.763149e-03 +Size 2 +PMPI: 5.996531e-04 +Pairwise: 7.529903e-03 +NonBlocking: 7.533126e-03 +Hierarchical: 8.045581e-03 +Multileader: 6.833652e-03 +Node Aware: 8.757691e-03 +Locality Aware: 8.450123e-03 +Multileader Locality: 7.144501e-03 +Size 4 +PMPI: 1.023707e-03 +Pairwise: 1.191586e-02 +NonBlocking: 2.119890e-02 +Hierarchical: 1.730058e-02 +Multileader: 1.315150e-02 +Node Aware: 1.623848e-02 +Locality Aware: 1.948328e-02 +Multileader Locality: 1.460327e-02 +Size 8 +PMPI: 1.411905e-03 +Pairwise: 2.392270e-02 +NonBlocking: 1.678213e-02 +Hierarchical: 1.779783e-02 +Multileader: 2.017602e-02 +Node Aware: 1.604240e-02 +Locality Aware: 1.295149e-02 +Multileader Locality: 1.589995e-02 +Size 16 +PMPI: 2.886692e-03 +Pairwise: 1.059647e-02 +NonBlocking: 1.948100e-02 +Hierarchical: 1.222075e-02 +Multileader: 1.251756e-02 +Node Aware: 2.394458e-02 +Locality Aware: 1.796435e-02 +Multileader Locality: 2.132234e-02 +Size 32 +PMPI: 4.901202e-03 +Pairwise: 1.912159e-02 +NonBlocking: 1.783776e-02 +Hierarchical: 1.548058e-02 +Multileader: 1.409386e-02 +Node Aware: 1.799427e-02 +Locality Aware: 1.097884e-02 +Multileader Locality: 1.447512e-02 +Size 64 +PMPI: 9.248981e-03 +Pairwise: 1.576393e-02 +NonBlocking: 1.421738e-02 +Hierarchical: 2.300418e-02 +Multileader: 1.626093e-02 +Node Aware: 1.753612e-02 +Locality Aware: 1.751501e-02 +Multileader Locality: 1.356827e-02 +Size 128 +PMPI: 1.729898e-02 +Pairwise: 2.013089e-02 +NonBlocking: 1.545002e-02 +Hierarchical: 2.212922e-02 +Multileader: 1.953587e-02 +Node Aware: 1.499573e-02 +Locality Aware: 2.546476e-02 +Multileader Locality: 1.716283e-02 +Size 256 +PMPI: 3.391043e-02 +Pairwise: 3.010999e-02 +NonBlocking: 2.719172e-02 +Hierarchical: 3.592644e-02 +Multileader: 3.091590e-02 +Node Aware: 2.681609e-02 +Locality Aware: 3.271652e-02 +Multileader Locality: 2.652162e-02 +Size 512 +PMPI: 1.409862e-01 +Pairwise: 4.624185e-02 +NonBlocking: 4.344486e-02 +Hierarchical: 4.787130e-02 +Multileader: 4.893120e-02 +Node Aware: 4.439047e-02 +Locality Aware: 4.674737e-02 +Multileader Locality: 4.425728e-02 +Size 1024 +PMPI: 8.272304e-02 +Pairwise: 8.806283e-02 +NonBlocking: 8.267868e-02 +Hierarchical: 7.932440e-02 +Multileader: 7.619979e-02 +Node Aware: 8.924131e-02 +Locality Aware: 7.627727e-02 +Multileader Locality: 8.224404e-02 +Size 2048 +PMPI: 1.657872e-01 +Pairwise: 1.537526e-01 +NonBlocking: 1.527351e-01 +Hierarchical: 1.573128e-01 +Multileader: 1.483606e-01 +Node Aware: 1.675702e-01 +Locality Aware: 1.501141e-01 +Multileader Locality: 1.478692e-01 +Size 4096 +PMPI: 3.250341e-01 +Pairwise: 3.051531e-01 +NonBlocking: 3.142614e-01 diff --git a/runscripts/wrapping_pmpi/alltoall_N2.2357787.err b/runscripts/wrapping_pmpi/alltoall_N2.2357787.err new file mode 100644 index 000000000..e69de29bb diff --git a/runscripts/wrapping_pmpi/alltoall_N2.2357787.out b/runscripts/wrapping_pmpi/alltoall_N2.2357787.out new file mode 100644 index 000000000..9d9568942 --- /dev/null +++ b/runscripts/wrapping_pmpi/alltoall_N2.2357787.out @@ -0,0 +1,136 @@ +Running Alltoall Benchmark on 2 nodes and 200 tasks +Size 1 +PMPI: 4.355267e-05 +Pairwise: 4.197910e-04 +NonBlocking: 3.843548e-04 +Hierarchical: 3.722437e-04 +Multileader: 3.775375e-04 +Node Aware: 3.826830e-04 +Locality Aware: 3.723335e-04 +Multileader Locality: 3.736743e-04 +Size 2 +PMPI: 4.755026e-05 +Pairwise: 3.800425e-04 +NonBlocking: 3.782890e-04 +Hierarchical: 3.887225e-04 +Multileader: 3.832761e-04 +Node Aware: 3.752927e-04 +Locality Aware: 3.787618e-04 +Multileader Locality: 3.898802e-04 +Size 4 +PMPI: 5.641340e-05 +Pairwise: 7.797026e-04 +NonBlocking: 8.123352e-04 +Hierarchical: 7.932194e-04 +Multileader: 8.175682e-04 +Node Aware: 8.044608e-04 +Locality Aware: 7.911799e-04 +Multileader Locality: 7.797765e-04 +Size 8 +PMPI: 7.942569e-05 +Pairwise: 8.212948e-04 +NonBlocking: 7.913009e-04 +Hierarchical: 7.881071e-04 +Multileader: 8.170724e-04 +Node Aware: 8.222846e-04 +Locality Aware: 8.068349e-04 +Multileader Locality: 8.225799e-04 +Size 16 +PMPI: 1.334695e-04 +Pairwise: 7.838361e-04 +NonBlocking: 8.238470e-04 +Hierarchical: 7.840044e-04 +Multileader: 7.835606e-04 +Node Aware: 8.261581e-04 +Locality Aware: 8.291649e-04 +Multileader Locality: 8.044197e-04 +Size 32 +PMPI: 2.621142e-04 +Pairwise: 8.159941e-04 +NonBlocking: 8.076026e-04 +Hierarchical: 8.467135e-04 +Multileader: 8.034245e-04 +Node Aware: 8.066115e-04 +Locality Aware: 8.346858e-04 +Multileader Locality: 8.094232e-04 +Size 64 +PMPI: 4.357225e-04 +Pairwise: 8.829906e-04 +NonBlocking: 8.369698e-04 +Hierarchical: 8.390744e-04 +Multileader: 8.808576e-04 +Node Aware: 8.362627e-04 +Locality Aware: 8.541259e-04 +Multileader Locality: 8.919005e-04 +Size 128 +PMPI: 8.411458e-04 +Pairwise: 9.813681e-04 +NonBlocking: 1.014320e-03 +Hierarchical: 9.845995e-04 +Multileader: 1.011071e-03 +Node Aware: 1.012293e-03 +Locality Aware: 9.724600e-04 +Multileader Locality: 9.655051e-04 +Size 256 +PMPI: 1.646762e-03 +Pairwise: 1.356325e-03 +NonBlocking: 1.369368e-03 +Hierarchical: 1.488945e-03 +Multileader: 1.335838e-03 +Node Aware: 1.355067e-03 +Locality Aware: 1.484623e-03 +Multileader Locality: 1.343428e-03 +Size 512 +PMPI: 2.226258e-03 +Pairwise: 2.471087e-03 +NonBlocking: 2.245607e-03 +Hierarchical: 2.395905e-03 +Multileader: 2.543720e-03 +Node Aware: 2.262197e-03 +Locality Aware: 2.330460e-03 +Multileader Locality: 2.416036e-03 +Size 1024 +PMPI: 4.363031e-03 +Pairwise: 4.268765e-03 +NonBlocking: 4.380784e-03 +Hierarchical: 4.319295e-03 +Multileader: 4.186983e-03 +Node Aware: 4.211646e-03 +Locality Aware: 4.447992e-03 +Multileader Locality: 4.207322e-03 +Size 2048 +PMPI: 8.312140e-03 +Pairwise: 9.189936e-03 +NonBlocking: 8.234386e-03 +Hierarchical: 8.463344e-03 +Multileader: 9.030145e-03 +Node Aware: 8.233315e-03 +Locality Aware: 8.564847e-03 +Multileader Locality: 8.871296e-03 +Size 4096 +PMPI: 1.802506e-02 +Pairwise: 1.630284e-02 +NonBlocking: 1.713112e-02 +Hierarchical: 1.631869e-02 +Multileader: 1.720022e-02 +Node Aware: 1.658163e-02 +Locality Aware: 1.607202e-02 +Multileader Locality: 1.778685e-02 +Size 8192 +PMPI: 3.057343e-02 +Pairwise: 3.114333e-02 +NonBlocking: 3.262174e-02 +Hierarchical: 3.250889e-02 +Multileader: 3.081412e-02 +Node Aware: 3.116112e-02 +Locality Aware: 3.273941e-02 +Multileader Locality: 3.256629e-02 +Size 16384 +PMPI: 5.900251e-02 +Pairwise: 5.769987e-02 +NonBlocking: 5.806596e-02 +Hierarchical: 5.601344e-02 +Multileader: 6.081367e-02 +Node Aware: 5.763630e-02 +Locality Aware: 5.945440e-02 +Multileader Locality: 5.844511e-02 diff --git a/runscripts/wrapping_pmpi/alltoall_N32.2357553.err b/runscripts/wrapping_pmpi/alltoall_N32.2357553.err new file mode 100644 index 000000000..c6233f6a6 --- /dev/null +++ b/runscripts/wrapping_pmpi/alltoall_N32.2357553.err @@ -0,0 +1,258 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_3100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa907010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fa907010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2000]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb570010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb14d010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_800]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fad2a010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fbdb6010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_2300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fbdb6010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +[cli_1900]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x1554fb993010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0x84000004) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2357553.0 ON dane96 CANCELLED AT 2025-04-28T12:12:45 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1262: tasks 2500-2599: Killed +srun: error: dane1167: tasks 2400-2499: Killed +srun: error: dane1466: tasks 2800-2899: Killed +srun: error: dane145: tasks 300-399: Killed +srun: error: dane964: tasks 2100-2199: Killed +srun: error: dane400: tasks 1300-1399: Killed +srun: error: dane1034: tasks 2200-2299: Killed +srun: error: dane304: tasks 700-799: Killed +srun: error: dane401: tasks 1400-1499: Killed +srun: error: dane619: tasks 1800-1899: Killed +srun: error: dane1115: tasks 2300-2399: Killed +srun: error: dane408: tasks 1500-1599: Killed +srun: error: dane1540: tasks 3100-3199: Killed +srun: error: dane172: tasks 500-599: Killed +srun: error: dane1526: tasks 2900-2999: Killed +srun: error: dane358: tasks 1100-1199: Killed +srun: error: dane1534: tasks 3000-3099: Killed +srun: error: dane699: tasks 2000-2099: Killed +srun: error: dane1418: tasks 2700-2799: Killed +srun: error: dane1277: tasks 2600-2699: Killed +srun: error: dane150: tasks 400-499: Killed +srun: error: dane377: tasks 1200-1299: Killed +srun: error: dane235: tasks 600-699: Killed +srun: error: dane645: tasks 1900-1999: Killed +srun: error: dane464: tasks 1700-1799: Killed +srun: error: dane130: tasks 200-299: Killed +srun: error: dane96: tasks 0-99: Killed +srun: error: dane312: tasks 1000-1099: Killed +srun: error: dane306: tasks 800-899: Killed +srun: error: dane425: tasks 1600-1699: Killed +srun: error: dane310: tasks 900-999: Killed +srun: error: dane97: tasks 100-199: Killed diff --git a/runscripts/wrapping_pmpi/alltoall_N32.2357553.out b/runscripts/wrapping_pmpi/alltoall_N32.2357553.out new file mode 100644 index 000000000..8244b68b3 --- /dev/null +++ b/runscripts/wrapping_pmpi/alltoall_N32.2357553.out @@ -0,0 +1,104 @@ +Running Alltoall Benchmark on 32 nodes and 3200 tasks +Size 1 +PMPI: 1.157201e-03 +Pairwise: 2.038441e-02 +NonBlocking: 1.796203e-02 +Hierarchical: 1.683883e-02 +Multileader: 1.894754e-02 +Node Aware: 2.100123e-02 +Locality Aware: 2.120233e-02 +Multileader Locality: 2.248687e-02 +Size 2 +PMPI: 1.303007e-03 +Pairwise: 2.244392e-02 +NonBlocking: 2.632298e-02 +Hierarchical: 3.594556e-02 +Multileader: 1.822220e-02 +Node Aware: 2.111917e-02 +Locality Aware: 1.860297e-02 +Multileader Locality: 1.721135e-02 +Size 4 +PMPI: 2.787666e-03 +Pairwise: 2.705035e-02 +NonBlocking: 4.215761e-02 +Hierarchical: 3.324128e-02 +Multileader: 4.017289e-02 +Node Aware: 3.159043e-02 +Locality Aware: 3.727193e-02 +Multileader Locality: 4.038978e-02 +Size 8 +PMPI: 3.678150e-03 +Pairwise: 3.299228e-02 +NonBlocking: 4.206826e-02 +Hierarchical: 3.748593e-02 +Multileader: 5.478355e-02 +Node Aware: 5.267694e-02 +Locality Aware: 3.954081e-02 +Multileader Locality: 4.265765e-02 +Size 16 +PMPI: 5.852959e-03 +Pairwise: 5.161680e-02 +NonBlocking: 4.066218e-02 +Hierarchical: 3.884295e-02 +Multileader: 3.834179e-02 +Node Aware: 2.766884e-02 +Locality Aware: 4.749155e-02 +Multileader Locality: 3.847687e-02 +Size 32 +PMPI: 1.178111e-02 +Pairwise: 4.635694e-02 +NonBlocking: 4.536293e-02 +Hierarchical: 3.257549e-02 +Multileader: 5.652991e-02 +Node Aware: 5.454569e-02 +Locality Aware: 5.401024e-02 +Multileader Locality: 4.547362e-02 +Size 64 +PMPI: 1.999311e-02 +Pairwise: 2.890957e-02 +NonBlocking: 5.805634e-02 +Hierarchical: 4.602905e-02 +Multileader: 5.803488e-02 +Node Aware: 5.805628e-02 +Locality Aware: 4.666267e-02 +Multileader Locality: 4.207176e-02 +Size 128 +PMPI: 4.255309e-02 +Pairwise: 4.518741e-02 +NonBlocking: 7.249236e-02 +Hierarchical: 5.434673e-02 +Multileader: 5.521096e-02 +Node Aware: 6.822446e-02 +Locality Aware: 5.173537e-02 +Multileader Locality: 6.439214e-02 +Size 256 +PMPI: 8.796651e-02 +Pairwise: 7.320004e-02 +NonBlocking: 6.985680e-02 +Hierarchical: 8.035205e-02 +Multileader: 7.563470e-02 +Node Aware: 7.806557e-02 +Locality Aware: 7.600677e-02 +Multileader Locality: 8.276267e-02 +Size 512 +PMPI: 5.763552e-01 +Pairwise: 1.169719e-01 +NonBlocking: 1.189793e-01 +Hierarchical: 1.127202e-01 +Multileader: 1.117703e-01 +Node Aware: 1.176601e-01 +Locality Aware: 1.138135e-01 +Multileader Locality: 1.071235e-01 +Size 1024 +PMPI: 2.065748e-01 +Pairwise: 2.149295e-01 +NonBlocking: 2.041914e-01 +Hierarchical: 2.159154e-01 +Multileader: 2.237137e-01 +Node Aware: 2.126513e-01 +Locality Aware: 2.148806e-01 +Multileader Locality: 2.130593e-01 +Size 2048 +PMPI: 4.196702e-01 +Pairwise: 4.021378e-01 +NonBlocking: 4.266402e-01 diff --git a/runscripts/wrapping_pmpi/alltoall_N4.2357789.err b/runscripts/wrapping_pmpi/alltoall_N4.2357789.err new file mode 100644 index 000000000..4d4c5f144 --- /dev/null +++ b/runscripts/wrapping_pmpi/alltoall_N4.2357789.err @@ -0,0 +1,34 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155505d09010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155505d09010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155509dec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2357789.0 ON dane122 CANCELLED AT 2025-04-28T12:12:34 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane664: tasks 200-299: Killed +srun: error: dane757: tasks 300-399: Killed +srun: error: dane517: tasks 100-199: Killed +srun: error: dane122: tasks 0-99: Killed diff --git a/runscripts/wrapping_pmpi/alltoall_N4.2357789.out b/runscripts/wrapping_pmpi/alltoall_N4.2357789.out new file mode 100644 index 000000000..30216f201 --- /dev/null +++ b/runscripts/wrapping_pmpi/alltoall_N4.2357789.out @@ -0,0 +1,131 @@ +Running Alltoall Benchmark on 4 nodes and 400 tasks +Size 1 +PMPI: 7.637312e-05 +Pairwise: 1.030168e-03 +NonBlocking: 1.119813e-03 +Hierarchical: 1.065954e-03 +Multileader: 1.167425e-03 +Node Aware: 1.223545e-03 +Locality Aware: 9.990675e-04 +Multileader Locality: 9.171154e-04 +Size 2 +PMPI: 9.303350e-05 +Pairwise: 9.398168e-04 +NonBlocking: 9.347979e-04 +Hierarchical: 9.893752e-04 +Multileader: 9.217796e-04 +Node Aware: 9.348522e-04 +Locality Aware: 9.329333e-04 +Multileader Locality: 9.098401e-04 +Size 4 +PMPI: 1.292009e-04 +Pairwise: 1.863215e-03 +NonBlocking: 1.888913e-03 +Hierarchical: 2.024815e-03 +Multileader: 1.807185e-03 +Node Aware: 2.364389e-03 +Locality Aware: 1.745879e-03 +Multileader Locality: 2.000924e-03 +Size 8 +PMPI: 2.189727e-04 +Pairwise: 1.809592e-03 +NonBlocking: 2.193350e-03 +Hierarchical: 1.808194e-03 +Multileader: 2.011686e-03 +Node Aware: 1.868832e-03 +Locality Aware: 1.800504e-03 +Multileader Locality: 1.818466e-03 +Size 16 +PMPI: 3.958992e-04 +Pairwise: 1.868932e-03 +NonBlocking: 1.769379e-03 +Hierarchical: 2.386258e-03 +Multileader: 1.930680e-03 +Node Aware: 2.622674e-03 +Locality Aware: 1.871438e-03 +Multileader Locality: 1.953758e-03 +Size 32 +PMPI: 7.238274e-04 +Pairwise: 1.869246e-03 +NonBlocking: 2.072701e-03 +Hierarchical: 2.064780e-03 +Multileader: 2.244054e-03 +Node Aware: 2.454485e-03 +Locality Aware: 2.102019e-03 +Multileader Locality: 2.161827e-03 +Size 64 +PMPI: 1.508343e-03 +Pairwise: 1.862283e-03 +NonBlocking: 2.370456e-03 +Hierarchical: 1.898151e-03 +Multileader: 2.260275e-03 +Node Aware: 2.014454e-03 +Locality Aware: 2.020186e-03 +Multileader Locality: 2.331808e-03 +Size 128 +PMPI: 2.408398e-03 +Pairwise: 2.810119e-03 +NonBlocking: 2.629042e-03 +Hierarchical: 2.573138e-03 +Multileader: 2.687838e-03 +Node Aware: 2.722993e-03 +Locality Aware: 3.548513e-03 +Multileader Locality: 2.904855e-03 +Size 256 +PMPI: 4.621797e-03 +Pairwise: 3.885299e-03 +NonBlocking: 5.147405e-03 +Hierarchical: 4.031954e-03 +Multileader: 4.897934e-03 +Node Aware: 3.868801e-03 +Locality Aware: 4.306192e-03 +Multileader Locality: 4.326771e-03 +Size 512 +PMPI: 7.183584e-03 +Pairwise: 7.182783e-03 +NonBlocking: 7.525357e-03 +Hierarchical: 7.314503e-03 +Multileader: 7.923790e-03 +Node Aware: 6.755616e-03 +Locality Aware: 7.474373e-03 +Multileader Locality: 6.973765e-03 +Size 1024 +PMPI: 1.478606e-02 +Pairwise: 1.318950e-02 +NonBlocking: 1.484534e-02 +Hierarchical: 1.383023e-02 +Multileader: 1.403273e-02 +Node Aware: 1.411955e-02 +Locality Aware: 1.310839e-02 +Multileader Locality: 1.403163e-02 +Size 2048 +PMPI: 2.480222e-02 +Pairwise: 2.654534e-02 +NonBlocking: 2.609041e-02 +Hierarchical: 2.723755e-02 +Multileader: 2.774500e-02 +Node Aware: 2.607511e-02 +Locality Aware: 2.763127e-02 +Multileader Locality: 2.549072e-02 +Size 4096 +PMPI: 6.031676e-02 +Pairwise: 5.245245e-02 +NonBlocking: 5.514836e-02 +Hierarchical: 5.514430e-02 +Multileader: 5.165940e-02 +Node Aware: 5.568199e-02 +Locality Aware: 5.134157e-02 +Multileader Locality: 5.615225e-02 +Size 8192 +PMPI: 9.613669e-02 +Pairwise: 1.090212e-01 +NonBlocking: 9.801875e-02 +Hierarchical: 1.022173e-01 +Multileader: 9.298925e-02 +Node Aware: 1.035150e-01 +Locality Aware: 1.010575e-01 +Multileader Locality: 1.025529e-01 +Size 16384 +PMPI: 1.867457e-01 +Pairwise: 1.656915e-01 +NonBlocking: 1.488461e-01 diff --git a/runscripts/wrapping_pmpi/alltoall_N8.2357547.err b/runscripts/wrapping_pmpi/alltoall_N8.2357547.err new file mode 100644 index 000000000..5f7f91cef --- /dev/null +++ b/runscripts/wrapping_pmpi/alltoall_N8.2357547.err @@ -0,0 +1,67 @@ +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_0]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_400]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_100]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_200]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_600]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_700]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_500]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +[cli_300]: aborting job: +Fatal error in PMPI_Gather: Invalid buffer pointer, error stack: +PMPI_Gather(929): MPI_Gather(sbuf=0x155506bec010, scount=6553600, MPI_FLOAT, rbuf=(nil), rcount=6553600, MPI_FLOAT, root=0, comm=0xc4000000) failed +PMPI_Gather(851): Null buffer pointer +slurmstepd: error: *** STEP 2357547.0 ON dane400 CANCELLED AT 2025-04-28T12:08:39 *** +slurmstepd: error: *** STEP 2357547.0 ON dane400 CANCELLED AT 2025-04-28T12:08:39 *** +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +srun: error: dane1466: tasks 500-599: Killed +srun: error: dane400: tasks 0-99: Killed +srun: error: dane1540: tasks 700-799: Killed +srun: error: dane1526: tasks 600-699: Killed +srun: error: dane408: tasks 100-199: Killed +srun: error: dane699: tasks 300-399: Killed +srun: error: dane1277: tasks 400-499: Killed +srun: error: dane425: tasks 200-299: Killed diff --git a/runscripts/wrapping_pmpi/alltoall_N8.2357547.out b/runscripts/wrapping_pmpi/alltoall_N8.2357547.out new file mode 100644 index 000000000..0cb8e6344 --- /dev/null +++ b/runscripts/wrapping_pmpi/alltoall_N8.2357547.out @@ -0,0 +1,122 @@ +Running Alltoall Benchmark on 8 nodes and 800 tasks +Size 1 +PMPI: 1.501666e-04 +Pairwise: 2.839238e-03 +NonBlocking: 3.838484e-03 +Hierarchical: 3.595755e-03 +Multileader: 2.381209e-03 +Node Aware: 2.123354e-03 +Locality Aware: 2.150113e-03 +Multileader Locality: 2.148841e-03 +Size 2 +PMPI: 2.131163e-04 +Pairwise: 2.373369e-03 +NonBlocking: 1.950898e-03 +Hierarchical: 2.411751e-03 +Multileader: 2.286992e-03 +Node Aware: 2.290889e-03 +Locality Aware: 2.082549e-03 +Multileader Locality: 2.378933e-03 +Size 4 +PMPI: 5.243588e-04 +Pairwise: 4.202990e-03 +NonBlocking: 4.632266e-03 +Hierarchical: 4.546598e-03 +Multileader: 5.422034e-03 +Node Aware: 4.789573e-03 +Locality Aware: 5.141107e-03 +Multileader Locality: 4.076108e-03 +Size 8 +PMPI: 5.846544e-04 +Pairwise: 4.633468e-03 +NonBlocking: 4.953250e-03 +Hierarchical: 4.250875e-03 +Multileader: 4.754030e-03 +Node Aware: 4.359182e-03 +Locality Aware: 4.494841e-03 +Multileader Locality: 4.284282e-03 +Size 16 +PMPI: 1.026319e-03 +Pairwise: 4.956252e-03 +NonBlocking: 4.130527e-03 +Hierarchical: 5.047275e-03 +Multileader: 4.567426e-03 +Node Aware: 4.568547e-03 +Locality Aware: 4.621714e-03 +Multileader Locality: 4.700105e-03 +Size 32 +PMPI: 2.075948e-03 +Pairwise: 5.243561e-03 +NonBlocking: 4.991259e-03 +Hierarchical: 4.698519e-03 +Multileader: 4.916353e-03 +Node Aware: 4.287631e-03 +Locality Aware: 5.588674e-03 +Multileader Locality: 4.191971e-03 +Size 64 +PMPI: 3.320248e-03 +Pairwise: 5.216261e-03 +NonBlocking: 5.159091e-03 +Hierarchical: 5.578737e-03 +Multileader: 4.987034e-03 +Node Aware: 5.476590e-03 +Locality Aware: 4.918118e-03 +Multileader Locality: 6.100232e-03 +Size 128 +PMPI: 6.658934e-03 +Pairwise: 7.334975e-03 +NonBlocking: 6.529542e-03 +Hierarchical: 7.466486e-03 +Multileader: 7.032817e-03 +Node Aware: 7.433340e-03 +Locality Aware: 6.495320e-03 +Multileader Locality: 7.562520e-03 +Size 256 +PMPI: 1.195027e-02 +Pairwise: 9.551239e-03 +NonBlocking: 1.067986e-02 +Hierarchical: 9.826371e-03 +Multileader: 1.143595e-02 +Node Aware: 1.002337e-02 +Locality Aware: 1.117640e-02 +Multileader Locality: 1.012191e-02 +Size 512 +PMPI: 2.761179e-02 +Pairwise: 1.754302e-02 +NonBlocking: 1.818211e-02 +Hierarchical: 1.786007e-02 +Multileader: 1.888388e-02 +Node Aware: 1.859146e-02 +Locality Aware: 1.800308e-02 +Multileader Locality: 1.856626e-02 +Size 1024 +PMPI: 3.470505e-02 +Pairwise: 3.384656e-02 +NonBlocking: 3.219511e-02 +Hierarchical: 3.477765e-02 +Multileader: 3.311291e-02 +Node Aware: 3.349933e-02 +Locality Aware: 3.368150e-02 +Multileader Locality: 3.436934e-02 +Size 2048 +PMPI: 6.827196e-02 +Pairwise: 6.585034e-02 +NonBlocking: 6.711745e-02 +Hierarchical: 6.768358e-02 +Multileader: 6.635954e-02 +Node Aware: 6.561488e-02 +Locality Aware: 6.789139e-02 +Multileader Locality: 6.873107e-02 +Size 4096 +PMPI: 1.394610e-01 +Pairwise: 1.280506e-01 +NonBlocking: 1.359771e-01 +Hierarchical: 1.343962e-01 +Multileader: 1.311742e-01 +Node Aware: 1.310061e-01 +Locality Aware: 1.344967e-01 +Multileader Locality: 1.286257e-01 +Size 8192 +PMPI: 2.642176e-01 +Pairwise: 2.625627e-01 +NonBlocking: 2.504849e-01 diff --git a/src/collective/alltoall.c b/src/collective/alltoall.c index 01698c63e..8d0262500 100644 --- a/src/collective/alltoall.c +++ b/src/collective/alltoall.c @@ -64,19 +64,21 @@ int MPIX_Alltoall(const void* sendbuf, mpi_comm); } -int alltoall_pairwise(const void* sendbuf, + +int pairwise_helper(const void* sendbuf, const int sendcount, MPI_Datatype sendtype, void* recvbuf, const int recvcount, MPI_Datatype recvtype, - MPIX_Comm* comm) + MPI_Comm comm) { int rank, num_procs; - MPI_Comm_rank(comm->global_comm, &rank); - MPI_Comm_size(comm->global_comm, &num_procs); + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &num_procs); + + int tag = 10238; - int tag = 102944; int send_proc, recv_proc; int send_pos, recv_pos; MPI_Status status; @@ -88,31 +90,9 @@ int alltoall_pairwise(const void* sendbuf, MPI_Type_size(sendtype, &send_size); MPI_Type_size(recvtype, &recv_size); -#ifdef GPU - gpuMemoryType send_type, recv_type; - gpuMemcpyKind memcpy_kind; - get_mem_types(sendbuf, recvbuf, &send_type, &recv_type); - - if (send_type == gpuMemoryTypeDevice || - recv_type == gpuMemoryTypeDevice) - { - get_memcpy_kind(send_type, recv_type, &memcpy_kind); - int ierr = gpuMemcpy(recv_buffer + (rank * recvcount * recv_size), - send_buffer + (rank * sendcount * send_size), - sendcount * send_size, - memcpy_kind); - gpu_check(ierr); - } - else -#endif - memcpy(recv_buffer + (rank * recvcount * recv_size), - send_buffer + (rank * sendcount * send_size), - sendcount * send_size); - - // Send to rank + i // Recv from rank - i - for (int i = 1; i < num_procs; i++) + for (int i = 0; i < num_procs; i++) { send_proc = rank + i; if (send_proc >= num_procs) @@ -133,27 +113,28 @@ int alltoall_pairwise(const void* sendbuf, recvtype, recv_proc, tag, - comm->global_comm, + comm, &status); } return MPI_SUCCESS; } -int alltoall_nonblocking(const void* sendbuf, +int nonblocking_helper(const void* sendbuf, const int sendcount, MPI_Datatype sendtype, void* recvbuf, const int recvcount, MPI_Datatype recvtype, - MPIX_Comm* comm) + MPI_Comm comm) { int rank, num_procs; - MPI_Comm_rank(comm->global_comm, &rank); - MPI_Comm_size(comm->global_comm, &num_procs); + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &num_procs); - int tag = 102944; + int tag = 10239; int send_proc, recv_proc; int send_pos, recv_pos; + MPI_Status status; char* recv_buffer = (char*)recvbuf; char* send_buffer = (char*)sendbuf; @@ -162,32 +143,11 @@ int alltoall_nonblocking(const void* sendbuf, MPI_Type_size(sendtype, &send_size); MPI_Type_size(recvtype, &recv_size); - MPI_Request* requests = (MPI_Request*)malloc(2*(num_procs-1)*sizeof(MPI_Request)); - -#ifdef GPU - gpuMemoryType send_type, recv_type; - gpuMemcpyKind memcpy_kind; - get_mem_types(sendbuf, recvbuf, &send_type, &recv_type); - - if (send_type == gpuMemoryTypeDevice || - recv_type == gpuMemoryTypeDevice) - { - get_memcpy_kind(send_type, recv_type, &memcpy_kind); - int ierr = gpuMemcpy(recv_buffer + (rank * recvcount * recv_size), - send_buffer + (rank * sendcount * send_size), - sendcount * send_size, - memcpy_kind); - gpu_check(ierr); - } - else -#endif - memcpy(recv_buffer + (rank * recvcount * recv_size), - send_buffer + (rank * sendcount * send_size), - sendcount * send_size); + MPI_Request* requests = (MPI_Request*)malloc(2*num_procs*sizeof(MPI_Request)); // Send to rank + i // Recv from rank - i - for (int i = 1; i < num_procs; i++) + for (int i = 0; i < num_procs; i++) { send_proc = rank + i; if (send_proc >= num_procs) @@ -203,20 +163,1145 @@ int alltoall_nonblocking(const void* sendbuf, sendtype, send_proc, tag, - comm->global_comm, - &(requests[i-1])); + comm, + &(requests[i])); MPI_Irecv(recv_buffer + recv_pos, recvcount, recvtype, recv_proc, tag, - comm->global_comm, - &(requests[num_procs + i - 2])); + comm, + &(requests[num_procs + i])); } - MPI_Waitall(2*(num_procs-1), requests, MPI_STATUSES_IGNORE); + MPI_Waitall(2*num_procs, requests, MPI_STATUSES_IGNORE); free(requests); - return 0; + return MPI_SUCCESS; +} + + + + + +int alltoall_pairwise(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm) +{ + return pairwise_helper(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm->global_comm); +} + +int alltoall_nonblocking(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm) +{ + return nonblocking_helper(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm->global_comm); +} + + + + + + + +/* Pairwise Versions */ +int alltoall_hierarchical(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm) +{ + + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + int tag = 10240; + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char* recv_buffer = (char*)recvbuf; + char* send_buffer = (char*)sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int local_rank, ppn; + MPI_Comm_rank(comm->local_comm, &local_rank); + MPI_Comm_size(comm->local_comm, &ppn); + + // TODO: currently assuming full nodes, even ppn per node + // this is common, so fair assumption for now + // likely need to fix before using in something like Trilinos + int n_nodes = num_procs / ppn; + + char* local_send_buffer = NULL; + char* local_recv_buffer = NULL; + + if (local_rank == 0) + { + local_send_buffer = (char*)malloc(ppn*num_procs*sendcount*send_size); + local_recv_buffer = (char*)malloc(ppn*num_procs*recvcount*recv_size); + } + else + { + local_send_buffer = (char*)malloc(sizeof(char)); + local_recv_buffer = (char*)malloc(sizeof(char)); + } + + // 1. Gather locally + MPI_Gather(send_buffer, sendcount*num_procs, sendtype, local_recv_buffer, sendcount*num_procs, sendtype, + 0, comm->local_comm); + + // 2. Re-pack for sends + // Assumes SMP ordering + // TODO: allow for other orderings + int ctr; + + if (local_rank == 0) + { + ctr = 0; + for (int dest_node = 0; dest_node < n_nodes; dest_node++) + { + int dest_node_start = dest_node * ppn * sendcount * send_size; + for (int origin_proc = 0; origin_proc < ppn; origin_proc++) + { + int origin_proc_start = origin_proc * num_procs * sendcount * send_size; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[origin_proc_start + dest_node_start]), + ppn * sendcount * send_size); + ctr += ppn * sendcount * send_size; + } + } + + // 3. MPI_Alltoall between leaders + pairwise_helper(local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->group_comm); + + // 4. Re-pack for local scatter + ctr = 0; + for (int dest_proc = 0; dest_proc < ppn; dest_proc++) + { + int dest_proc_start = dest_proc * recvcount * recv_size; + for (int orig_proc = 0; orig_proc < num_procs; orig_proc++) + { + int orig_proc_start = orig_proc * ppn * recvcount * recv_size; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[orig_proc_start + dest_proc_start]), + recvcount * recv_size); + ctr += recvcount * recv_size; + + } + } + } + + // 5. Scatter + MPI_Scatter(local_send_buffer, recvcount * num_procs, recvtype, recv_buffer, recvcount * num_procs, recvtype, + 0, comm->local_comm); + + + free(local_send_buffer); + free(local_recv_buffer); + + return MPI_SUCCESS; +} + + +int alltoall_multileader(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm) +{ + + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + int tag = 10241; + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int num_leaders_per_node = 4; + int procs_per_node; + MPI_Comm_size(comm->local_comm, &procs_per_node); + int procs_per_leader = procs_per_node / num_leaders_per_node; + if (procs_per_node < num_leaders_per_node) + { + num_leaders_per_node = procs_per_node; + procs_per_leader = 1; + } + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char* recv_buffer = (char*)recvbuf; + char* send_buffer = (char*)sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + if (comm->leader_comm == MPI_COMM_NULL) + MPIX_Comm_leader_init(comm, procs_per_leader); + + int local_rank, ppn; + MPI_Comm_rank(comm->leader_comm, &local_rank); + MPI_Comm_size(comm->leader_comm, &ppn); + + // TODO: currently assuming full nodes, even ppn per node + // this is common, so fair assumption for now + // likely need to fix before using in something like Trilinos + int n_nodes = num_procs / ppn; + + char* local_send_buffer = NULL; + char* local_recv_buffer = NULL; + + if (local_rank == 0) + { + local_send_buffer = (char*)malloc(ppn*num_procs*sendcount*send_size); + local_recv_buffer = (char*)malloc(ppn*num_procs*recvcount*recv_size); + } + else + { + local_send_buffer = (char*)malloc(sizeof(char)); + local_recv_buffer = (char*)malloc(sizeof(char)); + } + + // 1. Gather locally + MPI_Gather(send_buffer, sendcount*num_procs, sendtype, local_recv_buffer, sendcount*num_procs, sendtype, + 0, comm->leader_comm); + + // 2. Re-pack for sends + // Assumes SMP ordering + // TODO: allow for other orderings + int ctr; + + if (local_rank == 0) + { + ctr = 0; + for (int dest_node = 0; dest_node < n_nodes; dest_node++) + { + int dest_node_start = dest_node * ppn * sendcount * send_size; + for (int origin_proc = 0; origin_proc < ppn; origin_proc++) + { + int origin_proc_start = origin_proc * num_procs * sendcount * send_size; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[origin_proc_start + dest_node_start]), + ppn * sendcount * send_size); + ctr += ppn * sendcount * send_size; + } + } + + // 3. MPI_Alltoall between leaders + pairwise_helper(local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->leader_group_comm); + + // 4. Re-pack for local scatter + ctr = 0; + for (int dest_proc = 0; dest_proc < ppn; dest_proc++) + { + int dest_proc_start = dest_proc * recvcount * recv_size; + for (int orig_proc = 0; orig_proc < num_procs; orig_proc++) + { + int orig_proc_start = orig_proc * ppn * recvcount * recv_size; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[orig_proc_start + dest_proc_start]), + recvcount * recv_size); + ctr += recvcount * recv_size; + + } + } + } + + // 5. Scatter + MPI_Scatter(local_send_buffer, recvcount * num_procs, recvtype, recv_buffer, recvcount*num_procs, recvtype, + 0, comm->leader_comm); + + + free(local_send_buffer); + free(local_recv_buffer); + + return MPI_SUCCESS; +} + +int alltoall_node_aware(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm) +{ + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + int tag = 10242; + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int local_rank, ppn; + MPI_Comm_rank(comm->local_comm, &local_rank); + MPI_Comm_size(comm->local_comm, &ppn); + + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char* recv_buffer = (char*)recvbuf; + char* send_buffer = (char*)sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + int n_nodes = num_procs / ppn; + + char* tmpbuf = (char*)malloc(num_procs*sendcount*send_size); + + // 1. Alltoall between group_comms (all data for any process on node) + pairwise_helper(sendbuf, ppn*sendcount, sendtype, tmpbuf, ppn*recvcount, recvtype, + comm->group_comm); + + // 2. Re-pack + int ctr = 0; + for (int dest_proc = 0; dest_proc < ppn; dest_proc++) + { + int offset = dest_proc * recvcount * recv_size; + for (int origin = 0; origin < n_nodes; origin++) + { + int node_offset = origin * ppn * recvcount * recv_size; + memcpy(&(recvbuf[ctr]), &(tmpbuf[node_offset + offset]), recvcount*recv_size); + ctr += recvcount * recv_size; + } + } + + + // 3. Local alltoall + pairwise_helper(recvbuf, n_nodes*recvcount, recvtype, + tmpbuf, n_nodes*recvcount, recvtype, comm->local_comm); + + // 4. Re-order + ctr = 0; + for (int node = 0; node < n_nodes; node++) + { + int node_offset = node * recvcount * recv_size; + for (int dest = 0; dest < ppn; dest++) + { + int dest_offset = dest * n_nodes * recvcount * recv_size; + memcpy(&(recvbuf[ctr]), &(tmpbuf[node_offset + dest_offset]), + recvcount * recv_size); + ctr += recvcount * recv_size; + } + } + + + free(tmpbuf); + + return MPI_SUCCESS; +} + + +int alltoall_locality_aware(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm) +{ + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + int tag = 10243; + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int num_leaders_per_node = 4; + int procs_per_node; + MPI_Comm_size(comm->local_comm, &procs_per_node); + int procs_per_leader = procs_per_node / num_leaders_per_node; + if (procs_per_node < num_leaders_per_node) + { + num_leaders_per_node = procs_per_node; + procs_per_leader = 1; + } + + if (comm->leader_comm == MPI_COMM_NULL) + MPIX_Comm_leader_init(comm, procs_per_leader); + + int ppn; + MPI_Comm_size(comm->leader_comm, &ppn); + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char* recv_buffer = (char*)recvbuf; + char* send_buffer = (char*)sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + int n_nodes = num_procs / ppn; + + char* tmpbuf = (char*)malloc(num_procs*sendcount*send_size); + + // 1. Alltoall between group_comms (all data for any process on node) + pairwise_helper(sendbuf, ppn*sendcount, sendtype, tmpbuf, ppn*recvcount, recvtype, + comm->leader_group_comm); + + // 2. Re-pack + int ctr = 0; + for (int dest_proc = 0; dest_proc < ppn; dest_proc++) + { + int offset = dest_proc * recvcount * recv_size; + for (int origin = 0; origin < n_nodes; origin++) + { + int node_offset = origin * ppn * recvcount * recv_size; + memcpy(&(recvbuf[ctr]), &(tmpbuf[node_offset + offset]), recvcount*recv_size); + ctr += recvcount * recv_size; + } + } + + // 3. Local alltoall + pairwise_helper(recvbuf, n_nodes*recvcount, recvtype, + tmpbuf, n_nodes*recvcount, recvtype, comm->leader_comm); + + // 4. Re-order + ctr = 0; + for (int node = 0; node < n_nodes; node++) + { + int node_offset = node * recvcount * recv_size; + for (int dest = 0; dest < ppn; dest++) + { + int dest_offset = dest * n_nodes * recvcount * recv_size; + memcpy(&(recvbuf[ctr]), &(tmpbuf[node_offset + dest_offset]), + recvcount * recv_size); + ctr += recvcount * recv_size; + } + } + + free(tmpbuf); + return MPI_SUCCESS; +} +int alltoall_multileader_locality(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm) +{ + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + int tag = 10241; + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int local_rank, ppn; + MPI_Comm_rank(comm->local_comm, &local_rank); + MPI_Comm_size(comm->local_comm, &ppn); + + if (comm->leader_comm == MPI_COMM_NULL) + { + int num_leaders_per_node = 4; + if (ppn < num_leaders_per_node) + num_leaders_per_node = ppn; + MPIX_Comm_leader_init(comm, ppn / num_leaders_per_node); + } + + int procs_per_leader, leader_rank; + MPI_Comm_rank(comm->leader_comm, &leader_rank); + MPI_Comm_size(comm->leader_comm, &procs_per_leader); + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char* recv_buffer = (char*)recvbuf; + char* send_buffer = (char*)sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + // TODO: currently assuming full nodes, even procs_per_leader per node + // this is common, so fair assumption for now + // likely need to fix before using in something like Trilinos + int n_nodes = num_procs / ppn; + int n_leaders = num_procs / procs_per_leader; + + int leaders_per_node; + MPI_Comm_size(comm->leader_local_comm, &leaders_per_node); + + + char* local_send_buffer = NULL; + char* local_recv_buffer = NULL; + if (leader_rank == 0) + { + local_send_buffer = (char*)malloc(procs_per_leader*num_procs*sendcount*send_size); + local_recv_buffer = (char*)malloc(procs_per_leader*num_procs*recvcount*recv_size); + } + else + { + local_send_buffer = (char*)malloc(sizeof(char)); + local_recv_buffer = (char*)malloc(sizeof(char)); + } + // 1. Gather locally + MPI_Gather(send_buffer, sendcount*num_procs, sendtype, local_recv_buffer, sendcount*num_procs, sendtype, + 0, comm->leader_comm); + + + // 2. Re-pack for sends + // Assumes SMP ordering + // TODO: allow for other orderings + int ctr; + + if (leader_rank == 0) + { + ctr = 0; + for (int dest_node = 0; dest_node < n_leaders; dest_node++) + { + int dest_node_start = dest_node * procs_per_leader * sendcount * send_size; + for (int origin_proc = 0; origin_proc < procs_per_leader; origin_proc++) + { + int origin_proc_start = origin_proc * num_procs * sendcount * send_size; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[origin_proc_start + dest_node_start]), + procs_per_leader * sendcount * send_size); + ctr += procs_per_leader * sendcount * send_size; + } + } + + // 3. MPI_Alltoall between nodes + pairwise_helper(local_send_buffer, ppn*procs_per_leader*sendcount, sendtype, + local_recv_buffer, ppn*procs_per_leader*recvcount, recvtype, comm->group_comm); + + // Re-Pack for exchange between local leaders + ctr = 0; + for (int local_leader = 0; local_leader < leaders_per_node; local_leader++) + { + int leader_start = local_leader*procs_per_leader*procs_per_leader*sendcount*send_size; + for (int dest_node = 0; dest_node < n_nodes; dest_node++) + { + int dest_node_start = dest_node*ppn*procs_per_leader*sendcount*send_size; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[dest_node_start+leader_start]), + procs_per_leader*procs_per_leader*sendcount*send_size); + ctr += procs_per_leader*procs_per_leader*sendcount*send_size; + } + } + + pairwise_helper(local_send_buffer, n_nodes*procs_per_leader*procs_per_leader*sendcount, sendtype, + local_recv_buffer, n_nodes*procs_per_leader*procs_per_leader*recvcount, recvtype, comm->leader_local_comm); + + ctr = 0; + for (int dest_proc = 0; dest_proc < procs_per_leader; dest_proc++) + { + int dest_proc_start = dest_proc * recvcount * recv_size; + + for (int orig_node = 0; orig_node < n_nodes; orig_node++) + { + int orig_node_start = orig_node*procs_per_leader*procs_per_leader*recvcount*recv_size; + + for (int orig_leader = 0; orig_leader < leaders_per_node; orig_leader++) + { + int orig_leader_start = orig_leader*n_nodes*procs_per_leader*procs_per_leader*recvcount*recv_size; + for (int orig_proc = 0; orig_proc < procs_per_leader; orig_proc++) + { + int orig_proc_start = orig_proc*procs_per_leader*recvcount*recv_size; + int idx = orig_node_start + orig_leader_start + orig_proc_start + dest_proc_start; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[idx]), recvcount*recv_size); + ctr += recvcount * recv_size; + } + } + } + } + } + + // 5. Scatter + MPI_Scatter(local_send_buffer, recvcount * num_procs, recvtype, recv_buffer, recvcount*num_procs, recvtype, + 0, comm->leader_comm); + + free(local_send_buffer); + free(local_recv_buffer); + + + return MPI_SUCCESS; +} + + + + + + + + + +/* Non Blocking Versions:*/ +int alltoall_hierarchical_nb(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm) +{ + + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + int tag = 10240; + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char* recv_buffer = (char*)recvbuf; + char* send_buffer = (char*)sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int local_rank, ppn; + MPI_Comm_rank(comm->local_comm, &local_rank); + MPI_Comm_size(comm->local_comm, &ppn); + + // TODO: currently assuming full nodes, even ppn per node + // this is common, so fair assumption for now + // likely need to fix before using in something like Trilinos + int n_nodes = num_procs / ppn; + + char* local_send_buffer = NULL; + char* local_recv_buffer = NULL; + + if (local_rank == 0) + { + local_send_buffer = (char*)malloc(ppn*num_procs*sendcount*send_size); + local_recv_buffer = (char*)malloc(ppn*num_procs*recvcount*recv_size); + } + else + { + local_send_buffer = (char*)malloc(sizeof(char)); + local_recv_buffer = (char*)malloc(sizeof(char)); + } + + // 1. Gather locally + MPI_Gather(send_buffer, sendcount*num_procs, sendtype, local_recv_buffer, sendcount*num_procs, sendtype, + 0, comm->local_comm); + + // 2. Re-pack for sends + // Assumes SMP ordering + // TODO: allow for other orderings + int ctr; + + if (local_rank == 0) + { + ctr = 0; + for (int dest_node = 0; dest_node < n_nodes; dest_node++) + { + int dest_node_start = dest_node * ppn * sendcount * send_size; + for (int origin_proc = 0; origin_proc < ppn; origin_proc++) + { + int origin_proc_start = origin_proc * num_procs * sendcount * send_size; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[origin_proc_start + dest_node_start]), + ppn * sendcount * send_size); + ctr += ppn * sendcount * send_size; + } + } + + // 3. MPI_Alltoall between leaders + nonblocking_helper(local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->group_comm); + + // 4. Re-pack for local scatter + ctr = 0; + for (int dest_proc = 0; dest_proc < ppn; dest_proc++) + { + int dest_proc_start = dest_proc * recvcount * recv_size; + for (int orig_proc = 0; orig_proc < num_procs; orig_proc++) + { + int orig_proc_start = orig_proc * ppn * recvcount * recv_size; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[orig_proc_start + dest_proc_start]), + recvcount * recv_size); + ctr += recvcount * recv_size; + + } + } + } + + // 5. Scatter + MPI_Scatter(local_send_buffer, recvcount * num_procs, recvtype, recv_buffer, recvcount * num_procs, recvtype, + 0, comm->local_comm); + + + free(local_send_buffer); + free(local_recv_buffer); + + return MPI_SUCCESS; +} + + +int alltoall_multileader_nb(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm) +{ + + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + int tag = 10241; + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int num_leaders_per_node = 4; + int procs_per_node; + MPI_Comm_size(comm->local_comm, &procs_per_node); + int procs_per_leader = procs_per_node / num_leaders_per_node; + if (procs_per_node < num_leaders_per_node) + { + num_leaders_per_node = procs_per_node; + procs_per_leader = 1; + } + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char* recv_buffer = (char*)recvbuf; + char* send_buffer = (char*)sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + if (comm->leader_comm == MPI_COMM_NULL) + MPIX_Comm_leader_init(comm, procs_per_leader); + + int local_rank, ppn; + MPI_Comm_rank(comm->leader_comm, &local_rank); + MPI_Comm_size(comm->leader_comm, &ppn); + + // TODO: currently assuming full nodes, even ppn per node + // this is common, so fair assumption for now + // likely need to fix before using in something like Trilinos + int n_nodes = num_procs / ppn; + + char* local_send_buffer = NULL; + char* local_recv_buffer = NULL; + + if (local_rank == 0) + { + local_send_buffer = (char*)malloc(ppn*num_procs*sendcount*send_size); + local_recv_buffer = (char*)malloc(ppn*num_procs*recvcount*recv_size); + } + else + { + local_send_buffer = (char*)malloc(sizeof(char)); + local_recv_buffer = (char*)malloc(sizeof(char)); + } + + // 1. Gather locally + MPI_Gather(send_buffer, sendcount*num_procs, sendtype, local_recv_buffer, sendcount*num_procs, sendtype, + 0, comm->leader_comm); + + // 2. Re-pack for sends + // Assumes SMP ordering + // TODO: allow for other orderings + int ctr; + + if (local_rank == 0) + { + ctr = 0; + for (int dest_node = 0; dest_node < n_nodes; dest_node++) + { + int dest_node_start = dest_node * ppn * sendcount * send_size; + for (int origin_proc = 0; origin_proc < ppn; origin_proc++) + { + int origin_proc_start = origin_proc * num_procs * sendcount * send_size; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[origin_proc_start + dest_node_start]), + ppn * sendcount * send_size); + ctr += ppn * sendcount * send_size; + } + } + + // 3. MPI_Alltoall between leaders + nonblocking_helper(local_send_buffer, ppn * ppn * sendcount, sendtype, + local_recv_buffer, ppn * ppn * recvcount, recvtype, comm->leader_group_comm); + + // 4. Re-pack for local scatter + ctr = 0; + for (int dest_proc = 0; dest_proc < ppn; dest_proc++) + { + int dest_proc_start = dest_proc * recvcount * recv_size; + for (int orig_proc = 0; orig_proc < num_procs; orig_proc++) + { + int orig_proc_start = orig_proc * ppn * recvcount * recv_size; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[orig_proc_start + dest_proc_start]), + recvcount * recv_size); + ctr += recvcount * recv_size; + + } + } + } + + // 5. Scatter + MPI_Scatter(local_send_buffer, recvcount * num_procs, recvtype, recv_buffer, recvcount*num_procs, recvtype, + 0, comm->leader_comm); + + + free(local_send_buffer); + free(local_recv_buffer); + + return MPI_SUCCESS; +} + +int alltoall_node_aware_nb(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm) +{ + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + int tag = 10242; + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int local_rank, ppn; + MPI_Comm_rank(comm->local_comm, &local_rank); + MPI_Comm_size(comm->local_comm, &ppn); + + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char* recv_buffer = (char*)recvbuf; + char* send_buffer = (char*)sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + int n_nodes = num_procs / ppn; + + char* tmpbuf = (char*)malloc(num_procs*sendcount*send_size); + + // 1. Alltoall between group_comms (all data for any process on node) + nonblocking_helper(sendbuf, ppn*sendcount, sendtype, tmpbuf, ppn*recvcount, recvtype, + comm->group_comm); + + // 2. Re-pack + int ctr = 0; + for (int dest_proc = 0; dest_proc < ppn; dest_proc++) + { + int offset = dest_proc * recvcount * recv_size; + for (int origin = 0; origin < n_nodes; origin++) + { + int node_offset = origin * ppn * recvcount * recv_size; + memcpy(&(recvbuf[ctr]), &(tmpbuf[node_offset + offset]), recvcount*recv_size); + ctr += recvcount * recv_size; + } + } + + + // 3. Local alltoall + nonblocking_helper(recvbuf, n_nodes*recvcount, recvtype, + tmpbuf, n_nodes*recvcount, recvtype, comm->local_comm); + + // 4. Re-order + ctr = 0; + for (int node = 0; node < n_nodes; node++) + { + int node_offset = node * recvcount * recv_size; + for (int dest = 0; dest < ppn; dest++) + { + int dest_offset = dest * n_nodes * recvcount * recv_size; + memcpy(&(recvbuf[ctr]), &(tmpbuf[node_offset + dest_offset]), + recvcount * recv_size); + ctr += recvcount * recv_size; + } + } + + + free(tmpbuf); + + return MPI_SUCCESS; +} + + +int alltoall_locality_aware_nb(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm) +{ + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + int tag = 10243; + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int num_leaders_per_node = 4; + int procs_per_node; + MPI_Comm_size(comm->local_comm, &procs_per_node); + int procs_per_leader = procs_per_node / num_leaders_per_node; + if (procs_per_node < num_leaders_per_node) + { + num_leaders_per_node = procs_per_node; + procs_per_leader = 1; + } + + if (comm->leader_comm == MPI_COMM_NULL) + MPIX_Comm_leader_init(comm, procs_per_leader); + + int ppn; + MPI_Comm_size(comm->leader_comm, &ppn); + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char* recv_buffer = (char*)recvbuf; + char* send_buffer = (char*)sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + int n_nodes = num_procs / ppn; + + char* tmpbuf = (char*)malloc(num_procs*sendcount*send_size); + + // 1. Alltoall between group_comms (all data for any process on node) + nonblocking_helper(sendbuf, ppn*sendcount, sendtype, tmpbuf, ppn*recvcount, recvtype, + comm->leader_group_comm); + + // 2. Re-pack + int ctr = 0; + for (int dest_proc = 0; dest_proc < ppn; dest_proc++) + { + int offset = dest_proc * recvcount * recv_size; + for (int origin = 0; origin < n_nodes; origin++) + { + int node_offset = origin * ppn * recvcount * recv_size; + memcpy(&(recvbuf[ctr]), &(tmpbuf[node_offset + offset]), recvcount*recv_size); + ctr += recvcount * recv_size; + } + } + + // 3. Local alltoall + nonblocking_helper(recvbuf, n_nodes*recvcount, recvtype, + tmpbuf, n_nodes*recvcount, recvtype, comm->leader_comm); + + // 4. Re-order + ctr = 0; + for (int node = 0; node < n_nodes; node++) + { + int node_offset = node * recvcount * recv_size; + for (int dest = 0; dest < ppn; dest++) + { + int dest_offset = dest * n_nodes * recvcount * recv_size; + memcpy(&(recvbuf[ctr]), &(tmpbuf[node_offset + dest_offset]), + recvcount * recv_size); + ctr += recvcount * recv_size; + } + } + + free(tmpbuf); + return MPI_SUCCESS; +} + +int alltoall_multileader_locality_nb(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm) +{ + int rank, num_procs; + MPI_Comm_rank(comm->global_comm, &rank); + MPI_Comm_size(comm->global_comm, &num_procs); + + int tag = 10241; + + if (comm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(comm); + + int local_rank, ppn; + MPI_Comm_rank(comm->local_comm, &local_rank); + MPI_Comm_size(comm->local_comm, &ppn); + + if (comm->leader_comm == MPI_COMM_NULL) + { + int num_leaders_per_node = 4; + if (ppn < num_leaders_per_node) + num_leaders_per_node = ppn; + MPIX_Comm_leader_init(comm, ppn / num_leaders_per_node); + } + + int procs_per_leader, leader_rank; + MPI_Comm_rank(comm->leader_comm, &leader_rank); + MPI_Comm_size(comm->leader_comm, &procs_per_leader); + + int send_proc, recv_proc; + int send_pos, recv_pos; + MPI_Status status; + + char* recv_buffer = (char*)recvbuf; + char* send_buffer = (char*)sendbuf; + + int send_size, recv_size; + MPI_Type_size(sendtype, &send_size); + MPI_Type_size(recvtype, &recv_size); + + // TODO: currently assuming full nodes, even procs_per_leader per node + // this is common, so fair assumption for now + // likely need to fix before using in something like Trilinos + int n_nodes = num_procs / ppn; + int n_leaders = num_procs / procs_per_leader; + + int leaders_per_node; + MPI_Comm_size(comm->leader_local_comm, &leaders_per_node); + + + char* local_send_buffer = NULL; + char* local_recv_buffer = NULL; + if (leader_rank == 0) + { + local_send_buffer = (char*)malloc(procs_per_leader*num_procs*sendcount*send_size); + local_recv_buffer = (char*)malloc(procs_per_leader*num_procs*recvcount*recv_size); + } + else + { + local_send_buffer = (char*)malloc(sizeof(char)); + local_recv_buffer = (char*)malloc(sizeof(char)); + } + // 1. Gather locally + MPI_Gather(send_buffer, sendcount*num_procs, sendtype, local_recv_buffer, sendcount*num_procs, sendtype, + 0, comm->leader_comm); + + + // 2. Re-pack for sends + // Assumes SMP ordering + // TODO: allow for other orderings + int ctr; + + if (leader_rank == 0) + { + ctr = 0; + for (int dest_node = 0; dest_node < n_leaders; dest_node++) + { + int dest_node_start = dest_node * procs_per_leader * sendcount * send_size; + for (int origin_proc = 0; origin_proc < procs_per_leader; origin_proc++) + { + int origin_proc_start = origin_proc * num_procs * sendcount * send_size; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[origin_proc_start + dest_node_start]), + procs_per_leader * sendcount * send_size); + ctr += procs_per_leader * sendcount * send_size; + } + } + + // 3. MPI_Alltoall between nodes + nonblocking_helper(local_send_buffer, ppn*procs_per_leader*sendcount, sendtype, + local_recv_buffer, ppn*procs_per_leader*recvcount, recvtype, comm->group_comm); + + // Re-Pack for exchange between local leaders + ctr = 0; + for (int local_leader = 0; local_leader < leaders_per_node; local_leader++) + { + int leader_start = local_leader*procs_per_leader*procs_per_leader*sendcount*send_size; + for (int dest_node = 0; dest_node < n_nodes; dest_node++) + { + int dest_node_start = dest_node*ppn*procs_per_leader*sendcount*send_size; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[dest_node_start+leader_start]), + procs_per_leader*procs_per_leader*sendcount*send_size); + ctr += procs_per_leader*procs_per_leader*sendcount*send_size; + } + } + + nonblocking_helper(local_send_buffer, n_nodes*procs_per_leader*procs_per_leader*sendcount, sendtype, + local_recv_buffer, n_nodes*procs_per_leader*procs_per_leader*recvcount, recvtype, comm->leader_local_comm); + + ctr = 0; + for (int dest_proc = 0; dest_proc < procs_per_leader; dest_proc++) + { + int dest_proc_start = dest_proc * recvcount * recv_size; + + for (int orig_node = 0; orig_node < n_nodes; orig_node++) + { + int orig_node_start = orig_node*procs_per_leader*procs_per_leader*recvcount*recv_size; + + for (int orig_leader = 0; orig_leader < leaders_per_node; orig_leader++) + { + int orig_leader_start = orig_leader*n_nodes*procs_per_leader*procs_per_leader*recvcount*recv_size; + for (int orig_proc = 0; orig_proc < procs_per_leader; orig_proc++) + { + int orig_proc_start = orig_proc*procs_per_leader*recvcount*recv_size; + int idx = orig_node_start + orig_leader_start + orig_proc_start + dest_proc_start; + memcpy(&(local_send_buffer[ctr]), &(local_recv_buffer[idx]), recvcount*recv_size); + ctr += recvcount * recv_size; + } + } + } + } + } + + // 5. Scatter + MPI_Scatter(local_send_buffer, recvcount * num_procs, recvtype, recv_buffer, recvcount*num_procs, recvtype, + 0, comm->leader_comm); + + free(local_send_buffer); + free(local_recv_buffer); + + + return MPI_SUCCESS; } diff --git a/src/collective/alltoall.h b/src/collective/alltoall.h index a7d73d2bb..147669b09 100644 --- a/src/collective/alltoall.h +++ b/src/collective/alltoall.h @@ -29,6 +29,98 @@ int alltoall_nonblocking(const void* sendbuf, MPI_Datatype recvtype, MPIX_Comm* comm); + // Hack to support timing + int pairwise_helper(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPI_Comm comm); + + int nonblocking_helper(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPI_Comm comm); + + +int alltoall_hierarchical(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm); +int alltoall_multileader(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm); +int alltoall_node_aware(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm); +int alltoall_locality_aware(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm); +int alltoall_multileader_locality(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm); + + +int alltoall_hierarchical_nb(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm); +int alltoall_multileader_nb(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm); +int alltoall_node_aware_nb(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm); +int alltoall_locality_aware_nb(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm); +int alltoall_multileader_locality_nb(const void* sendbuf, + const int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcount, + MPI_Datatype recvtype, + MPIX_Comm* comm); + + #ifdef __cplusplus } diff --git a/src/collective/tests/test_alltoall.cpp b/src/collective/tests/test_alltoall.cpp index 0039d04a5..4d9ff514d 100644 --- a/src/collective/tests/test_alltoall.cpp +++ b/src/collective/tests/test_alltoall.cpp @@ -39,12 +39,12 @@ TEST(RandomCommTest, TestsInTests) std::vector local_data(max_s*num_procs); std::vector std_alltoall(max_s*num_procs); - std::vector pairwise_alltoall(max_s*num_procs); - std::vector loc_pairwise_alltoall(max_s*num_procs); + std::vector mpix_alltoall(max_s*num_procs); - MPIX_Comm* locality_comm; - MPIX_Comm_init(&locality_comm, MPI_COMM_WORLD); - update_locality(locality_comm, 4); + MPIX_Comm* xcomm; + MPIX_Comm_init(&xcomm, MPI_COMM_WORLD); + update_locality(xcomm, 20); + MPIX_Comm_leader_init(xcomm, 4); for (int i = 0; i < max_i; i++) { @@ -64,42 +64,156 @@ TEST(RandomCommTest, TestsInTests) MPI_INT, MPI_COMM_WORLD); + std::fill(mpix_alltoall.begin(), mpix_alltoall.end(), 0); + MPIX_Alltoall(local_data.data(), + s, + MPI_INT, + mpix_alltoall.data(), + s, + MPI_INT, + xcomm); + for (int j = 0; j < s*num_procs; j++) + ASSERT_EQ(std_alltoall[j], mpix_alltoall[j]); - // Pairwise Alltoall - MPI_Alltoall(local_data.data(), - s, - MPI_INT, - pairwise_alltoall.data(), + std::fill(mpix_alltoall.begin(), mpix_alltoall.end(), 0); + alltoall_pairwise(local_data.data(), s, MPI_INT, - MPI_COMM_WORLD); + mpix_alltoall.data(), + s, + MPI_INT, + xcomm); for (int j = 0; j < s*num_procs; j++) - ASSERT_EQ(std_alltoall[j], pairwise_alltoall[j]); + ASSERT_EQ(std_alltoall[j], mpix_alltoall[j]); - // Locality-Aware Pairwise Alltoall - MPIX_Alltoall(local_data.data(), + std::fill(mpix_alltoall.begin(), mpix_alltoall.end(), 0); + alltoall_nonblocking(local_data.data(), s, MPI_INT, - loc_pairwise_alltoall.data(), + mpix_alltoall.data(), s, MPI_INT, - locality_comm); + xcomm); for (int j = 0; j < s*num_procs; j++) - ASSERT_EQ(std_alltoall[j], loc_pairwise_alltoall[j]); + ASSERT_EQ(std_alltoall[j], mpix_alltoall[j]); - /* - alltoall_bruck(local_data.data(), + alltoall_hierarchical(local_data.data(), s, MPI_INT, - bruck_alltoall.data(), + mpix_alltoall.data(), s, MPI_INT, - MPI_COMM_WORLD); + xcomm); + for (int j = 0; j < s*num_procs; j++) + ASSERT_EQ(std_alltoall[j], mpix_alltoall[j]); + + std::fill(mpix_alltoall.begin(), mpix_alltoall.end(), 0); + alltoall_multileader(local_data.data(), + s, + MPI_INT, + mpix_alltoall.data(), + s, + MPI_INT, + xcomm); + for (int j = 0; j < s*num_procs; j++) + ASSERT_EQ(std_alltoall[j], mpix_alltoall[j]); + + + std::fill(mpix_alltoall.begin(), mpix_alltoall.end(), 0); + alltoall_node_aware(local_data.data(), + s, + MPI_INT, + mpix_alltoall.data(), + s, + MPI_INT, + xcomm); + for (int j = 0; j < s*num_procs; j++) + ASSERT_EQ(std_alltoall[j], mpix_alltoall[j]); + + + std::fill(mpix_alltoall.begin(), mpix_alltoall.end(), 0); + alltoall_locality_aware(local_data.data(), + s, + MPI_INT, + mpix_alltoall.data(), + s, + MPI_INT, + xcomm); + for (int j = 0; j < s*num_procs; j++) + ASSERT_EQ(std_alltoall[j], mpix_alltoall[j]); + + + std::fill(mpix_alltoall.begin(), mpix_alltoall.end(), 0); + alltoall_multileader_locality(local_data.data(), + s, + MPI_INT, + mpix_alltoall.data(), + s, + MPI_INT, + xcomm); + for (int j = 0; j < s*num_procs; j++) + ASSERT_EQ(std_alltoall[j], mpix_alltoall[j]); + + alltoall_hierarchical_nb(local_data.data(), + s, + MPI_INT, + mpix_alltoall.data(), + s, + MPI_INT, + xcomm); for (int j = 0; j < s*num_procs; j++) - ASSERT_EQ(std_alltoall[j], bruck_alltoall[j]);*/ + ASSERT_EQ(std_alltoall[j], mpix_alltoall[j]); + + std::fill(mpix_alltoall.begin(), mpix_alltoall.end(), 0); + alltoall_multileader_nb(local_data.data(), + s, + MPI_INT, + mpix_alltoall.data(), + s, + MPI_INT, + xcomm); + for (int j = 0; j < s*num_procs; j++) + ASSERT_EQ(std_alltoall[j], mpix_alltoall[j]); + + + std::fill(mpix_alltoall.begin(), mpix_alltoall.end(), 0); + alltoall_node_aware_nb(local_data.data(), + s, + MPI_INT, + mpix_alltoall.data(), + s, + MPI_INT, + xcomm); + for (int j = 0; j < s*num_procs; j++) + ASSERT_EQ(std_alltoall[j], mpix_alltoall[j]); + + + std::fill(mpix_alltoall.begin(), mpix_alltoall.end(), 0); + alltoall_locality_aware_nb(local_data.data(), + s, + MPI_INT, + mpix_alltoall.data(), + s, + MPI_INT, + xcomm); + for (int j = 0; j < s*num_procs; j++) + ASSERT_EQ(std_alltoall[j], mpix_alltoall[j]); + + + std::fill(mpix_alltoall.begin(), mpix_alltoall.end(), 0); + alltoall_multileader_locality_nb(local_data.data(), + s, + MPI_INT, + mpix_alltoall.data(), + s, + MPI_INT, + xcomm); + for (int j = 0; j < s*num_procs; j++) + ASSERT_EQ(std_alltoall[j], mpix_alltoall[j]); + } - MPIX_Comm_free(&locality_comm); + MPIX_Comm_free(&xcomm); } diff --git a/src/locality/topology.c b/src/locality/topology.c index a619c0676..f773f8e87 100644 --- a/src/locality/topology.c +++ b/src/locality/topology.c @@ -12,6 +12,10 @@ int MPIX_Comm_init(MPIX_Comm** xcomm_ptr, MPI_Comm global_comm) xcomm->local_comm = MPI_COMM_NULL; xcomm->group_comm = MPI_COMM_NULL; + xcomm->leader_comm = MPI_COMM_NULL; + xcomm->leader_group_comm = MPI_COMM_NULL; + xcomm->leader_local_comm = MPI_COMM_NULL; + xcomm->neighbor_comm = MPI_COMM_NULL; xcomm->win = MPI_WIN_NULL; @@ -58,6 +62,36 @@ int MPIX_Comm_topo_init(MPIX_Comm* xcomm) return MPI_SUCCESS; } +int MPIX_Comm_leader_init(MPIX_Comm* xcomm, int procs_per_leader) +{ + int rank, num_procs; + MPI_Comm_rank(xcomm->global_comm, &rank); + MPI_Comm_size(xcomm->global_comm, &num_procs); + + MPI_Comm_split(xcomm->global_comm, + rank / procs_per_leader, + rank, + &(xcomm->leader_comm)); + + int leader_rank; + MPI_Comm_rank(xcomm->leader_comm, &leader_rank); + + MPI_Comm_split(xcomm->global_comm, + leader_rank, + rank, + &(xcomm->leader_group_comm)); + + if (xcomm->local_comm == MPI_COMM_NULL) + MPIX_Comm_topo_init(xcomm); + + MPI_Comm_split(xcomm->local_comm, + leader_rank, + rank, + &(xcomm->leader_local_comm)); + + return MPI_SUCCESS; +} + int MPIX_Comm_device_init(MPIX_Comm* xcomm) { #ifdef GPU @@ -117,6 +151,7 @@ int MPIX_Comm_free(MPIX_Comm** xcomm_ptr) MPI_Comm_free(&(xcomm->neighbor_comm)); MPIX_Comm_topo_free(xcomm); + MPIX_Comm_leader_free(xcomm); MPIX_Comm_win_free(xcomm); MPIX_Comm_device_free(xcomm); @@ -127,14 +162,26 @@ int MPIX_Comm_free(MPIX_Comm** xcomm_ptr) int MPIX_Comm_topo_free(MPIX_Comm* xcomm) { - if (xcomm->local_comm != MPI_COMM_NULL) - MPI_Comm_free(&(xcomm->local_comm)); - if (xcomm->group_comm != MPI_COMM_NULL) + if (xcomm->local_comm != MPI_COMM_NULL) + MPI_Comm_free(&(xcomm->local_comm)); + if (xcomm->group_comm != MPI_COMM_NULL) MPI_Comm_free(&(xcomm->group_comm)); return MPI_SUCCESS; } +int MPIX_Comm_leader_free(MPIX_Comm* xcomm) +{ + if (xcomm->leader_comm != MPI_COMM_NULL) + MPI_Comm_free(&(xcomm->leader_comm)); + if (xcomm->leader_group_comm != MPI_COMM_NULL) + MPI_Comm_free(&(xcomm->leader_group_comm)); + if (xcomm->leader_local_comm != MPI_COMM_NULL) + MPI_Comm_free(&(xcomm->leader_local_comm)); + + return MPI_SUCCESS; +} + int MPIX_Comm_win_free(MPIX_Comm* xcomm) { int rank, num_procs; diff --git a/src/locality/topology.h b/src/locality/topology.h index f3a0dbd6d..5f28aade3 100644 --- a/src/locality/topology.h +++ b/src/locality/topology.h @@ -18,10 +18,19 @@ extern "C" typedef struct _MPIX_Comm { MPI_Comm global_comm; - MPI_Comm local_comm; + + // For persistent neighborhood collectives MPI_Comm neighbor_comm; + + // For hierarchical collectives + MPI_Comm local_comm; MPI_Comm group_comm; + // For multileader hierarchical collectives + MPI_Comm leader_comm; + MPI_Comm leader_group_comm; + MPI_Comm leader_local_comm; + int num_nodes; int rank_node; int ppn; @@ -47,6 +56,9 @@ int MPIX_Comm_free(MPIX_Comm** xcomm_ptr); int MPIX_Comm_topo_init(MPIX_Comm* xcomm); int MPIX_Comm_topo_free(MPIX_Comm* xcomm); +int MPIX_Comm_leader_init(MPIX_Comm* xcomm, int procs_per_leader); +int MPIX_Comm_leader_free(MPIX_Comm* xcomm); + int MPIX_Comm_win_init(MPIX_Comm* xcomm, int bytes, int type_bytes); int MPIX_Comm_win_free(MPIX_Comm* xcomm);