mpirun: add cli option to turn off gpu support

idea is that maybe one might want to build open mpi with gpu support,
but there may be performance benefit of disabling all gpu related
paths when using this library on systems without accelerators.

Signed-off-by: Howard Pritchard <[email protected]>

Author: hppritcha

src/mca/schizo/ompi/schizo-ompi-cli.rstxt

@@ -163,6 +163,8 @@ MPI Options
   attached to predefined communicators during the first MPI call.
 * ``--memory-alloc-kinds``: Value is a comma separated list of
    memory allocation kinds.
+* ``--disable-gpu-support``: Specify to disable any accelerator support
+  built in to the Open MPI installation at run time.
 * ``--display-comm``: Display table of communication methods between
   MPI_COMM_WORLD ranks during MPI_Init
 * ``--display-comm-finalize``: Display table of communication methods

src/mca/schizo/ompi/schizo_ompi.c

@@ -19,7 +19,7 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2018-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2021-2024 Nanook Consulting  All rights reserved.
- * Copyright (c) 2022-2024 Triad National Security, LLC. All rights
+ * Copyright (c) 2022-2025 Triad National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -194,6 +194,7 @@ static struct option ompioptions[] = {
     PMIX_OPTION_DEFINE(PRTE_CLI_DISABLE_RECOVERY, PMIX_ARG_NONE),
     PMIX_OPTION_DEFINE(PRTE_CLI_CONTINUOUS, PMIX_ARG_NONE),
     PMIX_OPTION_DEFINE("with-ft", PMIX_ARG_REQD),
+    PMIX_OPTION_DEFINE(PRTE_CLI_DISABLE_GPU_SUPPORT, PMIX_ARG_NONE),
 
     /* mpiexec mandated form launch key parameters - MPI 4.0 */
     PMIX_OPTION_DEFINE("initial-errhandler", PMIX_ARG_REQD),
@@ -1612,6 +1613,15 @@ static int parse_env(char **srcenv, char ***dstenv,
         }
     }
 
+   if (NULL != (opt = pmix_cmd_line_get_param(results, "disable-gpu-support"))) {
+        rc = check_cache(&cache, &cachevals, "disable_gpu_support", "true");
+        if (PRTE_SUCCESS != rc) {
+            PMIX_ARGV_FREE_COMPAT(cache);
+            PMIX_ARGV_FREE_COMPAT(cachevals);
+            return rc;
+        }
+    }
+
     if (pmix_cmd_line_is_taken(results, "display-comm") &&
         pmix_cmd_line_is_taken(results, "display-comm-finalize")) {
         PMIX_SETENV_COMPAT("OMPI_MCA_ompi_display_comm", "mpi_init,mpi_finalize", true, dstenv);

src/mca/schizo/prte/schizo_prte.c

@@ -19,6 +19,8 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2018-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2021-2024 Nanook Consulting  All rights reserved.
+ * Copyright (c) 2025      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -197,6 +199,7 @@ static struct option prterunoptions[] = {
     PMIX_OPTION_DEFINE(PRTE_CLI_DO_NOT_AGG_HELP, PMIX_ARG_NONE),
     PMIX_OPTION_DEFINE(PRTE_CLI_FWD_ENVIRON, PMIX_ARG_OPTIONAL),
     PMIX_OPTION_DEFINE(PRTE_CLI_MEM_ALLOC_KIND, PMIX_ARG_REQD),
+    PMIX_OPTION_DEFINE(PRTE_CLI_DISABLE_GPU_SUPPORT, PMIX_ARG_NONE),
 
     // output options
     PMIX_OPTION_DEFINE(PRTE_CLI_OUTPUT, PMIX_ARG_REQD),
@@ -312,6 +315,7 @@ static struct option prunoptions[] = {
     PMIX_OPTION_DEFINE(PRTE_CLI_DO_NOT_AGG_HELP, PMIX_ARG_NONE),
     PMIX_OPTION_DEFINE(PRTE_CLI_FWD_ENVIRON, PMIX_ARG_OPTIONAL),
     PMIX_OPTION_DEFINE(PRTE_CLI_MEM_ALLOC_KIND, PMIX_ARG_REQD),
+    PMIX_OPTION_DEFINE(PRTE_CLI_DISABLE_GPU_SUPPORT, PMIX_ARG_NONE),
 
     // output options
     PMIX_OPTION_DEFINE(PRTE_CLI_OUTPUT, PMIX_ARG_REQD),

src/prted/pmix/pmix_server_queries.c

@@ -20,7 +20,7 @@
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2020      IBM Corporation.  All rights reserved.
  * Copyright (c) 2021-2024 Nanook Consulting  All rights reserved.
- * Copyright (c) 2024      Triad National Security, LLC. All rights
+ * Copyright (c) 2024-2025 Triad National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -31,6 +31,7 @@
  */
 
 #include "prte_config.h"
+#include "pmix.h"
 
 #ifdef HAVE_UNISTD_H
 #    include <unistd.h>
@@ -825,6 +826,27 @@ static void _query(int sd, short args, void *cbdata)
                     PMIX_ERROR_LOG(rc);
                     goto done;
                 }
+#endif
+#ifdef PMIX_DISABLE_GPU_SUPPORT
+            } else if (0 == strcmp(q->keys[n], PMIX_DISABLE_GPU_SUPPORT)) {
+                pmix_value_t *value;
+                jdata = prte_get_job_data_object(jobid);
+                if (NULL == jdata) {
+                    ret = PMIX_ERR_NOT_FOUND;
+                    goto done;
+                }
+                PMIX_LOAD_PROCID(&pproc, jobid, PMIX_RANK_WILDCARD);
+                PMIX_INFO_LOAD(&info, PMIX_IMMEDIATE, NULL, PMIX_BOOL);
+                ret = PMIx_Get(&pproc, PMIX_DISABLE_GPU_SUPPORT, &info, 1, &value);
+                if (PMIX_SUCCESS != ret) {
+                    goto done;
+                }
+                PMIX_INFO_LIST_ADD(rc, results, PMIX_DISABLE_GPU, value->data.string, PMIX_STRING);
+                PMIX_VALUE_RELEASE(value);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    goto done;
+                }
 #endif
             } else {
                 fprintf(stderr, "Query for unrecognized attribute: %s\n", q->keys[n]);

src/prted/prun_common.c

@@ -667,6 +667,12 @@ int prun_common(pmix_cli_result_t *results,
         PMIX_INFO_LIST_ADD(ret, jinfo, PMIX_MEM_ALLOC_KIND, opt->values[0], PMIX_STRING);
     }
 #endif
+#ifdef PMIX_DISABLE_GPU_SUPPORT
+    /* mark if gpu support is disabled on the cmd line */
+    if (pmix_cmd_line_is_taken(results, PRTE_CLI_DISABLE_GPU_SUPPORT)) {
+        PMIX_INFO_LIST_ADD(ret, jinfo, PMIX_DISABLE_GPU_SUPPORT, &flag, PMIX_BOOL);
+    }
+#endif
 
     /* give the schizo components a chance to add to the job info */
     schizo->job_info(results, jinfo);

src/util/prte_cmd_line.h

@@ -108,7 +108,8 @@ BEGIN_C_DECLS
 #define PRTE_CLI_SET_CWD_SESSION        "set-cwd-to-session-dir"    // none
 #define PRTE_CLI_ENABLE_RECOVERY        "enable-recovery"           // none
 #define PRTE_CLI_DISABLE_RECOVERY       "disable-recovery"          // none
-#define PRTE_CLI_MEM_ALLOC_KIND			"memory-alloc-kinds"        // required
+#define PRTE_CLI_MEM_ALLOC_KIND		"memory-alloc-kinds"        // required
+#define PRTE_CLI_DISABLE_GPU_SUPPORT	"disable-gpu-support"       // none
 
 // Placement options
 #define PRTE_CLI_MAPBY                  "map-by"                    // required