Enable the PMIx notification callback system. This currently is only supported by the pmix120 component, which is not selected by default. All other components will ignore error registration requests, and thus do not support debugger attach when launched via mpirun. Note that direct launched applications will support such attachment, but may not do so in a scalable fashion.

Fixes ##1225

Author: Ralph Castain

ompi/mca/rte/orte/rte_orte.h

@@ -91,7 +91,7 @@ OMPI_DECLSPEC void __opal_attribute_noreturn__
 #define OMPI_ERROR_LOG ORTE_ERROR_LOG
 
 /* Init and finalize objects and operations */
-#define ompi_rte_init(a, b) orte_init(a, b, ORTE_PROC_MPI)
+OMPI_DECLSPEC int ompi_rte_init(int *pargc, char ***pargv);
 #define ompi_rte_finalize() orte_finalize()
 OMPI_DECLSPEC void ompi_rte_wait_for_debugger(void);
 

ompi/mca/rte/orte/rte_orte_module.c

@@ -52,6 +52,79 @@
 
 extern ompi_rte_orte_component_t mca_rte_orte_component;
 
+typedef struct {
+    volatile bool active;
+    int status;
+    int errhandler;
+} errhandler_t;
+
+static void register_cbfunc(int status, int errhndler, void *cbdata)
+{
+    errhandler_t *cd = (errhandler_t*)cbdata;
+    cd->status = status;
+    cd->errhandler = errhndler;
+    cd->active = false;
+}
+
+static volatile bool wait_for_release = true;
+static int errhandler = -1;
+
+static void notify_cbfunc(int status,
+                          opal_list_t *procs,
+                          opal_list_t *info,
+                          opal_pmix_release_cbfunc_t cbfunc,
+                          void *cbdata)
+{
+    if (NULL != cbfunc) {
+        cbfunc(cbdata);
+    }
+    wait_for_release = false;
+}
+
+
+int ompi_rte_init(int *pargc, char ***pargv)
+{
+    int rc;
+    opal_list_t info;
+    opal_value_t val;
+    errhandler_t cd;
+
+    if (ORTE_SUCCESS != (rc = orte_init(pargc, pargv, ORTE_PROC_MPI))) {
+        return rc;
+    }
+
+    if (!orte_standalone_operation) {
+        /* register to receive any debugger release */
+        OBJ_CONSTRUCT(&info, opal_list_t);
+        OBJ_CONSTRUCT(&val, opal_value_t);
+        val.key = strdup(OPAL_PMIX_ERROR_NAME);
+        val.type = OPAL_INT;
+        val.data.integer = OPAL_ERR_DEBUGGER_RELEASE;
+        opal_list_append(&info, &val.super);
+        cd.status = ORTE_ERROR;
+        cd.errhandler = -1;
+        cd.active = true;
+
+        opal_pmix.register_errhandler(&info, notify_cbfunc, register_cbfunc, &cd);
+
+        /* let the MPI progress engine run while we wait for
+         * registration to complete */
+        OMPI_WAIT_FOR_COMPLETION(cd.active);
+        /* safely deconstruct the list */
+        opal_list_remove_first(&info);
+        OBJ_DESTRUCT(&val);
+        OBJ_DESTRUCT(&info);
+        if (OPAL_SUCCESS != cd.status) {
+            /* ouch - we are doomed */
+            ORTE_ERROR_LOG(cd.status);
+            return OMPI_ERROR;
+        }
+        errhandler = cd.errhandler;
+    }
+
+    return OMPI_SUCCESS;
+}
+
 void ompi_rte_abort(int error_code, char *fmt, ...)
 {
     va_list arglist;
@@ -100,10 +173,10 @@ void ompi_rte_abort(int error_code, char *fmt, ...)
  * attaching debuggers -- see big comment in
  * orte/tools/orterun/debuggers.c explaining the two scenarios.
  */
+
 void ompi_rte_wait_for_debugger(void)
 {
     int debugger;
-    orte_rml_recv_cb_t xfer;
 
     /* See lengthy comment in orte/tools/orterun/debuggers.c about
        orte_in_parallel_debugger */
@@ -117,12 +190,12 @@ void ompi_rte_wait_for_debugger(void)
         /* if not, just return */
         return;
     }
-
     /* if we are being debugged, then we need to find
      * the correct plug-ins
      */
     ompi_debugger_setup_dlls();
 
+    /* wait for the debugger to attach */
     if (orte_standalone_operation) {
         /* spin until debugger attaches and releases us */
         while (MPIR_debug_gate == 0) {
@@ -133,23 +206,9 @@ void ompi_rte_wait_for_debugger(void)
 #endif
         }
     } else {
-        /* only the rank=0 proc waits for either a message from the
-         * HNP or for the debugger to attach - everyone else will just
-         * spin in * the grpcomm barrier in ompi_mpi_init until rank=0
-         * joins them.
-         */
-        if (0 != ORTE_PROC_MY_NAME->vpid) {
-            return;
-        }
-
-        /* VPID 0 waits for a message from the HNP */
-        OBJ_CONSTRUCT(&xfer, orte_rml_recv_cb_t);
-        xfer.active = true;
-        orte_rml.recv_buffer_nb(OMPI_NAME_WILDCARD,
-                                ORTE_RML_TAG_DEBUGGER_RELEASE,
-                                ORTE_RML_NON_PERSISTENT,
-                                orte_rml_recv_callback, &xfer);
-        /* let the MPI progress engine run while we wait */
-        OMPI_WAIT_FOR_COMPLETION(xfer.active);
+        /* now wait for the notification to occur */
+        OMPI_WAIT_FOR_COMPLETION(wait_for_release);
+        /* deregister the errhandler */
+        opal_pmix.deregister_errhandler(errhandler, NULL, NULL);
     }
 }

opal/dss/dss_compare.c

@@ -10,9 +10,9 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -449,3 +449,12 @@ int opal_dss_compare_jobid(opal_jobid_t *value1,
     return OPAL_EQUAL;
 }
 
+int opal_dss_compare_status(int *value1, int *value2, opal_data_type_t type)
+{
+    if (*value1 > *value2) return OPAL_VALUE1_GREATER;
+
+    if (*value2 > *value1) return OPAL_VALUE2_GREATER;
+
+    return OPAL_EQUAL;
+}
+

opal/dss/dss_copy.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -61,6 +61,7 @@ int opal_dss_std_copy(void **dest, void *src, opal_data_type_t type)
 
     case OPAL_INT:
     case OPAL_UINT:
+    case OPAL_STATUS:
         datasize = sizeof(int);
         break;
 

opal/dss/dss_internal.h

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
@@ -332,6 +332,9 @@ int opal_dss_pack_jobid(opal_buffer_t *buffer, const void *src,
 int opal_dss_pack_vpid(opal_buffer_t *buffer, const void *src,
                       int32_t num_vals, opal_data_type_t type);
 
+int opal_dss_pack_status(opal_buffer_t *buffer, const void *src,
+                         int32_t num_vals, opal_data_type_t type);
+
 /*
  * Internal unpack functions
  */
@@ -401,6 +404,8 @@ int opal_dss_unpack_jobid(opal_buffer_t *buffer, void *dest,
 int opal_dss_unpack_vpid(opal_buffer_t *buffer, void *dest,
                         int32_t *num_vals, opal_data_type_t type);
 
+int opal_dss_unpack_status(opal_buffer_t *buffer, void *dest,
+                           int32_t *num_vals, opal_data_type_t type);
 
 /*
  * Internal copy functions
@@ -497,6 +502,8 @@ int opal_dss_compare_jobid(opal_jobid_t *value1,
                            opal_jobid_t *value2,
                            opal_data_type_t type);
 
+int opal_dss_compare_status(int *value1, int *value2, opal_data_type_t type);
+
 /*
  * Internal print functions
  */
@@ -536,6 +543,7 @@ int opal_dss_print_time(char **output, char *prefix, time_t *src, opal_data_type
 int opal_dss_print_name(char **output, char *prefix, opal_process_name_t *name, opal_data_type_t type);
 int opal_dss_print_jobid(char **output, char *prefix, opal_process_name_t *src, opal_data_type_t type);
 int opal_dss_print_vpid(char **output, char *prefix, opal_process_name_t *src, opal_data_type_t type);
+int opal_dss_print_status(char **output, char *prefix, int *src, opal_data_type_t type);
 
 
 /*

opal/dss/dss_open_close.c

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -611,6 +611,17 @@ int opal_dss_open(void)
         return rc;
     }
 
+
+    tmp = OPAL_STATUS;
+    if (OPAL_SUCCESS != (rc = opal_dss.register_type(opal_dss_pack_status,
+                                          opal_dss_unpack_status,
+                                          (opal_dss_copy_fn_t)opal_dss_std_copy,
+                                          (opal_dss_compare_fn_t)opal_dss_compare_status,
+                                          (opal_dss_print_fn_t)opal_dss_print_status,
+                                          OPAL_DSS_UNSTRUCTURED,
+                                          "OPAL_STATUS", &tmp))) {
+        return rc;
+    }
     /* All done */
 
     opal_dss_initialized = true;

opal/dss/dss_pack.c

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -1240,3 +1240,20 @@ int opal_dss_pack_vpid(opal_buffer_t *buffer, const void *src,
     return ret;
 }
 
+/*
+ * STATUS
+ */
+int opal_dss_pack_status(opal_buffer_t *buffer, const void *src,
+                         int32_t num_vals, opal_data_type_t type)
+{
+    int ret;
+
+    /* Turn around and pack the real type */
+    ret = opal_dss_pack_buffer(buffer, src, num_vals, OPAL_INT);
+    if (OPAL_SUCCESS != ret) {
+        OPAL_ERROR_LOG(ret);
+    }
+
+    return ret;
+}
+

opal/dss/dss_print.c

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -25,6 +25,7 @@
 #include "opal_stdint.h"
 #include <stdio.h>
 
+#include "opal/util/error.h"
 #include "opal/dss/dss_internal.h"
 
 int opal_dss_print(char **output, char *prefix, void *src, opal_data_type_t type)
@@ -1060,3 +1061,29 @@ int opal_dss_print_vpid(char **output, char *prefix,
 
     return OPAL_SUCCESS;
 }
+
+int opal_dss_print_status(char **output, char *prefix,
+                          int *src, opal_data_type_t type)
+{
+    char *prefx;
+
+    /* deal with NULL prefix */
+    if (NULL == prefix) asprintf(&prefx, " ");
+    else prefx = prefix;
+
+    /* if src is NULL, just print data type and return */
+    if (NULL == src) {
+        asprintf(output, "%sData type: OPAL_STATUS\tValue: NULL pointer", prefx);
+        if (prefx != prefix) {
+            free(prefx);
+        }
+        return OPAL_SUCCESS;
+    }
+
+    asprintf(output, "%sData type: OPAL_STATUS\tValue: %s", prefx, opal_strerror(*src));
+    if (prefx != prefix) {
+        free(prefx);
+    }
+
+    return OPAL_SUCCESS;
+}

opal/dss/dss_types.h

@@ -13,9 +13,9 @@
  * Copyright (c) 2007-2011 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights
  *                         reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -120,6 +120,8 @@ typedef struct {
 #define    OPAL_NAME                (opal_data_type_t)   50
 #define    OPAL_JOBID               (opal_data_type_t)   51
 #define    OPAL_VPID                (opal_data_type_t)   52
+#define    OPAL_STATUS              (opal_data_type_t)   53
+
     /* OPAL Dynamic */
 #define    OPAL_DSS_ID_DYNAMIC      (opal_data_type_t)  100
 
@@ -245,6 +247,7 @@ typedef struct {
         float fval;
         double dval;
         struct timeval tv;
+        int status;
         opal_process_name_t name;
         opal_bool_array_t flag_array;
         opal_uint8_array_t byte_array;

opal/dss/dss_unpack.c

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012-2015 Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -1519,3 +1519,20 @@ int opal_dss_unpack_vpid(opal_buffer_t *buffer, void *dest,
 
     return ret;
 }
+
+/*
+ * STATUS
+ */
+int opal_dss_unpack_status(opal_buffer_t *buffer, void *dest,
+                           int32_t *num_vals, opal_data_type_t type)
+{
+    int ret;
+
+    /* Turn around and unpack the real type */
+    ret = opal_dss_unpack_buffer(buffer, dest, num_vals, OPAL_INT);
+    if (OPAL_SUCCESS != ret) {
+        OPAL_ERROR_LOG(ret);
+    }
+
+    return ret;
+}