Fix non-MPI applications not being able to run

gvallee · gvallee · commit e59d994c6db8 · 2019-12-17T11:36:18.000-05:00
Fix non-MPI applications not being able to run locally. We know cleanly deal with MPI vs. non-MPI configurations. This is only using the native launcher (no integration with any job manager, for which it was not tested at all)

Signed-off-by: Geoffroy Vallee &lt;geoffroy.vallee@gmail.com&gt;
diff --git a/pkg/container/container.go b/pkg/container/container.go
@@ -384,10 +384,13 @@ func GetMetadata(imgPath string, sysCfg *sys.Config) (Config, implem.Info, error
 }
 
 func getDefaultExecArgs() []string {
-	return strings.Split(defaultExecArgs, " ")
+	args := []string{"exec"}
+	args = append(args, strings.Split(defaultExecArgs, " ")...)
+
+	return args
 }
 
-func getBindArguments(hostMPI *implem.Info, hostBuildenv *buildenv.Info, c *Config) []string {
+func getMPIBindArguments(hostMPI *implem.Info, hostBuildenv *buildenv.Info, c *Config) []string {
 	var bindArgs []string
 
 	if c.Model == BindModel {
@@ -401,20 +404,24 @@ func getBindArguments(hostMPI *implem.Info, hostBuildenv *buildenv.Info, c *Conf
 	return bindArgs
 }
 
-// GetExecArgs figures out the singularity exec arguments to be used for executing a container
-func GetExecArgs(myHostMPICfg *implem.Info, hostBuildEnv *buildenv.Info, syContainer *Config, sysCfg *sys.Config) []string {
+// GetMPIExecCfg figures out the singularity exec arguments to be used for executing a container
+func GetMPIExecCfg(myHostMPICfg *implem.Info, hostBuildEnv *buildenv.Info, syContainer *Config, sysCfg *sys.Config) []string {
 	args := getDefaultExecArgs()
 	if sysCfg.Nopriv {
 		args = append(args, "-u")
 	}
-
-	bindArgs := getBindArguments(myHostMPICfg, hostBuildEnv, syContainer)
+	bindArgs := getMPIBindArguments(myHostMPICfg, hostBuildEnv, syContainer)
 	if len(bindArgs) > 0 {
 		args = append(args, "--bind")
 		args = append(args, bindArgs...)
 	}
-
 	log.Printf("-> Exec args to use: %s\n", strings.Join(args, " "))
+	return args
+}
 
+// GetDefaultExecCfg returns the default way to run a container
+func GetDefaultExecCfg() []string {
+	args := getDefaultExecArgs()
+	log.Printf("-> Exec args to use: %s\n", strings.Join(args, " "))
 	return args
 }
diff --git a/pkg/implem/implem.go b/pkg/implem/implem.go
@@ -33,3 +33,12 @@ type Info struct {
 	// Tarball is the name of the tarball of the MPI implementation
 	Tarball string
 }
+
+// IsMPI checks if information passed in is an MPI implementation
+func IsMPI(i *Info) bool {
+	if i != nil && (i.ID == OMPI || i.ID == MPICH || i.ID == IMPI) {
+		return true
+	}
+
+	return false
+}
diff --git a/pkg/jm/jobmgr_native.go b/pkg/jm/jobmgr_native.go
@@ -15,6 +15,7 @@ import (
 	"github.com/sylabs/singularity-mpi/internal/pkg/impi"
 	"github.com/sylabs/singularity-mpi/internal/pkg/job"
 	"github.com/sylabs/singularity-mpi/pkg/buildenv"
+	"github.com/sylabs/singularity-mpi/pkg/container"
 	"github.com/sylabs/singularity-mpi/pkg/implem"
 	"github.com/sylabs/singularity-mpi/pkg/mpi"
 	"github.com/sylabs/singularity-mpi/pkg/syexec"
@@ -63,18 +64,11 @@ func NativeGetError(j *job.Job, sysCfg *sys.Config) string {
 	return j.ErrBuffer.String()
 }
 
-// NativeSubmit is the function to call to submit a job through the native job manager
-func NativeSubmit(j *job.Job, env *buildenv.Info, sysCfg *sys.Config) (syexec.SyCmd, error) {
-	var sycmd syexec.SyCmd
-
-	if j.App.BinPath == "" {
-		return sycmd, fmt.Errorf("application binary is undefined")
-	}
-
+func prepareMPISubmit(sycmd *syexec.SyCmd, j *job.Job, env *buildenv.Info, sysCfg *sys.Config) error {
 	var err error
 	sycmd.BinPath, err = mpi.GetPathToMpirun(j.HostCfg, env)
 	if err != nil {
-		return sycmd, err
+		return err
 	}
 	if j.NP > 0 {
 		sycmd.CmdArgs = append(sycmd.CmdArgs, "-np")
@@ -83,7 +77,7 @@ func NativeSubmit(j *job.Job, env *buildenv.Info, sysCfg *sys.Config) (syexec.Sy
 
 	mpirunArgs, err := mpi.GetMpirunArgs(j.HostCfg, env, &j.App, j.Container, sysCfg)
 	if err != nil {
-		return sycmd, fmt.Errorf("unable to get mpirun arguments: %s", err)
+		return fmt.Errorf("unable to get mpirun arguments: %s", err)
 	}
 	if len(mpirunArgs) > 0 {
 		sycmd.CmdArgs = append(sycmd.CmdArgs, mpirunArgs...)
@@ -98,6 +92,37 @@ func NativeSubmit(j *job.Job, env *buildenv.Info, sysCfg *sys.Config) (syexec.Sy
 	sycmd.Env = append([]string{"LD_LIBRARY_PATH=" + newLDPath}, os.Environ()...)
 	sycmd.Env = append([]string{"PATH=" + newPath}, os.Environ()...)
 
+	return nil
+}
+
+func prepareStdSubmit(sycmd *syexec.SyCmd, j *job.Job, env *buildenv.Info, sysCfg *sys.Config) error {
+	sycmd.BinPath = sysCfg.SingularityBin
+	sycmd.CmdArgs = container.GetDefaultExecCfg()
+	sycmd.CmdArgs = append(sycmd.CmdArgs, j.Container.Path, j.App.BinPath)
+
+	return nil
+}
+
+// NativeSubmit is the function to call to submit a job through the native job manager
+func NativeSubmit(j *job.Job, env *buildenv.Info, sysCfg *sys.Config) (syexec.SyCmd, error) {
+	var sycmd syexec.SyCmd
+
+	if j.App.BinPath == "" {
+		return sycmd, fmt.Errorf("application binary is undefined")
+	}
+
+	if implem.IsMPI(j.HostCfg) {
+		err := prepareMPISubmit(&sycmd, j, env, sysCfg)
+		if err != nil {
+			return sycmd, fmt.Errorf("unable to prepare MPI job: %s", err)
+		}
+	} else {
+		err := prepareStdSubmit(&sycmd, j, env, sysCfg)
+		if err != nil {
+			return sycmd, fmt.Errorf("unable to prepare MPI job: %s", err)
+		}
+	}
+
 	j.GetOutput = NativeGetOutput
 	j.GetError = NativeGetError
 
diff --git a/pkg/jm/jobmgr_prun.go b/pkg/jm/jobmgr_prun.go
@@ -68,7 +68,7 @@ func PrunSubmit(j *job.Job, env *buildenv.Info, sysCfg *sys.Config) (syexec.SyCm
 	sycmd.CmdArgs = append(sycmd.CmdArgs, j.Container.AppExe)
 
 	// Get the exec arguments and set the env var
-	execArgs := container.GetExecArgs(j.HostCfg, env, j.Container, sysCfg)
+	execArgs := container.GetMPIExecCfg(j.HostCfg, env, j.Container, sysCfg)
 	syExecArgsEnv := "SY_EXEC_ARGS=\"" + strings.Join(execArgs, " ") + "\""
 	log.Printf("Command to be executed: %s %s", sycmd.BinPath, strings.Join(sycmd.CmdArgs, " "))
 	log.Printf("SY_EXEC_ARGS to be used: %s", strings.Join(execArgs, " "))
diff --git a/pkg/launcher/launcher.go b/pkg/launcher/launcher.go
@@ -44,6 +44,11 @@ type Info struct {
 func prepareLaunchCmd(j *job.Job, jobmgr *jm.JM, hostEnv *buildenv.Info, sysCfg *sys.Config) (syexec.SyCmd, error) {
 	var cmd syexec.SyCmd
 
+	// Sanity checks
+	if j == nil || jobmgr == nil || hostEnv == nil || sysCfg == nil {
+		return cmd, fmt.Errorf("invalid parameter(s)")
+	}
+
 	launchCmd, err := jobmgr.Submit(j, hostEnv, sysCfg)
 	if err != nil {
 		return cmd, fmt.Errorf("failed to create a launcher object: %s", err)
diff --git a/pkg/mpi/mpi.go b/pkg/mpi/mpi.go
@@ -6,6 +6,7 @@
 package mpi
 
 import (
+	"fmt"
 	"log"
 	"path/filepath"
 
@@ -35,6 +36,11 @@ type Config struct {
 
 // GetPathToMpirun returns the path to mpirun based a configuration of MPI
 func GetPathToMpirun(mpiCfg *implem.Info, env *buildenv.Info) (string, error) {
+	// Sanity checks
+	if mpiCfg == nil || env == nil {
+		return "", fmt.Errorf("invalid parameter(s)")
+	}
+
 	path := filepath.Join(env.InstallDir, "bin", "mpirun")
 	// Intel MPI is installing the binaries and libraries in a quite complex setup
 	if mpiCfg.ID == implem.IMPI {
@@ -54,10 +60,10 @@ func GetPathToMpirun(mpiCfg *implem.Info, env *buildenv.Info) (string, error) {
 
 // GetMpirunArgs returns the arguments required by a mpirun
 func GetMpirunArgs(myHostMPICfg *implem.Info, hostBuildEnv *buildenv.Info, app *app.Info, syContainer *container.Config, sysCfg *sys.Config) ([]string, error) {
-	args := []string{"singularity", "exec"}
-	args = append(args, container.GetExecArgs(myHostMPICfg, hostBuildEnv, syContainer, sysCfg)...)
-	args = append(args, syContainer.Path, app.BinPath)
 	var extraArgs []string
+	args := []string{"singularity"}
+	args = append(args, container.GetMPIExecCfg(myHostMPICfg, hostBuildEnv, syContainer, sysCfg)...)
+	args = append(args, syContainer.Path, app.BinPath)
 
 	// We really do not want to do this but MPICH is being picky about args so for now, it will do the job.
 	switch myHostMPICfg.ID {
