Mpas testing

.ci/env/derecho.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+echo "Setting up derecho environment"
+workingDirectory=$PWD
+. /etc/profile.d/z00_modules.sh
+echo "Loading modules : $*"
+cmd="module --force purge"
+echo $cmd && eval "${cmd}"
+
+# We should be handed in the modules to load
+while [ $# -gt 0 ]; do 
+  cmd="module load $1"
+  echo $cmd && eval "${cmd}"
+  shift
+done
+
+#  Go back to working directory if for unknown reason HPC config changing your directory on you
+if [ "$workingDirectory" != "$PWD" ]; then
+  echo "derecho module loading changed working directory"
+  echo "  Moving back to $workingDirectory"
+  cd $workingDirectory
+fi

.ci/env/eris.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+echo "Setting up ERIS environment"
+workingDirectory=$PWD
+
+ulimit -s unlimited
+
+source /nfs/coe-sw/lmod/8.7/init/bash
+
+#module use /home/agopal/software/modules
+# Overwriting the system value of MODULEPATH to include our own modules
+export MODULEPATH=/home/agopal/software/modules/compilers:/home/agopal/software/modules/cdep
+
+echo "Loading modules : $*"
+cmd="module --force purge"
+echo $cmd && eval "${cmd}"
+
+# We should be handed in the modules to load
+while [ $# -gt 0 ]; do 
+  cmd="module load $1"
+  echo $cmd && eval "${cmd}"
+  shift
+done
+
+#  Go back to working directory if for unknown reason HPC config changing your directory on you
+if [ "$workingDirectory" != "$PWD" ]; then
+  echo "Eris module loading changed working directory"
+  echo "  Moving back to $workingDirectory"
+  cd $workingDirectory
+fi

.ci/env/helpers.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Useful string manipulation functions, leaving in for posterity
+# https://stackoverflow.com/a/8811800
+# contains(string, substring)
+#
+# Returns 0 if the specified string contains the specified substring,
+# otherwise returns 1.
+contains()
+{
+  string="$1"
+  substring="$2"
+
+  if [ "${string#*"$substring"}" != "$string" ]; then
+    echo 0    # $substring is in $string
+  else
+    echo 1    # $substring is not in $string
+  fi
+}
+
+setenvStr()
+{
+  # Changing IFS produces the most consistent results
+  tmpIFS=$IFS
+  IFS=","
+  string="$1"
+  for s in $string; do
+    if [ ! -z $s ]; then 
+      eval "echo export \"$s\""
+      eval "export \"$s\""
+    fi
+  done
+  IFS=$tmpIFS
+}
+
+banner()
+{
+  lengthBanner=$1
+  shift
+  # https://www.shellscript.sh/examples/banner/
+  printf "#%${lengthBanner}s#\n" | tr " " "="
+  printf "# %-$(( ${lengthBanner} - 2 ))s #\n" "`date`"
+  printf "# %-$(( ${lengthBanner} - 2 ))s #\n" " "
+  printf "# %-$(( ${lengthBanner} - 2 ))s #\n" "$*"
+  printf "#%${lengthBanner}s#\n" | tr " " "="
+}

.ci/env/hostenv.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Allow selection of hostname, and if none is provided use the current machine
+# While this may seem unintuitive at first, it provides the flexibility of using
+# "named" configurations without being explicitly tied to fqdn
+hostname=$AS_HOST
+if [ -z "$hostname" ]; then
+  hostname=$( python3 -c "import socket; print( socket.getfqdn() )" )
+fi
+
+if [ $( contains ${hostname} hsn.de.hpc ) -eq 0 ]; then
+  # Derecho HPC SuSE PBS
+  . .ci/env/derecho.sh
+elif [ $( contains ${hostname} eris ) -eq 0 ]; then
+  # Derecho HPC SuSE PBS
+  . .ci/env/eris.sh
+else
+  echo "No known environment for '${hostname}', using current"
+fi

.ci/mpas_compilation.jsonc

@@ -0,0 +1,48 @@
+// MPAS Compilation tests
+{
+  "submit_options" :
+  {
+    // Default values
+    "timelimit" : "00:15:00",
+    "working_directory" : "..",
+    "arguments" :
+    {
+      "base_env_numprocs"      : [ "-e", "NUM_PROCS=4" ],
+
+      ".*make.*::args_build_core"     : [ "-c", "atmosphere" ],
+      ".*debug.*::args_build_debug"     : [ "-d" ],
+      ".*make.*::args_build_options"     : [ "-b", "-j $NUM_PROCS" ],
+      ".*make.*gnu.*::args_target"     : [ "-t", "gnu" ]
+    },
+    // Derecho-specifics
+    "hsn.de.hpc"  :
+    {
+      "submission" : "PBS",
+      "queue"      : "main",
+      "hpc_arguments"  : 
+      {
+        "node_select" : { "-l " : { "select"       : 1, "ncpus" : 16 } },
+        "priority"    : { "-l " : { "job_priority" : "economy"       } }
+      },
+      "arguments"  : 
+      {
+        // We want NUM_PROCS to match ncpus
+        "base_env_numprocs"      : [ "-e", "NUM_PROCS=16" ],
+        "very_last_modules"       : [  "cray-mpich", "parallel-netcdf" ],
+        ".*gnu.*::test_modules"   : [ "gcc/12.2.0" ]
+      }
+    }
+  },
+  // Specific GNU compilation test
+  "make-gnu" :
+  {
+    "steps" :
+    {
+      // Eventually include debug and other compile modes
+      "optimized" :
+      {
+        "command"      : ".ci/tests/build.sh"
+      }
+    }
+  }
+}

.ci/mpas_refgen.jsonc

@@ -0,0 +1,179 @@
+// MPAS Compilation tests
+{
+  "submit_options" :
+  {
+    // Default values
+    "timelimit" : "00:15:00",
+    "working_directory" : "..",
+    "arguments" :
+    {
+      "base_env_numprocs"      : [ "-e", "NUM_PROCS=4" ],
+
+      ".*make.*::args_build_core"     : [ "-c", "atmosphere" ],
+      ".*debug.*::args_build_debug"     : [ "-d" ],
+      ".*make.*double.*::args_build_precision"     : [ "-p" ],
+      //".*make.*gpu.*::args_build_options_0"     : [ "-g" ],
+      //".*make.*cpu.*::args_build_options"     : [ "-b", "-j $NUM_PROCS" ],
+      ".*make.*::args_build_options"     : [ "-b", "-j $NUM_PROCS"],
+      ".*gnu.*::args_target"     : [ "-t", "gnu" ],
+      ".*make.*nvhpc.*::args_target"     : [ "-t", "nvhpc" ],
+      ".*make.*nvhpc.*gpu.*::args_openacc"     : [ "-g" ],
+      ".*test_.*::args_namelist" : [ "-b", "atmosphere_model"],
+      ".*test_jw.*::args_namelist" : [ "-f", "/glade/campaign/mmm/wmr/mpas_ci/jw_baroclinic"],
+      ".*test_conus.*::args_namelist" : [ "-f", "/glade/campaign/mmm/wmr/mpas_ci/conus_lam"],
+      ".*test_.*cpu.*::args_namelist" :
+        [ 
+          "-p", "mpiexec -n $CI_NTASKS -ppn $CI_TASKS_PER_NODE"
+        ],
+      ".*test_.*gpu.*::args_namelist" :
+        [ 
+          "-p", "mpiexec -n $CI_NTASKS -ppn $CI_TASKS_PER_NODE set_gpu_rank "
+        ]
+    },
+    // Derecho-specifics
+    "hsn.de.hpc"  :
+    {
+      "submission" : "PBS",
+      "queue"      : "develop",
+      "-m" : "n",
+      "hpc_arguments"  : 
+      {
+        ".*make.*::node_select" : 
+        { 
+          "-l " : { "select"       : 1, "ncpus" : 64, "mpiprocs" : 16  }
+        },
+        ".*test_jw.*(refgen|restart).*cpu.*::node_select" : 
+        { 
+          "-l " : { "select"       : 1, "ncpus" : 1  }
+        },
+        ".*test_conus.*refgen.*cpu.*::node_select" : 
+        { 
+          "-l " : { "select"       : 1, "ncpus" : 64, "mpiprocs" : 1   }
+        },
+        ".*test_jw.*parallel.*cpu.*::node_select" :
+        {
+           "-l " : { "select"       : 1, "ncpus" : 16, "mpiprocs" : 4 } 
+        },
+        ".*test_jw.*perf.*cpu.*::node_select" :
+        {
+           "-l " : { "select"       : 1, "ncpus" : 16, "mpiprocs" : 4 }
+        },
+        ".*test_jw.*(refgen|restart).*gpu.*::node_select" :
+        {
+           "-l " : { "select": 1, "ncpus": 16, "mpiprocs":1, "ngpus":1 } 
+        },
+        ".*test_jw.*(multigpu|perf).*gpu.*::node_select" :
+        {
+           "-l " : { "select": 1, "ncpus": 16, "mpiprocs":2, "ngpus":2 } 
+        },
+        ".*test_conus.*(refgen|restart).*gpu.*::node_select" :
+        {
+           "-l " : { "select": 1, "ncpus": 16, "mpiprocs":2, "ngpus":2 } 
+        },
+        "priority"    : { "-l " : { "job_priority" : "economy"       } },
+        "mail_events" : { "-m " : { "n" : ""}}
+      },
+      "arguments"  : 
+      {
+        // We want NUM_PROCS to match ncpus
+        //"base_env_numprocs"      : [ "-e", "NUM_PROCS=16, LD_LIBRARY_PATH=/glade/u/apps/derecho/23.09/spack/opt/spack/parallel-netcdf/1.12.3/cray-mpich/8.1.27/gcc/12.2.0/sq5u/lib:/glade/u/apps/derecho/23.09/spack/opt/spack/netcdf/4.9.2/gcc/12.2.0/gjc6/lib:\\\\$LD_LIBRARY_PATH" ],
+        "base_env_numprocs"      : [ "-e", "NUM_PROCS=16" ],
+        "global_modules" : [ "ncarenv/23.09"],
+        ".*gnu.*::test_modules"   : [ "gcc/12.2.0","cray-mpich", "parallel-netcdf", "netcdf" ],
+        ".*nvhpc.*::test_modules"   : [ "nvhpc/24.3","cray-mpich","ncarcompilers", "parallel-netcdf", "cuda" ]
+
+      }
+    }
+  },
+
+  "gnu" :
+    {
+      "submit_options" :
+      {
+        "arguments" :
+        {
+          // Remove auto-filled dir and supply sub-testcase specific ones
+          ".*test_.*::args_namelist_dir" : ["-e", "LD_LIBRARY_PATH=/glade/u/apps/derecho/23.09/spack/opt/spack/parallel-netcdf/1.12.3/cray-mpich/8.1.27/gcc/12.2.0/sq5u/lib:/glade/u/apps/derecho/23.09/spack/opt/spack/netcdf/4.9.2/gcc/12.2.0/gjc6/lib:\\\\$LD_LIBRARY_PATH"]
+        }
+      },
+      "steps" :
+      {
+        // "make-release" :
+        // {
+        //   "command"      : ".ci/tests/build.sh"
+        // },
+        // "test_jw-refgen-cpu" :
+        // {
+        //   "submit_options" : { "timelimit" : "00:15:00" },
+        //   "command"      : ".ci/tests/runTestCases.sh",
+        //   "arguments" : ["-d", "cpu", "-t", "gnu", "-c", "jw", "-g", "refgen"],
+        //   "dependencies" : { "make-release" : "afterok" }
+        // },
+        "test_conus-refgen-cpu" :
+        {
+          "submit_options" : { "timelimit" : "00:15:00" },
+          "command"      : ".ci/tests/runTestCases.sh",
+          "arguments" : ["-d", "gpu", "-t", "nvhpc", "-c", "conus", "-g", "refgen"]
+          //"dependencies" : { "make-release" : "afterok" }
+        }
+      }
+    },
+    "gnu-double" :
+    {
+      "submit_options" :
+      {
+        "arguments" :
+        {
+          ".*test_.*::args_namelist_dir" : ["-e", "LD_LIBRARY_PATH=/glade/u/apps/derecho/23.09/spack/opt/spack/parallel-netcdf/1.12.3/cray-mpich/8.1.27/gcc/12.2.0/sq5u/lib:/glade/u/apps/derecho/23.09/spack/opt/spack/netcdf/4.9.2/gcc/12.2.0/gjc6/lib:\\\\$LD_LIBRARY_PATH" ]
+        }
+      },
+      "steps" :
+      {
+        "make-release-double" :
+        {
+          "command"      : ".ci/tests/build.sh"
+        },
+        "test_jw-refgen-cpu" :
+        {
+          "submit_options" : { "timelimit" : "00:15:00" },
+          "command"      : ".ci/tests/runTestCases.sh",
+          "arguments" : ["-d", "cpu", "-t", "gnu", "-c", "jw", "-g", "refgen", "-n"],
+          "dependencies" : { "make-release-double" : "afterok" }
+        }
+      }
+    },
+    "nvhpc-cpu" :
+    {
+      "steps" :
+      {      
+        "make-nvhpc-release-cpu" :
+        {
+        "command"      : ".ci/tests/build.sh"
+        },
+        "test_jw-nvhpc-refgen-cpu" :
+        {
+         "submit_options" : { "timelimit" : "00:15:00" },
+         "command"      : ".ci/tests/runTestCases.sh",
+         "arguments" : ["-d", "cpu", "-t", "nvhpc", "-c", "jw", "-g", "refgen"],
+         "dependencies" : { "make-nvhpc-release-cpu" : "afterok" }
+        }
+      }
+    },
+    "nvhpc-gpu" :
+    {
+      "steps" :
+      {
+         "make-nvhpc-release-gpu" :
+         {
+           "command"      : ".ci/tests/build.sh"
+         },
+        "test_jw-nvhpc-refgen-gpu" :
+        {
+          "submit_options" : { "timelimit" : "00:15:00" },
+          "command"      : ".ci/tests/runTestCases.sh",
+          "arguments" : ["-d", "gpu", "-t", "nvhpc", "-c", "jw", "-g", "refgen"],
+          "dependencies" : { "make-nvhpc-release-gpu" : "afterok" }
+        }
+      }
+    }
+  }