Some more changes

abishekg7 · abishekg7 · commit 3e4801b9b8c3 · 2025-06-02T21:19:18.000Z
diff --git a/.ci/mpas_refgen.jsonc b/.ci/mpas_refgen.jsonc
@@ -48,7 +48,7 @@
         },
         ".*test_conus.*refgen.*cpu.*::node_select" : 
         { 
-          "-l " : { "select"       : 1, "ncpus" : 64, "mpiprocs" : 4   }
+          "-l " : { "select"       : 1, "ncpus" : 64, "mpiprocs" : 1   }
         },
         ".*test_jw.*parallel.*cpu.*::node_select" :
         {
diff --git a/.ci/mpas_tests.jsonc b/.ci/mpas_tests.jsonc
@@ -11,12 +11,51 @@
 
       ".*make.*::args_build_core"     : [ "-c", "atmosphere" ],
       ".*debug.*::args_build_debug"     : [ "-d" ],
+      ".*make.*double.*::args_build_precision"     : [ "-p" ],
       //".*make.*gpu.*::args_build_options_0"     : [ "-g" ],
-      ".*make.*cpu.*::args_build_options"     : [ "-b", "-j $NUM_PROCS" ],
-      ".*make.*gpu.*::args_build_options"     : [ "-b", "-j $NUM_PROCS"],
-      ".*make.*gnu.*::args_target"     : [ "-t", "gnu" ],
+      //".*make.*cpu.*::args_build_options"     : [ "-b", "-j $NUM_PROCS" ],
+      ".*make.*::args_build_options"     : [ "-b", "-j $NUM_PROCS"],
+      ".*gnu.*::args_target"     : [ "-t", "gnu" ],
       ".*make.*nvhpc.*::args_target"     : [ "-t", "nvhpc" ],
-      ".*make.*nvhpc.*gpu.*::args_openacc"     : [ "-g" ]
+      ".*make.*nvhpc.*gpu.*::args_openacc"     : [ "-g" ],
+      ".*test_.*::args_namelist" : [ "-b", "atmosphere_model"],
+      ".*test_jw.*::args_namelist" : [ "-f", "/glade/campaign/mmm/wmr/mpas_ci/jw_baroclinic"],
+      ".*test_conus.*::args_namelist" : [ "-f", "/glade/campaign/mmm/wmr/mpas_ci/conus_lam"],
+      ".*test_.*cpu.*::args_namelist" :
+        [ 
+          "-p", "mpiexec -n $CI_NTASKS -ppn $CI_TASKS_PER_NODE"
+        ],
+      ".*test_.*gpu.*::args_namelist" :
+        [ 
+          "-p", "mpiexec -n $CI_NTASKS -ppn $CI_TASKS_PER_NODE set_gpu_rank "
+        ],
+      //".*test.*(base|parallel|multinode).*::args_namelist" :
+      //  [ 
+      //    "-q", "01:00:00",
+      //    "-w", "02:00:00"
+      //  ],
+        ".*test_jw.*restart.*::args_namelist" :
+        [ 
+          "-y", "0000-01-01_01:00:00",
+          "-z", "01:00:00"
+        ],
+        ".*test_conus.*restart.*::args_namelist" :
+        [ 
+          "-y", "2019-09-01_00:10:00",
+          "-z", "00:10:00"
+        ],
+        ".*test_jw.*perf.*::args_namelist" :
+        [ 
+          "-z", "01:00:00"
+        ],
+        ".*test_conus.*perf.*::args_namelist" :
+        [ 
+          "-z", "00:10:00"
+        ],
+      ".*test_.*gnu.*::args_namelist" :
+        [ 
+          "-e", "LD_LIBRARY_PATH=/glade/u/apps/derecho/23.09/spack/opt/spack/parallel-netcdf/1.12.3/cray-mpich/8.1.27/gcc/12.2.0/sq5u/lib:/glade/u/apps/derecho/23.09/spack/opt/spack/netcdf/4.9.2/gcc/12.2.0/gjc6/lib:\\\\$LD_LIBRARY_PATH"
+        ]
     },
     // Derecho-specifics
     "hsn.de.hpc"  :
@@ -34,7 +73,7 @@
           { 
             "-l " : { "select"       : 1, "ncpus" : 1  }
           },
-        ".*test_jw.*parallel.*cpu.*::node_select" :
+        ".*test_jw.*mpi.*cpu.*::node_select" :
         {
            "-l " : { "select"       : 1, "ncpus" : 16, "mpiprocs" : 4 } 
         },
@@ -73,45 +112,8 @@
         "global_modules" : [ "ncarenv/23.09"],
         //"very_last_modules"       : [  "cray-mpich", "parallel-netcdf" ],
         ".*gnu.*::test_modules"   : [ "gcc/12.2.0","cray-mpich", "parallel-netcdf", "netcdf" ],
-        ".*nvhpc.*::test_modules"   : [ "nvhpc/24.3","cray-mpich","ncarcompilers", "parallel-netcdf", "cuda" ],
-        ".*test.*::args_namelist" : [ "-b", "atmosphere_model"],
-        ".*test_jw.*::args_namelist" : [ "-f", "/glade/campaign/mmm/wmr/mpas_ci/jw_baroclinic"],
-        ".*test_conus.*::args_namelist" : [ "-f", "/glade/campaign/mmm/wmr/mpas_ci/conus_lam"],
-        ".*test_.*cpu.*::args_namelist" :
-          [ 
-            "-p", "mpiexec -n $CI_NTASKS -ppn $CI_TASKS_PER_NODE"
-          ],
-        ".*test_.*gpu.*::args_namelist" :
-          [ 
-            "-p", "mpiexec -n $CI_NTASKS -ppn $CI_TASKS_PER_NODE set_gpu_rank "
-          ],
-        //".*test.*(base|parallel|multinode).*::args_namelist" :
-        //  [ 
-        //    "-q", "01:00:00",
-        //    "-w", "02:00:00"
-        //  ],
-          ".*test_jw.*restart.*::args_namelist" :
-          [ 
-            "-y", "0000-01-01_01:00:00",
-            "-z", "01:00:00"
-          ],
-          ".*test_conus.*restart.*::args_namelist" :
-          [ 
-            "-y", "2019-09-01_00:10:00",
-            "-z", "00:10:00"
-          ],
-          ".*test_jw.*perf.*::args_namelist" :
-          [ 
-            "-z", "01:00:00"
-          ],
-          ".*test_conus.*perf.*::args_namelist" :
-          [ 
-            "-z", "00:10:00"
-          ],
-        ".*test.*gnu.*::args_namelist" :
-          [ 
-            "-e", "LD_LIBRARY_PATH=/glade/u/apps/derecho/23.09/spack/opt/spack/parallel-netcdf/1.12.3/cray-mpich/8.1.27/gcc/12.2.0/sq5u/lib:/glade/u/apps/derecho/23.09/spack/opt/spack/netcdf/4.9.2/gcc/12.2.0/gjc6/lib:\\\\$LD_LIBRARY_PATH"
-          ]
+        ".*nvhpc.*::test_modules"   : [ "nvhpc/24.3","cray-mpich","ncarcompilers", "parallel-netcdf", "cuda" ]
+        
       }
     }
   },
@@ -123,28 +125,60 @@
         "arguments" :
         {
           // Remove auto-filled dir and supply sub-testcase specific ones
-          ".*testcase.*::args_namelist_dir" : [ ]
+          ".*test_.*::args_namelist_dir" : ["-e", "LD_LIBRARY_PATH=/glade/u/apps/derecho/23.09/spack/opt/spack/parallel-netcdf/1.12.3/cray-mpich/8.1.27/gcc/12.2.0/sq5u/lib:/glade/u/apps/derecho/23.09/spack/opt/spack/netcdf/4.9.2/gcc/12.2.0/gjc6/lib:\\\\$LD_LIBRARY_PATH"  ]
         }
       },
       "steps" :
       {
-        "make-release" :
+        // "make-release" :
+        // {
+        //   "command"      : ".ci/tests/build.sh"
+        // },
+        "test_jw-base-cpu" :
         {
-          "command"      : ".ci/tests/build.sh"
+          "submit_options" : { "timelimit" : "00:15:00" },
+          "command"      : ".ci/tests/runTestCases.sh",
+          "arguments" : ["-d", "cpu", "-t", "gnu", "-c", "jw", "-g", "base"]
+          //"dependencies" : { "make-release" : "afterok" }
         },
-        "testcase-jw-base" :
+        "test_jw-restart-cpu" :
         {
           "submit_options" : { "timelimit" : "00:15:00" },
           "command"      : ".ci/tests/runTestCases.sh",
-          "arguments" : ["-d", "cpu", "-t", "gnu", "-c", "jw", "-g", "base"],
-          "dependencies" : { "make-release" : "afterok" }
+          "arguments" : ["-d", "cpu", "-t", "gnu", "-c", "jw", "-g", "restart"],
+          "dependencies" : { "test_jw-base-cpu" : "afterok" }
         },
-        "testcase-jw-restart" :
+        "test_jw-mpi-cpu" :
         {
           "submit_options" : { "timelimit" : "00:15:00" },
           "command"      : ".ci/tests/runTestCases.sh",
-          "arguments" : ["-d", "cpu", "-t", "gnu", "-c", "jw", "-g", "restart"],
-          "dependencies" : { "testcase-jw-base" : "afterok" }
+          "arguments" : ["-d", "cpu", "-t", "gnu", "-c", "jw", "-g", "mpi"],
+          "dependencies" : { "test_jw-restart-cpu" : "afterok" }
+        }
+      }
+    },
+    "gnu-double" :
+    {
+      "submit_options" :
+      {
+        "arguments" :
+        {
+          // Remove auto-filled dir and supply sub-testcase specific ones
+          ".*test_.*::args_namelist_dir" : ["-e", "LD_LIBRARY_PATH=/glade/u/apps/derecho/23.09/spack/opt/spack/parallel-netcdf/1.12.3/cray-mpich/8.1.27/gcc/12.2.0/sq5u/lib:/glade/u/apps/derecho/23.09/spack/opt/spack/netcdf/4.9.2/gcc/12.2.0/gjc6/lib:\\\\$LD_LIBRARY_PATH" ]
+        }
+      },
+      "steps" :
+      {
+        "make-release-double" :
+        {
+          "command"      : ".ci/tests/build.sh"
+        },
+        "test_jw-base-cpu" :
+        {
+          "submit_options" : { "timelimit" : "00:15:00" },
+          "command"      : ".ci/tests/runTestCases.sh",
+          "arguments" : ["-d", "cpu", "-t", "gnu", "-c", "jw", "-g", "base"],
+          "dependencies" : { "make-release-double" : "afterok" }
         }
       }
     },
@@ -206,7 +240,7 @@
         {
           "submit_options" : { "timelimit" : "00:15:00" },
           "command"      : ".ci/tests/runTestCases.sh",
-          "arguments" : ["-d", "gpu", "-t", "nvhpc", "-c", "jw", "-g", "parallel"],
+          "arguments" : ["-d", "gpu", "-t", "nvhpc", "-c", "jw", "-g", "multigpu"],
           "dependencies" : { "test_jw-nvhpc-restart-gpu" : "afterok" }
         },
         "test_jw-nvhpc-perf-gpu" :
diff --git a/.ci/tests/runTestCases.sh b/.ci/tests/runTestCases.sh
@@ -17,7 +17,7 @@ help()
   echo "  -z <duration>             Run duration"
   echo "  -w <interval>             Output interval"
   echo "  -p <mpirun cmd>           Parallel launch command (MPI), e.g. mpirun, mpiexec_mpt, mpiexec -np 8 --oversubscribe"
-  echo "  -t <target>               Target for the test"
+  echo "  -t <toolchain>               toolchain for the test"
   echo "  -k <diff exec>            Diff executable"
   echo "  -s <folder>               Save result data to prefix location, full path for run constructed as <work>/<thisfolder>/<namelist>/"
   echo "  -i <folder>               Folder for bitwise-identical results, full path for run constructed as <work>/<thisfolder>/<namelist>/"
@@ -96,7 +96,7 @@ while getopts c:g:r:b:f:d:q:y:z:w:p:t:k:s:i:e:nh opt; do
       parallelExec="$OPTARG"
     ;;
     t)
-      target="$OPTARG"
+      toolchain="$OPTARG"
     ;;
     k)
       diffExec="$OPTARG"
@@ -121,24 +121,39 @@ if [ ! -z $envVars ]; then
   setenvStr "$envVars"
 fi
 
-
 # Check if testcase is valid
 if [ "$testcase" != "jw" ] && [ "$testcase" != "conus" ] && [ "$testcase" != "aquaplanet" ]; then
-  echo "Error: Invalid testcase '$testcase'. Must be one of 'jw', 'conus', or 'aquaplanet'."
-  exit 1
+    echo "Error: Invalid testcase '$testcase'. Must be one of 'jw', 'conus', or 'aquaplanet'."
+    exit 1
 fi
 
-
 # Check if testType is valid
-if [ "$testType" != "base" ] && [ "$testType" != "refgen" ] && [ "$testType" != "restart" ] && [ "$testType" != "mpi" ] && [ "$testType" != "multigpu" ] && [ "$testType" != "omp" ] && [ "$testType" != "perf" ]; then
-  echo "Error: Invalid testType '$testType'. Must be one of 'base', 'restart', 'mpi', or 'omp'."
-  exit 1
+if [ "$testType" != "base" ] && [ "$testType" != "refgen" ] && 
+   [ "$testType" != "restart" ] && [ "$testType" != "mpi" ] && [ "$testType" != "multinode" ] &&
+   [ "$testType" != "multigpu" ] && [ "$testType" != "omp" ] &&
+   [ "$testType" != "perfgen" ] && [ "$testType" != "perfcmp" ]; then  
+    echo "Error: Invalid testType '$testType'. Must be one of 'base', 'refgen', 'restart', 'mpi', 'multigpu', 'omp', 'perfgen', or 'perfcmp'."
+    exit 1
+fi
+
+# Check if toolchain is valid
+if [ "$toolchain" != "gnu" ] && [ "$toolchain" != "nvhpc" ] && [ "$toolchain" != "intel" ]; then
+    echo "Error: Invalid toolchain '$toolchain'. Must be one of 'gnu', 'nvhpc', or 'intel'."
+    exit 1
+fi
+
+# Check if device is valid
+if [ "$device" != "cpu" ] && [ "$device" != "gpu" ]; then
+    echo "Error: Invalid device '$device'. Must be one of 'cpu' or 'gpu'."
+    exit 1
 fi
-runDir=${testcase}_${target}_${testType}_${device}_${precision}
 
-baserunDir=${testcase}_${target}_base_${device}_${precision}
 
-TESTNAME="${testcase} ${target} ${testType} ${device} ${precision}"
+runDir=${testcase}_${toolchain}_${testType}_${device}_${precision}
+
+baserunDir=${testcase}_${toolchain}_base_${device}_${precision}
+
+TESTNAME="${testcase} ${toolchain} ${testType} ${device} ${precision}"
 echo "TEST : $TESTNAME"
 
 # from https://ncar-hpc-docs.readthedocs.io/en/latest/pbs/job-scripts/#derecho
@@ -193,19 +208,14 @@ eval "repo_id=\$( git rev-parse --short=20 HEAD )"  # github actions doesn't fet
 eval "repo_id_short=\$( git rev-parse --short=10 HEAD )"  # github actions doesn't fetch tags yet
 eval "repo_timestamp=\$( git show --no-patch --format=%ci )"
 
-# Clean up previous runs
-# rm wrfinput_d* wrfbdy_d* wrfout_d* wrfchemi_d* wrf_chem_input_d* rsl* real.print.out* wrf.print.out* wrf_d0*_runstats.out qr_acr_qg_V4.dat fort.98 fort.88 -rf
-
-
 # Go to run location now - We only operate here from now on
 cd $runDir || exit $?
-# Clean up previous runs
-# rm wrfinput_d* wrfbdy_d* wrfout_d* wrfchemi_d* wrf_chem_input_d* rsl* real.print.out* wrf.print.out* wrf_d0*_runstats.out qr_acr_qg_V4.dat fort.98 fort.88 -rf
+# TODO: Clean up previous runs
 
 
 # Copy namelist
 echo "Setting $caseInputDir/namelist.atmosphere  as namelist.atmosphere "
-# remove old namelist.input which may be a symlink in which case this would have failed
+# TODO: remove old namelist.input which may be a symlink in which case this would have failed
 #rm namelist.input
 cp $caseInputDir/namelist.atmosphere namelist.atmosphere || exit $?
 cp $caseInputDir/streams.atmosphere streams.atmosphere || exit $?
@@ -221,6 +231,7 @@ if [ -n "$outputInterval" ]; then
   stream_replace "output" "output_interval" "$outputInterval" streams.atmosphere
 fi
 
+
 if [ "$testType" = "restart" ]; then
     nml_replace "config_do_restart" "true" namelist.atmosphere
     nml_replace_quotes "config_run_duration" "$runDuration" namelist.atmosphere
@@ -315,19 +326,19 @@ if [ "$testType" = "restart" ] || [ "$testType" = "mpi" ] || [ "$testType" = "mu
 
 elif [ "$testType" = "base" ]; then
 
-    head_sha=$( cat $caseInputDir/reference/head_${target}_${device}_${precision} ) || exit $?
+    head_sha=$( cat $caseInputDir/reference/head_${toolchain}_${device}_${precision} ) || exit $?
     echo "Comparing base with reference SHA: $head_sha"
-    echo "Restart file: $caseInputDir/reference/restart.${restart_compare_time}_${head_sha}_${target}_${device}.nc"
-    diff_output $runDir/restart.${restart_compare_time}.nc "$caseInputDir/reference/restart.${restart_compare_time}_${head_sha}_${target}_${device}_${precision}.nc"
+    echo "Restart file: $caseInputDir/reference/restart.${restart_compare_time}_${head_sha}_${toolchain}_${device}.nc"
+    diff_output $runDir/restart.${restart_compare_time}.nc "$caseInputDir/reference/restart.${restart_compare_time}_${head_sha}_${toolchain}_${device}_${precision}.nc"
     result=$?
 
 elif [ "$testType" = "refgen" ]; then
   
-    mv "$runDir/restart.${restart_compare_time}.nc" "$caseInputDir/reference/restart.${restart_compare_time}_${repo_id_short}_${target}_${device}_${precision}.nc" || exit $?
+    mv "$runDir/restart.${restart_compare_time}.nc" "$caseInputDir/reference/restart.${restart_compare_time}_${repo_id_short}_${toolchain}_${device}_${precision}.nc" || exit $?
     
-    echo "${repo_id_short}" > $caseInputDir/reference/head_${target}_${device}_${precision} || exit $?
+    echo "${repo_id_short}" > $caseInputDir/reference/head_${toolchain}_${device}_${precision} || exit $?
 
-elif [ "$testType" = "perf" ]; then
+elif [ "$testType" = "perfgen" ] || [ "$testType" = "perfcmp" ]; then
 
     log_file_path=$runDir/$log_file
 
@@ -351,9 +362,12 @@ elif [ "$testType" = "perf" ]; then
 
     machine="derecho"
 
-    eval "python $workingDirectory/.ci/tests/perf_stats.py $db_file $testcase $machine $device $target $repo_id $totaltime_1,$totaltime_2,$totaltime_3,$totaltime_4,$totaltime_5"
+    if [ "$testType" = "perfgen" ]; then
+        eval "python $workingDirectory/.ci/tests/perf_stats.py $db_file $testcase $machine $device $toolchain $repo_id $totaltime_1,$totaltime_2,$totaltime_3,$totaltime_4,$totaltime_5"
+    elif [ "$testType" = "perfcmp" ]; then
+        eval "python $workingDirectory/.ci/tests/query_perf_db.py compare_to_ref $db_file $testcase $machine $device $toolchain $totaltime_1,$totaltime_2,$totaltime_3,$totaltime_4,$totaltime_5"
+    fi
     result=$?
-    #eval "python $workingDirectory/.ci/tests/query_perf_db.py compare_to_ref $db_file $testcase $machine $device $target $totaltime_1,$totaltime_2,$totaltime_3,$totaltime_4,$totaltime_5"
 
 fi
 
@@ -365,6 +379,10 @@ else
     echo "TEST $TESTNAME FAIL"
     exit 1
 fi
+
+
+
+
 #if [ -z "$errorMsg" ]; then
 # Unlink everything we linked in
 #ls $data/ | xargs -I{} rm {}

Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@`
`48`	`48`	`},`
`49`	`49`	`".test_conus.refgen.cpu.::node_select" :`
`50`	`50`	`{`
`51`		`- "-l " : { "select" : 1, "ncpus" : 64, "mpiprocs" : 4 }`
	`51`	`+ "-l " : { "select" : 1, "ncpus" : 64, "mpiprocs" : 1 }`
`52`	`52`	`},`
`53`	`53`	`".test_jw.parallel.cpu.::node_select" :`
`54`	`54`	`{`