Performance Testing and Assessment

.gitlab-ci.yml

@@ -6,6 +6,7 @@ variables:
 
   SCRIPT_DIR: "scripts"
   ATS_FILE: "tests/integration.ats"
+  PERF_ATS_FILE: "tests/performance.py"
   SPHERAL_MODULE: "Spheral"
   SPACK_NAME: spheral
   SPACK_PKG_NAME: spheral
@@ -20,6 +21,7 @@ stages:
   - build_and_install
   - run_ats
   - update_tpls
+  - run_perf
   - generate_buildcache
   - install_production
   - update_permissions

.gitlab/jobs-mpi.yml

@@ -29,6 +29,10 @@ toss_gcc_mvapich2_cleanup:
   extends: [.toss_resource2, .gcc_mvapich2, .cleanup_dir]
   needs: [toss_gcc_mvapich2_test]
 
+toss_gcc_mvapich2_perf:
+  extends: [.toss_resource2, .gcc_mvapich2, .run_perf, .merge_pr_rule]
+  needs: [toss_gcc_mvapich2_build]
+
 
 
 toss_clang_mvapich2_tpls:
@@ -46,6 +50,10 @@ toss_clang_mvapich2_cleanup:
   extends: [.toss_resource2, .clang_mvapich2, .cleanup_dir]
   needs: [toss_clang_mvapich2_test]
 
+toss_clang_mvapich2_perf:
+  extends: [.toss_resource2, .clang_mvapich2, .run_perf, .merge_pr_rule]
+  needs: [toss_clang_mvapich2_build]
+
 
 
 #toss_oneapi_2022_1_mvapich2:
@@ -68,6 +76,10 @@ blueos_gcc_spectrum_cleanup:
   extends: [.blueos_resource1, .gcc_spectrum, .cleanup_dir]
   needs: [blueos_gcc_spectrum_test]
 
+blueos_gcc_spectrum_perf:
+  extends: [.blueos_resource1, .gcc_spectrum, .run_perf, .merge_pr_rule]
+  needs: [blueos_gcc_spectrum_build]
+
 
 
 blueos_cuda_11_gcc_spectrum_tpls:
@@ -85,5 +97,6 @@ blueos_cuda_11_gcc_spectrum_cleanup:
   extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .cleanup_dir]
   needs: [blueos_cuda_11_gcc_spectrum_test]
 
-
-
+blueos_cuda_11_gcc_spectrum_perf:
+  extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .run_perf, .merge_pr_rule]
+  needs: [blueos_cuda_11_gcc_spectrum_build]

.gitlab/scripts.yml

@@ -27,7 +27,7 @@
   script:
     - CI_BUILD_DIR=$(cat ci-dir.txt)
     - cd $CI_BUILD_DIR && cat job-name.txt
-    - ./$SCRIPT_DIR/devtools/host-config-build.py --no-clean --build --nprocs $NPROC --host-config gitlab.cmake $EXTRA_CMAKE_ARGS
+    - ./$SCRIPT_DIR/devtools/host-config-build.py --no-clean --build --nprocs $NPROC --host-config gitlab.cmake -DENABLE_TIMER=ON $EXTRA_CMAKE_ARGS
   artifacts:
     paths:
       - ci-dir.txt
@@ -67,6 +67,22 @@
     exit_codes:
       - 80
 
+# Run the performance suite and move the resulting Caliper files into the benchmark location
+.run_perf:
+  stage: run_perf
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - CI_BUILD_DIR=$(cat ci-dir.txt)
+    - cd $CI_BUILD_DIR && cat job-name.txt
+
+    - ./build_gitlab/install/spheral-ats --ciRun --numNodes 2 --logs perf_logs ./build_gitlab/install/$PERF_ATS_FILE
+  artifacts:
+    when: always
+    paths:
+      - ci-dir.txt
+      - perf_logs/
+
 .cleanup_dir:
   stage: cleanup
   variables:

.gitlab/specs.yml

@@ -37,7 +37,7 @@
 .cuda_11_gcc_spectrum:
   variables:
     SPEC: 'gcc@$GCC_VERSION+cuda cuda_arch=70'
-    EXTRA_CMAKE_ARGS: '-DENABLE_TIMER=On'
+    EXTRA_CMAKE_ARGS: ''
 
 .oneapi_2022_1_mvapich2:
   variables:

RELEASE_NOTES.md

@@ -32,6 +32,7 @@ Notable changes include:
     * Cleaned up use of std::any in State objects using a visitor pattern to be rigorous ensuring all state entries are handled properly
       during assignement, equality, and cloning operations. This is intended to help ensure our Physics advance during time integration
       is correct.
+    * Performance regression testing is now available. All developers are encouraged to run the performance testing suite for any code changes that might impact performance. See documentation for more details.
 
   * Build changes / improvements:
     * Distributed source directory must always be built now.

cmake/SetupSpheral.cmake

@@ -153,15 +153,10 @@ endif()
 # Build C++ tests and install tests to install directory
 #-------------------------------------------------------------------------------
 if (ENABLE_TESTS)
-  install(DIRECTORY ${SPHERAL_ROOT_DIR}/tests/
-    USE_SOURCE_PERMISSIONS
-    DESTINATION "${SPHERAL_TEST_INSTALL_PREFIX}"
-    PATTERN "*CMakeLists.txt*" EXCLUDE
-    PATTERN "*.cmake" EXCLUDE
-    PATTERN "*.in" EXCLUDE
-    PATTERN "*.pyc" EXCLUDE
-    PATTERN "*~" EXCLUDE)
-  add_subdirectory(${SPHERAL_ROOT_DIR}/tests/unit)
+  spheral_install_python_tests(${SPHERAL_ROOT_DIR}/tests/ ${SPHERAL_TEST_INSTALL_PREFIX})
+  # Always install performance.py in the top of the testing script
+  install(FILES ${SPHERAL_ROOT_DIR}/tests/performance.py
+    DESTINATION ${CMAKE_INSTALL_PREFIX}/tests)
 endif()
 
 include(${SPHERAL_ROOT_DIR}/cmake/SpheralConfig.cmake)

cmake/spheral/SpheralInstallPythonFiles.cmake

@@ -19,3 +19,24 @@ function(spheral_install_python_files)
   endif()
 
 endfunction()
+
+#----------------------------------------------------------------------------------------
+#                                   spheral_instalL_python_tests
+#----------------------------------------------------------------------------------------
+# ----------------------
+# INPUT VARIABLES
+# ----------------------
+# test_dir  : REQUIRED : Source directory of tests to install
+# test_dest : REQUIRED : Destination for tests
+function(spheral_install_python_tests test_dir test_dest)
+  install(DIRECTORY ${test_dir}
+    USE_SOURCE_PERMISSIONS
+    DESTINATION "${test_dest}"
+    PATTERN "*CMakeLists.txt*" EXCLUDE
+    PATTERN "*.cmake" EXCLUDE
+    PATTERN "*.in" EXCLUDE
+    PATTERN "*.pyc" EXCLUDE
+    PATTERN "performance.py" EXCLUDE
+    PATTERN "*~" EXCLUDE)
+  # performance.py must be installed in the top test directory
+endfunction()

docs/build_guide/external/quickstart.rst

@@ -24,5 +24,5 @@ Quickstart
    :end-before: [ex_install-section-end]
 
 .. include:: ../include/quickstart.rst.inc
-   :start-after: [ex_test-section-start]
-   :end-before: [ex_test-section-end]
+   :start-after: [test-section-start]
+   :end-before: [test-section-end]

docs/build_guide/external/tests.rst

@@ -6,13 +6,5 @@ Running Tests
    :end-before: [smoke_test-end]
 
 .. include:: ../include/tests.rst.inc
-   :start-after: [ex_ats_test-start]
-   :end-before: [ex_ats_test-end]
-
-.. include:: ../include/tests.rst.inc
-   :start-after: [filter_options-start]
-   :end-before: [filter_options-end]
-
-.. include:: ../include/tests.rst.inc
-   :start-after: [ex_filter_options_cmd-start]
-   :end-before: [ex_filter_options_cmd-end]
+   :start-after: [ats_test-start]
+   :end-before: [ats_test-end]

docs/build_guide/include/quickstart.rst.inc

@@ -120,7 +120,7 @@ Build and install Spheral.
    TESTING
    ----------------------------------------
 
-[ex_test-section-start]
+[test-section-start]
 Run a basic smoke test for Spheral
 ::
 
@@ -130,22 +130,7 @@ Run a basic smoke test for Spheral
 Run our full test suite.
 ::
 
-  ./.venv/bin/ats -e spheral test/integration.ats
+  ./spheral-ats test/integration.ats
 
 These commands are explained in further sections.
-[ex_test-section-end]
-
-[lc_test-section-start]
-Run a basic smoke test for Spheral
-::
-
-  cd ../install
-  ./spheral -c "import Spheral"
-
-Run our full test suite.
-::
-
-  ./spheral-atstest test/integration.ats
-
-These commands are explained in further sections.
-[lc_test-section-end]
+[test-section-end]

docs/build_guide/include/tests.rst.inc

@@ -8,14 +8,14 @@
 Basic Smoke Test
 ================
 
-After a build and install it's recommended to perform a quick smoke test with Spheral to see if the Spheral environment was installed and all of the libraries were built and linked together correctly.
+After a build and install, we recommend you perform a smoke test with Spheral to ensure the Spheral environment is installed and linked correctly.
 
 From your install directory run:
 ::
 
   ./spheral -c "import Spheral"
 
-This will run the Spheral Python process and try importing the Spheral modules.  If successful you should see a banner printed with information about the version of Spheral, after which Python exits.
+This will run the Spheral python process and import the Spheral modules.  If successful, you should see a banner printed with information about the version of Spheral, after which Python exits.
 
 [smoke_test-end]
 
@@ -24,94 +24,20 @@ This will run the Spheral Python process and try importing the Spheral modules.
 ..
    RUNNING ATS
    ----------------------------------------
-
-[ex_ats_test-start]
-ATS Testing Suite
-=================
-
-Spheral uses ATS to execute a suite of parallel tests. To run this on an external system we need to use Spheral's virtual-env installation of ATS, as external users will not have access to some LC available scripts.
-
-From the install directory run:
-::
-
-  ./.venv/bin/ats -e spheral tests/integration.ats 
-
-[ex_ats_test-end]
-[lc_ats_test-start]
+[ats_test-start]
 ATS Testing Suite
 =================
 
-Spheral uses ATS (the Python Automated Testing System) to execute a suite of parallel tests. During install a script is generated ``spheral-atstest`` which handles calling the ats script in ``/usr/gapps/Spheral/bin`` and passing the generated ``spheral`` executor script to it.
+Spheral uses ATS (Automated Testing System) to execute a suite of parallel tests. During install, a script ``spheral-ats`` is created. This script handles launching allocations and setting appropriate ATS and hardware flags for running the testing suite.
 
 From the install directory run:
 ::
 
-  ./spheral-atstest tests/integration.ats 
-
-[lc_ats_test-end]
-
-
-
-..
-   ATS FILTER OPTIONS
-   ----------------------------------------
-
-[filter_options-start]
-ATS Filters and Options
------------------------
-
-Appropriate filters are injected into the spheral-atstest script based on build configurations. However if you are using ats directly or wish to override some of these filters you can pass them as defined below.
-
-Non MPI Filter
-..............
-
-If Spheral was built without ``MPI`` support we will need to pass a filter to our ``ats`` command to prevent parallel tests from being performed:
-:: 
-
-  --filter='"np<2"'
-
-This stops the ATS suite from attempting to run any tests that rely on more than one process/rank.
-
-Debug Build Filter
-..................
-
-If Spheral was built in Debug mode it is recommended to pass the below filter if you value your time.
-::
-
-  --filter='"level<100"'
-
-These filters stack when invoked. So if you are running the test suite on a non-mpi debug build the command would be:
-
-[filter_options-end]
-
-[ex_filter_options_cmd-start]
-::
-
-  ./.venv/bin/ats -e spheral tests/integration.ats --filter='"np<2"' --filter='"level<100"'
-
-[ex_filter_options_cmd-end]
-[lc_filter_options_cmd-start]
-::
-
-  ./spheral-atstest tests/integration.ats --filter='"np<2"' --filter='"level<100"'
-
-[lc_filter_options_cmd-end]
-
-[lc_cuda_filter-start]
-CUDA Filter
-...........
-
-When building Spheral with CUDA support and without MPI support on Blue-OS systems we need to pass --smpi_off to ATS. When using spheral-atstest this flag is injected for you. However, for veboseness the flag is:
-::
-
-  --addOp "--smpi_off"
-
-.. note::
-   We need to pass ``--addOp`` as a way to forward the ``-smpi_off`` option to ATS. This is because spheral-atstest uses an intermediate layer to control ATS execution.
+  ./spheral-ats tests/integration.ats 
 
-If you are using ATS directly you only need to pass:
+For more information about using ``spheral-ats``, run the following from the install directory:
 ::
 
-  --smpi_off
+  ./spheral-ats --help
 
-[lc_cuda_filter-end]
+[ats_test-end]

docs/build_guide/lc/quickstart.rst

@@ -20,5 +20,5 @@ Quickstart
    :end-before: [lc_install-section-end]
 
 .. include:: ../include/quickstart.rst.inc
-   :start-after: [lc_test-section-start]
-   :end-before: [lc_test-section-end]
+   :start-after: [test-section-start]
+   :end-before: [test-section-end]

docs/build_guide/lc/tests.rst

@@ -6,17 +6,5 @@ Running Tests
    :end-before: [smoke_test-end]
 
 .. include:: ../include/tests.rst.inc
-   :start-after: [lc_ats_test-start]
-   :end-before: [lc_ats_test-end]
-
-.. include:: ../include/tests.rst.inc
-   :start-after: [filter_options-start]
-   :end-before: [filter_options-end]
-
-.. include:: ../include/tests.rst.inc
-   :start-after: [lc_filter_options_cmd-start]
-   :end-before: [lc_filter_options_cmd-end]
-
-.. include:: ../include/tests.rst.inc
-   :start-after: [lc_cuda_filter-start]
-   :end-before: [lc_cuda_filter-end]
+   :start-after: [ats_test-start]
+   :end-before: [ats_test-end]