Merge pull request #1598 from LLNL/v2024.02.0-RC

V2024.02.0 RC

Author: rhornung67

.gitlab-ci.yml

@@ -1,67 +1,84 @@
 ###############################################################################
-# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC
+# Copyright (c) 2016-24, Lawrence Livermore National Security, LLC
 # and RAJA project contributors. See the RAJA/LICENSE file for details.
 #
 # SPDX-License-Identifier: (BSD-3-Clause)
 ###############################################################################
 
+# DESCRIPTION:
 ###############################################################################
 # General GitLab pipelines configurations for supercomputers and Linux clusters
 # at Lawrence Livermore National Laboratory (LLNL).
-#
 # This entire pipeline is LLNL-specific
 #
-# Important note: This file is a template provided by
-# llnl/radiuss-shared-ci. It should not require any change from the project to
-# get started but could feature project-specific stages.
+# Important note: This file is a template provided by llnl/radiuss-shared-ci.
+# Remains to set variable values, change the reference to the radiuss-shared-ci
+# repo, opt-in and out optional features. The project can then extend it with
+# additional stages.
 #
-# However, each project should provide:
+# In addition, each project should copy over and complete:
 # - .gitlab/custom-jobs-and-variables.yml
 # - .gitlab/subscribed-pipelines.yml
-# - .gitlab/${MACHINE}-build-and-test-extra.yml
+#
+# The jobs should be specified in a file local to the project,
+# - .gitlab/jobs/${CI_MACHINE}.yml
+# or generated (see LLNL/Umpire for an example).
 ###############################################################################
 
 # We define the following GitLab pipeline variables:
 variables:
-  MP_BRANCH: "develop"
-  GITHUB_PROJECT_NAME: "RAJA"
-  GITHUB_PROJECT_ORG: "LLNL"
-# Use a service user to run CI. This prevents from running pipelines as an
+##### LC GITLAB CONFIGURATION
+# Use an LLNL service user to run CI. This prevents from running pipelines as an
 # actual user.
   LLNL_SERVICE_USER: rajasa
-# Use a service user workspace. Solves permission issues, stores everything
+# Use the service user workspace. Solves permission issues, stores everything
 # at the same location whoever triggers a pipeline.
-#  CUSTOM_CI_BUILDS_DIR: ""
+#  CUSTOM_CI_BUILDS_DIR: "/usr/workspace/rajasa/gitlab-runner"
 # Tells Gitlab to recursively update the submodules when cloning the project.
   GIT_SUBMODULE_STRATEGY: recursive
-# We build the projects in the CI clone directory.
-# TODO: add a clean-up mechanism
+
+##### PROJECT VARIABLES
+  MP_BRANCH: "develop"
+# We build the projects in the CI clone directory (used in
+# script/gitlab/build_and_test.sh script).
+# TODO: add a clean-up mechanism.
   BUILD_ROOT: ${CI_PROJECT_DIR}
+
+##### SHARED_CI CONFIGURATION
+# Required information about GitHub repository
+  GITHUB_PROJECT_NAME: "RAJA"
+  GITHUB_PROJECT_ORG: "LLNL"
 # Set the build-and-test command.
-  BUILD_AND_TEST_CMD: "./scripts/gitlab/build_and_test.sh"
-# Override the list of branch that will skip the "draft PR test".
-# Add protected branches here. Defaults to "develop main master".
-  ALWAYS_RUN_LIST: "develop main"
+# Nested variables are allowed and useful to customize the job command. We
+# prevent variable expansion so that you can define them at job level.
+  JOB_CMD:
+    value: "./scripts/gitlab/build_and_test.sh"
+    expand: false
+# Override the pattern describing branches that will skip the "draft PR filter
+# test".  Add protected branches here. See default value in
+# preliminary-ignore-draft-pr.yml.
+  ALWAYS_RUN_PATTERN: "^develop$|^main$|^v[0-9.]*-RC$"
 
-# We organize the build-and-test stage in sub-pipelines. Each sub-pipeline
+# We organize the build-and-test stage with sub-pipelines. Each sub-pipeline
 # corresponds to a test batch on a given machine.
 
 # High level stages
 stages:
-  - machine-checks
+  - prerequisites
   - build-and-test
-  - multi_project
+  - multi-project
 
-# Template for jobs triggering a build-and-test sub-pipelines:
+# Template for jobs triggering a build-and-test sub-pipeline:
 .build-and-test:
   stage: build-and-test
   trigger:
     include:
       - local: '.gitlab/custom-jobs-and-variables.yml'
       - project: 'radiuss/radiuss-shared-ci'
-        ref: v2023.06.0
-        file: '${CI_MACHINE}-build-and-test.yml'
-      - local: '.gitlab/${CI_MACHINE}-build-and-test-extra.yml'
+        ref: 'v2023.12.0'
+        file: 'pipelines/${CI_MACHINE}.yml'
+      - artifact: '${CI_MACHINE}-jobs.yml'
+        job: 'generate-job-lists'
     strategy: depend
     forward:
       pipeline_variables: true
@@ -72,7 +89,7 @@ stages:
 #       This will prevent from sticking to a branch (here develop).
 #       MP_BRANCH is short for "Multi-Project Branch" and will usually be develop.
 trigger-rajaperf:
-  stage: multi_project
+  stage: multi-project
   rules:
     - if: '$CI_COMMIT_BRANCH == "${MP_BRANCH}" || $MULTI_PROJECT == "ON"' #run only if ...
   variables:
@@ -83,9 +100,9 @@ trigger-rajaperf:
     strategy: depend
 
 include:
-  # checks preliminary to running the actual CI test (optional)
+  # [Optional] checks preliminary to running the actual CI test
   - project: 'radiuss/radiuss-shared-ci'
-    ref: v2023.03.1
-    file: 'preliminary-ignore-draft-pr.yml'
+    ref: 'v2023.12.0'
+    file: 'utilities/preliminary-ignore-draft-pr.yml'
   # pipelines subscribed by the project
   - local: '.gitlab/subscribed-pipelines.yml'

.gitlab/custom-jobs-and-variables.yml

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2016-23, Lawrence Livermore National Security, LLC and RADIUSS
+# Copyright (c) 2016-24, Lawrence Livermore National Security, LLC and RADIUSS
 # project contributors. See the RAJA/LICENSE file for details.
 #
 # SPDX-License-Identifier: (MIT)
@@ -14,29 +14,40 @@ variables:
 
 # Ruby
 # Arguments for top level allocation
-  RUBY_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --reservation=ci --qos=ci_ruby --time=60 --nodes=2"
+  RUBY_SHARED_ALLOC: "--exclusive --reservation=ci --time=30 --nodes=2"
 # Arguments for job level allocation
-  RUBY_BUILD_AND_TEST_JOB_ALLOC: "--reservation=ci --qos=ci_ruby --time=45 --nodes=1"
+# Note: We repeat the reservation, necessary when jobs are manually re-triggered.
+  RUBY_JOB_ALLOC: "--overlap --reservation=ci --nodes=1"
 # Project specific variants for ruby
   PROJECT_RUBY_VARIANTS: "~shared +openmp +tests"
 # Project specific deps for ruby
   PROJECT_RUBY_DEPS: ""
 
+# Poodle
+# Arguments for top level allocation
+  POODLE_SHARED_ALLOC: "--exclusive --time=40 --nodes=1"
+# Arguments for job level allocation
+  POODLE_JOB_ALLOC: "--overlap --nodes=1"
+# Project specific variants for poodle
+  PROJECT_POODLE_VARIANTS: "~shared +openmp +tests"
+# Project specific deps for poodle
+  PROJECT_POODLE_DEPS: ""
+
 # Corona
 # Arguments for top level allocation
-  CORONA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=60m --nodes=1"
+  CORONA_SHARED_ALLOC: "--exclusive --time-limit=60m --nodes=1 -o per-resource.count=2"
 # Arguments for job level allocation
-  CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=45m --nodes=1 --begin-time=+5s"
+  CORONA_JOB_ALLOC: "--nodes=1 --begin-time=+5s"
 # Project specific variants for corona
   PROJECT_CORONA_VARIANTS: "~shared ~openmp +tests"
 # Project specific deps for corona
   PROJECT_CORONA_DEPS: "^blt@develop "
 
 # Tioga
 # Arguments for top level allocation
-  TIOGA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=60m --nodes=1"
+  TIOGA_SHARED_ALLOC: "--exclusive --time-limit=60m --nodes=1 -o per-resource.count=2"
 # Arguments for job level allocation
-  TIOGA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=45m --nodes=1 --begin-time=+5s"
+  TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s"
 # Project specific variants for corona
   PROJECT_TIOGA_VARIANTS: "~shared ~openmp +tests"
 # Project specific deps for corona
@@ -45,17 +56,17 @@ variables:
 # Lassen and Butte use a different job scheduler (spectrum lsf) that does not
 # allow pre-allocation the same way slurm does.
 # Arguments for job level allocation
-  LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 60"
+  LASSEN_JOB_ALLOC: "1 -W 30 -q pci"
 # Project specific variants for lassen
   PROJECT_LASSEN_VARIANTS: "~shared +openmp +tests cuda_arch=70"
 # Project specific deps for lassen
-  PROJECT_LASSEN_DEPS: ""
+  PROJECT_LASSEN_DEPS: "^blt@develop "
 
 # Configuration shared by build and test jobs specific to this project.
 # Not all configuration can be shared. Here projects can fine tune the
 # CI behavior.
 # See Umpire for an example (export junit test reports).
-.custom_build_and_test:
+.custom_job:
   artifacts:
     reports:
       junit: junit.xml

.gitlab/corona-build-and-test-extra.yml .gitlab/jobs/corona.yml.gitlab/corona-build-and-test-extra.yml renamed to .gitlab/jobs/corona.yml

@@ -1,16 +1,22 @@
 #############################################################################
-# Copyright (c) 2016-23, Lawrence Livermore National Security, LLC
+# Copyright (c) 2016-24, Lawrence Livermore National Security, LLC
 # and RAJA project contributors. See the RAJA/LICENSE file for details.
 #
 # SPDX-License-Identifier: (BSD-3-Clause)
 #############################################################################
 
+# Override reproducer section to define project specific variables.
+.corona_reproducer_vars: &corona_reproducer_vars
+    - |
+      echo -e "export MODULE_LIST=\"${MODULE_LIST}\""
+      echo -e "export SPEC=\"${SPEC//\"/\\\"}\""
+
 ########################
 # Overridden shared jobs
 ########################
-# We duplicate the shared jobs description and add necessary changes for RAJA.
-# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
-# the comparison with the original job is easier.
+# We duplicate the shared jobs description and add necessary changes for this
+# project. We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS}
+# So that the comparison with the original job is easier.
 
 # No overridden jobs so far.
 
@@ -21,8 +27,8 @@
 # ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is no reason not to fully
 # describe the spec here.
 
-rocmcc_5_5_0_hip_desul_atomics:
+rocmcc_5_7_0_hip_desul_atomics:
   variables:
-    SPEC: " ~shared +rocm ~openmp +tests +desul amdgpu_target=gfx906 %rocmcc@5.5.0 ^hip@5.5.0 ^blt@develop"
-  extends: .build_and_test_on_corona
+    SPEC: " ~shared +rocm ~openmp +tests +desul amdgpu_target=gfx906 %rocmcc@5.7.0 ^hip@5.7.0 ^blt@develop"
+  extends: .job_on_corona
 

.gitlab/lassen-build-and-test-extra.yml .gitlab/jobs/lassen.yml.gitlab/lassen-build-and-test-extra.yml renamed to .gitlab/jobs/lassen.yml

@@ -1,33 +1,32 @@
 ##############################################################################
-# Copyright (c) 2016-23, Lawrence Livermore National Security, LLC
+# Copyright (c) 2016-24, Lawrence Livermore National Security, LLC
 # and RAJA project contributors. See the RAJA/LICENSE file for details.
 #
 # SPDX-License-Identifier: (BSD-3-Clause)
 ##############################################################################
 
+# Override reproducer section to define project specific variables.
+.lassen_reproducer_vars: &lassen_reproducer_vars
+    - |
+      echo -e "export MODULE_LIST=\"${MODULE_LIST}\""
+      echo -e "export SPEC=\"${SPEC//\"/\\\"}\""
+
 ########################
 # Overridden shared jobs
 ########################
-# We duplicate the shared jobs description and add necessary changes for RAJA.
-# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
-# the comparison with the original job is easier.
+# We duplicate the shared jobs description and add necessary changes for this
+# project. We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS}
+# So that the comparison with the original job is easier.
 
 # Overriding shared spec: Longer allocation + extra flags
+# Warning: allowed to fail because of a bug in Spack > 0.20.3
 xl_2022_08_19_gcc_8_3_1_cuda_11_2_0:
   variables:
-    SPEC: "${PROJECT_LASSEN_VARIANTS} +cuda cxxflags==\"-qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qpic -qsmp=omp -qsuppress=1500-029 -qsuppress=1500-036\" %xl@16.1.1.12.gcc.8.3.1 ^cuda@11.2.0+allow-unsupported-compilers ${PROJECT_LASSEN_DEPS}"
+    SPEC: "${PROJECT_LASSEN_VARIANTS} +cuda cxxflags==\"-qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qpic -qsmp=omp -qsuppress=1500-029 -qsuppress=1500-036\" %xl@16.1.1.12.gcc.8.3.1 ^cuda@11.2.0+allow-unsupported-compilers ${PROJECT_LASSEN_DEPS} ^blt@develop"
     MODULE_LIST: "cuda/11.2.0"
-    LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 120"
-  extends: .build_and_test_on_lassen
-
-# Overriding shared spec: Longer allocation + extra flags
-xl_2022_08_19_gcc_8_3_1_cuda_11_7_0:
-  variables:
-    SPEC: "${PROJECT_LASSEN_VARIANTS} +cuda cxxflags==\"-qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qpic -qsmp=omp -qsuppress=1500-029 -qsuppress=1500-036\" %xl@16.1.1.12.gcc.8.3.1 ^cuda@11.7.0+allow-unsupported-compilers ${PROJECT_LASSEN_DEPS}"
-    MODULE_LIST: "cuda/11.7.0"
-    LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 120"
-  extends: .build_and_test_on_lassen
-
+    LASSEN_JOB_ALLOC: "1 -W 60 -q pci"
+  extends: .job_on_lassen
+  allow_failure: true
 
 ############
 # Extra jobs
@@ -38,33 +37,33 @@ xl_2022_08_19_gcc_8_3_1_cuda_11_7_0:
 
 gcc_8_3_1_omptask:
   variables:
-    SPEC: " ~shared +openmp +omptask +tests %gcc@8.3.1"
-  extends: .build_and_test_on_lassen
+    SPEC: " ~shared +openmp +omptask +tests %gcc@8.3.1 ^blt@develop"
+  extends: .job_on_lassen
 
 gcc_8_3_1_cuda_11_5_0_ats_disabled:
-  extends: .build_and_test_on_lassen
+  extends: .job_on_lassen
   variables:
-    SPEC: " ~shared +openmp +tests +cuda %gcc@8.3.1 cuda_arch=70 ^cuda@11.5.0+allow-unsupported-compilers"
+    SPEC: " ~shared +openmp +tests +cuda %gcc@8.3.1 cuda_arch=70 ^cuda@11.5.0+allow-unsupported-compilers ^blt@develop"
     MODULE_LIST: "cuda/11.5.0"
-    LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 --atsdisable -W 60"
+    LASSEN_JOB_ALLOC: "1 --atsdisable -W 30 -q pci"
 
 ##########
 # OTHERS
 ##########
 
 clang_13_0_1_libcpp:
   variables:
-    SPEC: " ~shared +openmp +tests %clang@13.0.1 cflags==\"-DGTEST_HAS_CXXABI_H_=0\" cxxflags==\"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0\""
-  extends: .build_and_test_on_lassen
+    SPEC: " ~shared +openmp +tests %clang@13.0.1 cflags==\"-DGTEST_HAS_CXXABI_H_=0\" cxxflags==\"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0\" ^blt@develop"
+  extends: .job_on_lassen
 
 #clang_14_0_5_asan:
 #  variables:
 #    SPEC: " ~shared +openmp +tests %clang@14.0.5 cxxflags==\"-fsanitize=address\" "
 #    ASAN_OPTIONS: "detect_leaks=1"
 #    LSAN_OPTIONS: "suppressions=${CI_PROJECT_DIR}/suppressions.asan"
-#  extends: .build_and_test_on_lassen
+#  extends: .job_on_lassen
 
 gcc_8_3_1_cuda_10_1_243_desul_atomics:
   variables:
-    SPEC: " ~shared +openmp +tests +cuda +desul %gcc@8.3.1 cuda_arch=70 ^cuda@10.1.243+allow-unsupported-compilers"
-  extends: .build_and_test_on_lassen
+    SPEC: " ~shared +openmp +tests +cuda +desul %gcc@8.3.1 cuda_arch=70 ^cuda@10.1.243+allow-unsupported-compilers ^blt@develop"
+  extends: .job_on_lassen