Skip to content

Commit 19c9fce

Browse files
awnawabawnawab
committed
Merge branch 'develop'
2 parents cdb202f + 6715149 commit 19c9fce

3 files changed

Lines changed: 47 additions & 112 deletions

File tree

‎.github/workflows/build-hpc.yml‎

Lines changed: 33 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -42,16 +42,9 @@ jobs:
4242
#SBATCH --gpus-per-task=1
4343
#SBATCH --mem=0
4444
#SBATCH --qos=dg
45-
modules:
46-
- cmake
47-
- fcm
48-
- ninja
49-
- ecbuild
50-
- prgenv/nvidia
51-
- nvidia/22.11
52-
- hpcx-openmpi/2.14.0-cuda
53-
- python3/3.11.10-01
54-
gpu: 1
45+
arch: ecmwf/hpc2020/nvhpc/24.5/hpcx-openmpi/2.19.0-cuda
46+
modules: ~
47+
build_options: --with-fckit --with-loki --with-acc --with-cuda --with-gpu-aware-mpi --with-static-linking
5548

5649
- name: ac-cpu intel sp
5750
site: ac-batch
@@ -63,16 +56,9 @@ jobs:
6356
#SBATCH --hint=nomultithread
6457
#SBATCH --mem=60GB
6558
#SBATCH --qos=np
66-
modules:
67-
- cmake
68-
- fcm
69-
- ninja
70-
- ecbuild
71-
- prgenv/intel
72-
- intel/2021.4.0
73-
- hpcx-openmpi/2.9.0
74-
- python3
75-
gpu: 0
59+
arch: ecmwf/hpc2020/intel/2021.4.0/hpcx-openmpi/2.9.0
60+
modules: ~
61+
build_options: --with-fckit
7662

7763
runs-on: [self-hosted, linux, hpc]
7864
env:
@@ -83,94 +69,41 @@ jobs:
8369
site: ${{ matrix.site }}
8470
troika_user: ${{ secrets.HPC_CI_SSH_USER }}
8571
sbatch_options: ${{ matrix.sbatch_options }}
86-
template_data: |
87-
cmake_options:
88-
- -DENABLE_MPI=ON
89-
- -DENABLE_LOKI=${{ matrix.gpu }}
90-
- -DENABLE_ACC=${{ matrix.gpu }}
91-
- -DENABLE_CUDA=${{ matrix.gpu }}
92-
- -DENABLE_GPU_AWARE_MPI=${{ matrix.gpu }}
93-
- -DENABLE_SINGLE_PRECISION=ON
94-
- -DENABLE_DOUBLE_PRECISION=OFF
95-
- -DBUILD_SHARED_LIBS=${{ !matrix.gpu }}
96-
dependencies:
97-
ecmwf/eccodes:
98-
version: 2.44.0
99-
cmake_options:
100-
- -DENABLE_MEMFS=ON
101-
- -DENABLE_JPG=OFF
102-
- -DENABLE_PNG=OFF
103-
- -DBUILD_SHARED_LIBS=${{ !matrix.gpu }}
104-
ecmwf/fckit:
105-
version: 0.13.0
106-
cmake_options:
107-
- -DENABLE_TESTS=OFF
108-
- -DENABLE_FCKIT_VENV=ON
109-
- -DBUILD_SHARED_LIBS=${{ !matrix.gpu }}
110-
ecmwf-ifs/fiat:
111-
cmake_options:
112-
- -DENABLE_MPI=ON
113-
- -DENABLE_SINGLE_PRECISION=ON
114-
- -DENABLE_DOUBLE_PRECISION=OFF
115-
- -DBUILD_SHARED_LIBS=${{ !matrix.gpu }}
116-
- -DENABLE_DR_HOOK_NVTX=OFF
117-
ecmwf-ifs/field_api:
118-
version: v0.3.4
119-
cmake_options:
120-
- -DENABLE_TESTS=OFF
121-
- -DENABLE_ACC=${{ matrix.gpu }}
122-
- -DENABLE_CUDA=${{ matrix.gpu }}
123-
- -DENABLE_SINGLE_PRECISION=ON
124-
- -DENABLE_DOUBLE_PRECISION=OFF
125-
- -DBUILD_SHARED_LIBS=${{ !matrix.gpu }}
126-
ecmwf-ifs/loki:
127-
version: 0.3.2
128-
cmake_options:
129-
- -DENABLE_NO_INSTALL=${{ !matrix.gpu }}
130-
- -DENABLE_TESTS=OFF
13172
template: |
73+
REPO=${{ github.event.pull_request.head.repo.full_name || github.repository }}
74+
SHA=${{ github.event.pull_request.head.sha || github.sha }}
75+
{ set +x; } 2>/dev/null # trace off
76+
77+
echo "::group::Load modules"
13278
{% for module in "${{ join(matrix.modules, ',') }}".split(',') %}
13379
module load {{module}}
13480
{% endfor %}
81+
echo "::endgroup::"
13582
136-
BASEDIR=$PWD
137-
138-
{% for name, options in dependencies.items() %}
139-
mkdir -p {{name}}
140-
pushd {{name}}
141-
142-
git init
143-
git remote add origin ${{ github.server_url }}/{{name}}
144-
git fetch origin {{options['version']}}
145-
git reset --hard FETCH_HEAD
146-
147-
cmake -G Ninja -S . -B build \
148-
{% for name in dependencies %}
149-
{% set org, proj = name.split('/') %}
150-
-D{{proj}}_ROOT=$BASEDIR/{{name}}/installation \
151-
{% endfor %}
152-
{{ options['cmake_options']|join(' ') }}
153-
cmake --build build
154-
cmake --install build --prefix installation
155-
popd
156-
{% endfor %}
157-
158-
REPO=${{ github.event.pull_request.head.repo.full_name || github.repository }}
159-
SHA=${{ github.event.pull_request.head.sha || github.sha }}
83+
echo "::group::Checkout $(basename $REPO)"
84+
{ set -x; } 2>/dev/null # trace on
16085
mkdir -p $REPO
16186
pushd $REPO
16287
git init
16388
git remote add origin ${{ github.server_url }}/$REPO
16489
git fetch origin $SHA
16590
git reset --hard FETCH_HEAD
166-
popd
167-
168-
cmake -G Ninja -S $REPO -B build \
169-
{% for name in dependencies %}
170-
{% set org, proj = name.split('/') %}
171-
-D{{proj}}_ROOT=$BASEDIR/{{name}}/installation \
172-
{% endfor %}
173-
{{ cmake_options|join(' ') }}
174-
175-
cmake --build build
176-
ctest --test-dir build
91+
{ set +x; } 2>/dev/null # trace off
92+
echo "::endgroup::"
93+
94+
echo "::group::ecwam-bundle create"
95+
{ set -x; } 2>/dev/null # trace on
96+
./package/bundle/ecwam-bundle create --bundle package/bundle/bundle.yml
97+
{ set +x; } 2>/dev/null # trace off
98+
echo "::endgroup::"
99+
100+
echo "::group::ecwam-bundle build"
101+
{ set -x; } 2>/dev/null # trace on
102+
./package/bundle/ecwam-bundle build --arch package/bundle/arch/${{matrix.arch}} --ninja --keep-going --retry-verbose ${{matrix.build_options}}
103+
{ set +x; } 2>/dev/null # trace off
104+
echo "::endgroup::"
105+
106+
echo "::group::ecwam test"
107+
source build/env.sh
108+
ctest --test-dir build/ecwam
109+
echo "::endgroup::"

‎src/ecwam/mpexchng.F90‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -121,10 +121,10 @@ SUBROUTINE MPEXCHNG(FLD, NDIM2, ND3S, ND3E)
121121
! -------------------------------------------------
122122
CALL GSTATS(1892,0)
123123
#ifdef _OPENACC
124-
!$acc kernels loop independent present(ZCOMBUFS,FLD,NTOPELST,NTOPE,IJTOPE)
124+
!$acc parallel loop gang present(ZCOMBUFS,FLD,NTOPELST,NTOPE,IJTOPE)
125125
DO INGB=1,NGBTOPE !Total number of PE's to which information will be sent
126126
IPROC=NTOPELST(INGB) !To which PE to send informations
127-
!$acc loop independent collapse(3) private(IJ,KCOUNT,M,K,IH)
127+
!$acc loop vector collapse(3) private(IJ,KCOUNT,M,K,IH)
128128
DO M = ND3S, ND3E
129129
DO K = 1, NDIM2
130130
DO IH = 1, NTOPE(IPROC) !How many halo points to be sent
@@ -135,7 +135,7 @@ SUBROUTINE MPEXCHNG(FLD, NDIM2, ND3S, ND3E)
135135
ENDDO
136136
ENDDO
137137
ENDDO
138-
!$acc end kernels
138+
!$acc end parallel loop
139139
#else
140140
!$OMP PARALLEL DO SCHEDULE(STATIC) PRIVATE(INGB,IPROC,KCOUNT,M,K,IH,IJ)
141141
DO INGB=1,NGBTOPE
@@ -214,10 +214,10 @@ SUBROUTINE MPEXCHNG(FLD, NDIM2, ND3S, ND3E)
214214

215215
CALL GSTATS(1893,0)
216216
#ifdef _OPENACC
217-
!$acc kernels loop independent present(ZCOMBUFR,FLD,NFROMPELST,NFROMPE,NIJSTART)
217+
!$acc parallel loop gang present(ZCOMBUFR,FLD,NFROMPELST,NFROMPE,NIJSTART)
218218
DO INGB=1,NGBFROMPE
219219
IPROC=NFROMPELST(INGB)
220-
!$acc loop vector independent collapse(3) private(IJ,KCOUNT,M,K,IH)
220+
!$acc loop vector collapse(3) private(IJ,KCOUNT,M,K,IH)
221221
DO M = ND3S, ND3E
222222
DO K = 1, NDIM2
223223
DO IH = 1, NFROMPE(IPROC)
@@ -228,7 +228,7 @@ SUBROUTINE MPEXCHNG(FLD, NDIM2, ND3S, ND3E)
228228
ENDDO
229229
ENDDO
230230
ENDDO
231-
!$acc end kernels
231+
!$acc end parallel loop
232232
#else
233233
!$OMP PARALLEL DO SCHEDULE(STATIC) PRIVATE(INGB,IPROC,KCOUNT,M,K,IH,IJ)
234234
DO INGB=1,NGBFROMPE

‎src/ecwam/propag_wam.F90‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ SUBROUTINE PROPAG_WAM (BLK2GLO, WAVNUM, CGROUP, OMOSNH2KD, FL1, &
117117
!!! the advection schemes are still written in block structure
118118
!!! mapping chuncks to block ONLY for actual grid points !!!!
119119
#ifdef _OPENACC
120-
!$acc parallel loop private(KIJS, IJSB, KIJL, IJLB)
120+
!$acc parallel loop gang private(KIJS, IJSB, KIJL, IJLB) vector_length(NPROMA_WAM)
121121
#else
122122
!$OMP PARALLEL DO SCHEDULE(STATIC) PRIVATE(ICHNK, KIJS, IJSB, KIJL, IJLB, M, K)
123123
#endif /*_OPENACC*/
@@ -126,10 +126,12 @@ SUBROUTINE PROPAG_WAM (BLK2GLO, WAVNUM, CGROUP, OMOSNH2KD, FL1, &
126126
IJSB = IJFROMCHNK(KIJS, ICHNK)
127127
KIJL = KIJL4CHNK(ICHNK)
128128
IJLB = IJFROMCHNK(KIJL, ICHNK)
129-
!$acc loop independent collapse(2)
130129
DO M = 1, NFRE_RED
131130
DO K = 1, NANG
132-
FL1_EXT(IJSB:IJLB, K, M) = FL1(KIJS:KIJL, K, M, ICHNK)
131+
!$acc loop vector
132+
DO IJ = KIJS,KIJL
133+
FL1_EXT(IJ-KIJS+IJSB, K, M) = FL1(IJ, K, M, ICHNK)
134+
ENDDO
133135
ENDDO
134136
ENDDO
135137
ENDDO
@@ -264,24 +266,24 @@ SUBROUTINE PROPAG_WAM (BLK2GLO, WAVNUM, CGROUP, OMOSNH2KD, FL1, &
264266
DO WHILE (ISUBST <= NSTEP_LF)
265267

266268
#ifdef _OPENACC
267-
!$acc kernels loop private(KIJS, KIJL)
269+
!$acc parallel loop gang private(KIJS, KIJL) vector_length(NPROMA)
268270
#else
269271
!$OMP PARALLEL DO SCHEDULE(STATIC,1) PRIVATE(JKGLO, KIJS, KIJL, M, K, IJ)
270272
#endif /*_OPENACC*/
271273
DO JKGLO = IJSG, IJLG, NPROMA
272274
KIJS=JKGLO
273275
KIJL=MIN(KIJS+NPROMA-1, IJLG)
274-
!$acc loop independent collapse(3)
275276
DO M = ND3S, ND3E
276277
DO K = 1, NANG
278+
!$acc loop vector
277279
DO IJ = KIJS, KIJL
278280
FL1_EXT(IJ, K, M) = FL3_EXT(IJ, K, M)
279281
ENDDO
280282
ENDDO
281283
ENDDO
282284
ENDDO
283285
#ifdef _OPENACC
284-
!$acc end kernels
286+
!$acc end parallel loop
285287
#else
286288
!$OMP END PARALLEL DO
287289
#endif /*_OPENACC*/

0 commit comments

Comments
 (0)