Skip to content

Commit b161f23

Browse files
author
Changho Kim
committed
compressible_stag_mui: slurm and non-slurm implementation and test cases
1 parent 313fa9a commit b161f23

15 files changed

Lines changed: 403 additions & 50 deletions

File tree

‎exec/compressible_stag_mui/GNUmakefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ COMP = gnu
1111
DIM = 3
1212
TINY_PROFILE = FALSE
1313
USE_MUI = TRUE
14+
USE_SLURM = FALSE
1415

1516
USE_PARTICLES = FALSE
1617

@@ -72,3 +73,7 @@ endif
7273
ifeq ($(USE_MUI), TRUE)
7374
DEFINES += -DMUI
7475
endif
76+
77+
ifeq ($(USE_SLURM), TRUE)
78+
DEFINES += -DSLURM
79+
endif

‎exec/compressible_stag_mui/SPPARKS_MUI/MAKE/Makefile.mui‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,31 @@ SHELL = /bin/sh
66
# compiler/linker settings
77
# specify flags and libraries needed for your compiler
88

9-
CC = mpicxx
10-
CCFLAGS = -O -std=c++11 -g -DMUI
11-
SHFLAGS = -fPIC
12-
DEPFLAGS = -M
9+
CC = mpicxx
10+
CCFLAGS = -O -std=c++11
11+
SHFLAGS = -fPIC
12+
DEPFLAGS = -M
1313

14-
LINK = mpicxx
15-
LINKFLAGS = -O -g -DMUI
16-
LIB =
17-
SIZE = size
14+
LINK = mpicxx
15+
LINKFLAGS = -O
16+
LIB =
17+
SIZE = size
1818

19-
ARCHIVE = ar
20-
ARFLAGS = -rc
21-
SHLIBFLAGS = -shared
19+
ARCHIVE = ar
20+
ARFLAGS = -rc
21+
SHLIBFLAGS = -shared
22+
23+
# ---------------------------------------------------------------------
24+
25+
MUI_INC = -DMUI -I../../../../src_mui
2226

2327
# ---------------------------------------------------------------------
2428
# SPPARKS-specific settings
2529
# specify settings for SPPARKS features you will use
2630

2731
# SPPARKS ifdef options, see doc/Section_start.html
2832

29-
SPK_INC = -DSPPARKS_GZIP -DSPPARKS_PNG -DSPPARKS_JPEG
33+
SPK_INC = -DSPPARKS_GZIP -DSPPARKS_PNG -DSPPARKS_JPEG
3034

3135
# MPI library, can be src/STUBS dummy lib
3236
# INC = path for mpi.h, MPI compiler settings
@@ -42,15 +46,15 @@ MPI_LIB =
4246
# PATH = path for JPEG library
4347
# LIB = name of JPEG library
4448

45-
JPG_INC =
46-
JPG_PATH =
49+
JPG_INC =
50+
JPG_PATH =
4751
JPG_LIB = -lpng -ljpeg
4852

4953
# ---------------------------------------------------------------------
5054
# build rules and dependencies
5155
# no need to edit this section
5256

53-
EXTRA_INC = $(SPK_INC) $(MPI_INC) $(JPG_INC) -I../../../../src_mui
57+
EXTRA_INC = $(MUI_INC) $(SPK_INC) $(MPI_INC) $(JPG_INC)
5458
EXTRA_PATH = $(MPI_PATH) $(JPG_PATH)
5559
EXTRA_LIB = $(MPI_LIB) $(JPG_LIB)
5660

‎exec/compressible_stag_mui/SPPARKS_MUI/MAKE/Makefile.mui_cori‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,31 @@ SHELL = /bin/sh
66
# compiler/linker settings
77
# specify flags and libraries needed for your compiler
88

9-
CC = CC
10-
CCFLAGS = -O -std=c++11
11-
SHFLAGS = -fPIC
12-
DEPFLAGS = -M
9+
CC = CC
10+
CCFLAGS = -O -std=c++11
11+
SHFLAGS = -fPIC
12+
DEPFLAGS = -M
1313

14-
LINK = CC
15-
LINKFLAGS = -O
16-
LIB =
17-
SIZE = size
14+
LINK = CC
15+
LINKFLAGS = -O
16+
LIB =
17+
SIZE = size
1818

19-
ARCHIVE = ar
20-
ARFLAGS = -rc
21-
SHLIBFLAGS = -shared
19+
ARCHIVE = ar
20+
ARFLAGS = -rc
21+
SHLIBFLAGS = -shared
22+
23+
# ---------------------------------------------------------------------
24+
25+
MUI_INC = -DMUI -DSLURM -I../../../../src_mui
2226

2327
# ---------------------------------------------------------------------
2428
# SPPARKS-specific settings
2529
# specify settings for SPPARKS features you will use
2630

2731
# SPPARKS ifdef options, see doc/Section_start.html
2832

29-
SPK_INC = -DSPPARKS_GZIP -DSPPARKS_PNG -DSPPARKS_JPEG
33+
SPK_INC = -DSPPARKS_GZIP -DSPPARKS_PNG -DSPPARKS_JPEG
3034

3135
# MPI library, can be src/STUBS dummy lib
3236
# INC = path for mpi.h, MPI compiler settings
@@ -42,15 +46,15 @@ MPI_LIB =
4246
# PATH = path for JPEG library
4347
# LIB = name of JPEG library
4448

45-
JPG_INC =
46-
JPG_PATH =
49+
JPG_INC =
50+
JPG_PATH =
4751
JPG_LIB = -lpng -ljpeg
4852

4953
# ---------------------------------------------------------------------
5054
# build rules and dependencies
5155
# no need to edit this section
5256

53-
EXTRA_INC = $(SPK_INC) $(MPI_INC) $(JPG_INC) -I../../MUI
57+
EXTRA_INC = $(MUI_INC) $(SPK_INC) $(MPI_INC) $(JPG_INC)
5458
EXTRA_PATH = $(MPI_PATH) $(JPG_PATH)
5559
EXTRA_LIB = $(MPI_LIB) $(JPG_LIB)
5660

‎exec/compressible_stag_mui/SPPARKS_MUI/MAKE/Makefile.mui_slurm‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
# serial = MPI with its default compiler
2+
3+
SHELL = /bin/sh
4+
5+
# ---------------------------------------------------------------------
6+
# compiler/linker settings
7+
# specify flags and libraries needed for your compiler
8+
9+
CC = mpicxx
10+
CCFLAGS = -O -std=c++11
11+
SHFLAGS = -fPIC
12+
DEPFLAGS = -M
13+
14+
LINK = mpicxx
15+
LINKFLAGS = -O
16+
LIB =
17+
SIZE = size
18+
19+
ARCHIVE = ar
20+
ARFLAGS = -rc
21+
SHLIBFLAGS = -shared
22+
23+
# ---------------------------------------------------------------------
24+
25+
MUI_INC = -DMUI -DSLURM -I../../../../src_mui
26+
27+
# ---------------------------------------------------------------------
28+
# SPPARKS-specific settings
29+
# specify settings for SPPARKS features you will use
30+
31+
# SPPARKS ifdef options, see doc/Section_start.html
32+
33+
SPK_INC = -DSPPARKS_GZIP -DSPPARKS_PNG -DSPPARKS_JPEG
34+
35+
# MPI library, can be src/STUBS dummy lib
36+
# INC = path for mpi.h, MPI compiler settings
37+
# PATH = path for MPI library
38+
# LIB = name of MPI library
39+
40+
MPI_INC =
41+
MPI_PATH =
42+
MPI_LIB =
43+
44+
# JPEG library, only needed if -DLAMMPS_JPEG listed with LMP_INC
45+
# INC = path for jpeglib.h
46+
# PATH = path for JPEG library
47+
# LIB = name of JPEG library
48+
49+
JPG_INC =
50+
JPG_PATH =
51+
JPG_LIB = -lpng -ljpeg
52+
53+
# ---------------------------------------------------------------------
54+
# build rules and dependencies
55+
# no need to edit this section
56+
57+
EXTRA_INC = $(MUI_INC) $(SPK_INC) $(MPI_INC) $(JPG_INC)
58+
EXTRA_PATH = $(MPI_PATH) $(JPG_PATH)
59+
EXTRA_LIB = $(MPI_LIB) $(JPG_LIB)
60+
61+
# Link target
62+
63+
$(EXE): $(OBJ)
64+
$(LINK) $(LINKFLAGS) $(EXTRA_PATH) $(OBJ) $(EXTRA_LIB) $(LIB) -o $(EXE)
65+
$(SIZE) $(EXE)
66+
67+
# Library targets
68+
69+
lib: $(OBJ)
70+
$(ARCHIVE) $(ARFLAGS) $(EXE) $(OBJ)
71+
72+
shlib: $(OBJ)
73+
$(CC) $(CCFLAGS) $(SHFLAGS) $(SHLIBFLAGS) $(EXTRA_PATH) -o $(EXE) \
74+
$(OBJ) $(EXTRA_LIB) $(LIB)
75+
76+
# Compilation rules
77+
78+
%.o:%.cpp
79+
$(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c $<
80+
81+
%.d:%.cpp
82+
$(CC) $(CCFLAGS) $(EXTRA_INC) $(DEPFLAGS) $< > $@
83+
84+
# Individual dependencies
85+
86+
DEPENDS = $(OBJ:.o=.d)
87+
include $(DEPENDS)

‎exec/compressible_stag_mui/SPPARKS_MUI/main.cpp‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,20 @@ using namespace SPPARKS_NS;
2828

2929
int main(int argc, char **argv)
3030
{
31-
//MPI_Init(&argc,&argv);
3231
#ifdef MUI
3332
MPI_Comm comm = mui::mpi_split_by_app(argc,argv);
3433
mui::uniface2d uniface( "mpi://KMC-side/FHD-KMC-coupling" );
35-
//SPPARKS *spk = new SPPARKS(argc,argv,MPI_COMM_WORLD);
34+
#ifdef SLURM
35+
// expected args for MUI-SLURM: exec %t %o inputs_file ...
36+
// since %t and %o were already used by mpi_split_by_app
37+
// we remove them from args list
38+
argc -= 2;
39+
for (int i=1;i<argc;i++) argv[i] = argv[i+2];
40+
#endif
3641
SPPARKS *spk = new SPPARKS(argc,argv,comm);
3742
spk->uniface = &uniface;
3843
#else
44+
// nonmui
3945
MPI_Init(&argc,&argv);
4046
SPPARKS *spk = new SPPARKS(argc,argv,MPI_COMM_WORLD);
4147
#endif

‎exec/compressible_stag_mui/main.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,18 @@ void main_driver (const char* argv);
1111
int main (int argc, char* argv[])
1212
{
1313
MPI_Comm comm = mui::mpi_split_by_app( argc, argv );
14+
15+
#ifdef SLURM
16+
// expected args: exec %t %o inputs_file ...
17+
if (argc<4) MPI_Abort(comm,0);
18+
// remove %t and %o from args list
19+
argc -= 2;
20+
for (int i=1;i<argc;i++) argv[i] = argv[i+2];
21+
#else
22+
// expected args: exec inputs_file ...
23+
if (argc<2) MPI_Abort(comm,0);
24+
#endif
25+
1426
amrex::Initialize(argc,argv,true,comm);
1527

1628
// argv[1] contains the name of the inputs file entered at the command line

‎exec/compressible_stag_mui/test_COAr_eq/inputs_fhd_stag‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ visc_type = 2
4040
# Advection method
4141
# if advection_type = 1, interpolate primitive quantities
4242
# if advection_type = 2, interpolate conserved quantities
43-
advection_type = 1
43+
advection_type = 2
4444

4545
transport_type = 1
4646

‎exec/compressible_stag_mui/test_COAr_eq_slurm/clean.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
rm -rf slurm-*.out log.* plt* res.* *.png Backtrace*

‎exec/compressible_stag_mui/test_COAr_eq_slurm/diff.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
DIR1=./
4+
DIR2=../test_COAr_eq_slurm
5+
FILES="submit_job.sh job_script.sh mpmd.conf inputs_fhd_stag in.kmc clean.sh diff.sh"
6+
7+
for file in $FILES
8+
do
9+
echo "** $file"
10+
diff $DIR1/$file $DIR2/$file
11+
done

‎exec/compressible_stag_mui/test_COAr_eq_slurm/in.kmc‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
app_style surfchemtest
2+
seed ${SEED}
3+
4+
dimension 2
5+
boundary p p p
6+
7+
variable lat_const equal 2.8e-8
8+
variable lat_const_sqrt3 equal sqrt(3)*${lat_const}
9+
10+
lattice tri ${lat_const}
11+
12+
variable n1x equal 300
13+
variable n1y equal 150
14+
variable n2x equal 4
15+
variable n2y equal 4
16+
variable Nx equal ${n1x}*${n2x}
17+
variable Ny equal ${n1y}*${n2y}
18+
19+
region sys_domain block 0 ${Nx} 0 ${Ny} -0.5 0.5
20+
create_box sys_domain
21+
create_sites box
22+
23+
sector yes tstop 1.e-12
24+
solve_style tree
25+
26+
temperature 1000.
27+
28+
variable dxFHD equal ${n1x}*${lat_const}
29+
variable dyFHD equal ${n1y}*${lat_const_sqrt3}
30+
variable lat_off_x equal ${lat_const}/4
31+
variable lat_off_y equal ${lat_const_sqrt3}/4
32+
33+
mui_fhd_lattice_size ${dxFHD} ${dyFHD}
34+
mui_kmc_lattice_offset ${lat_off_x} ${lat_off_y}
35+
36+
###########################
37+
38+
# i1: 1 (siteA) 2 (siteB) 3 (siteC)
39+
# i2: 0 (vac) 1 (spec1) 2 (spec2) 3 (spec3) 4 (spec4) 5 (spec5)
40+
# i3 - i7: ac1-ac5 = number of adsorption events of the species at the site
41+
# i8 - i12: dc1-dc5 = number of desoprtion events of the species at the site
42+
# d1 - d5: density1-5 = number density of the fluid cell right above the site
43+
# d6: temperature
44+
45+
set i1 value 1 # all siteA
46+
set i2 value 0 # all vac
47+
set i2 value 1 fraction 2.053333e-01 # equilibrium coverage
48+
49+
#event 4 siteA vac 7.562496e+06 spec1 rate # adsorption of spec1 (rate)
50+
event 4 siteA vac 1.475765e-11 spec1 beta 0.5 # adsorption of spec1 (rate const)
51+
event 5 siteA spec1 2.926784e+07 vac # desorption of spec1
52+
53+
###########################
54+
55+
variable Trun equal 1.e-12
56+
variable nstep equal 5000000
57+
58+
variable Tstat equal ${nstep}*${Trun}
59+
stats ${Tstat}
60+
diag_style surfchemtest stats yes list vac spec1
61+
62+
mui_init_agg
63+
64+
mui_push_agg 0 one occ1
65+
mui_commit 0
66+
67+
variable ts loop ${nstep}
68+
label ts_loop
69+
# MPI_FETCH
70+
mui_fetch_agg ${ts} density1 temp
71+
mui_forget ${ts}
72+
73+
# RUN KMC
74+
label jump_label1
75+
run ${Trun} pre no post no
76+
77+
# MPI_PUSH
78+
label jump_label2
79+
mui_push_agg ${ts} ac1 dc1 occ1
80+
mui_commit ${ts}
81+
next ts
82+
jump in.kmc ts_loop

0 commit comments

Comments
 (0)