11#!/usr/bin/env python
2- import configparser
32import getopt
43import os
5- import re
64import sys
75import time
86
119
1210import pyEnsLib
1311import pyTools
14- from pyTools import Duplicate , EqualStride
12+ from pyTools import EqualStride
1513
1614#files should have member, then year, then month in the filename (in that order) to be recognized by this script
15+ # we want 1 rank per timeslice (so months*years)
1716
1817def main (argv ):
1918 # Get command line stuff and store in a dictionary
20- s = 'tag= mach= nyear= nmonth= esize= nbin= min_range = maxrange= res= sumfile= indir= jsonfile= verbose mpi_enable mpi_disable nrand= rand seq= jsondir= '
19+ s = 'tag= mach= nyear= nmonth= esize= nbin= minrange = maxrange= res= sumfile= indir= jsonfile= verbose mpi_enable mpi_disable'
2120 optkeys = s .split ()
2221
2322 try :
2423 opts , args = getopt .getopt (argv , 'h' , optkeys )
2524 except getopt .GetoptError :
26- pyEnsLib .EnsSumPop_usage ()
25+ pyEnsLib .EnsSumMom_usage ()
2726 sys .exit (2 )
2827
2928 # Put command line options in a dictionary - also set defaults
@@ -43,12 +42,10 @@ def main(argv):
4342 opts_dict ['sumfile' ] = 'mom6.ens.summary.nc'
4443 opts_dict ['indir' ] = './'
4544 opts_dict ['jsonfile' ] = 'mom6_ensemble.json'
46- opts_dict ['verbose' ] = True
45+ opts_dict ['verbose' ] = False
4746 opts_dict ['mpi_enable' ] = True
4847 opts_dict ['mpi_disable' ] = False
49- opts_dict ['seq' ] = 0
50- opts_dict ['jsondir' ] = './'
51-
48+
5249 # TO DO: why not listed in help: seq, minrange, maxrange, nbin, mpi_enable
5350
5451 # This creates the dictionary of input arguments
@@ -75,25 +72,18 @@ def main(argv):
7572 # Z, Y,X
7673 if opts_dict ['jsonfile' ]:
7774 # Read in the included var list
78- Var_lhh , Var_ihh , Var_lhq , Var_lqh = pyEnsLib .read_jsonlist (opts_dict ['jsonfile' ], 'ES_MOM' )
79- # check for error opening file
80- if len (Var_lhh ) > 0 :
81- if Var_lhh [0 ] == 'JSONERROR' :
82- me .abort ()
75+ json_vars = pyEnsLib .read_jsonlist (opts_dict ['jsonfile' ], 'ES_MOM' )
76+ if len (json_vars ) != 4 :
77+ if me .get_rank () == 0 :
78+ print ('ERROR: could not read MOM6 variable lists from ' , opts_dict ['jsonfile' ])
79+ sys .exit (2 )
80+ Var_lhh , Var_ihh , Var_lhq , Var_lqh = json_vars
81+
82+
8383 # get max size of var names
8484 str_size = 0
85- for str in Var_lhh :
86- if str_size < len (str ):
87- str_size = len (str )
88- for str in Var_ihh :
89- if str_size < len (str ):
90- str_size = len (str )
91- for str in Var_lhq :
92- if str_size < len (str ):
93- str_size = len (str )
94- for str in Var_lqh :
95- if str_size < len (str ):
96- str_size = len (str )
85+ str_size = max (len (v ) for v in Var_lhh + Var_ihh + Var_lhq + Var_lqh )
86+
9787
9888 # get number of each variable type
9989 n_var_lhh = len (Var_lhh )
@@ -113,8 +103,7 @@ def main(argv):
113103 in_files = []
114104 if os .path .exists (input_dir ):
115105 # Get the list of files
116- in_files_temp = os .listdir (input_dir )
117- in_files = sorted (in_files_temp )
106+ in_files = sorted (f for f in os .listdir (input_dir ) if f .endswith ('.nc' ))
118107 num_files = len (in_files )
119108 else :
120109 if me .get_rank () == 0 :
@@ -128,41 +117,47 @@ def main(argv):
128117 print (
129118 'ERROR: Input directory must contain exactly esize*nyear*nmonth = ' ,
130119 files_needed ,
131- ' ) but it has' ,
120+ ' ( but it has' ,
132121 num_files ,
133122 ' files).' ,
134123 )
135124 sys .exit (2 )
136125
137- # Don't want more processors than months
138- if me .get_size () > opts_dict ['nmonth' ]:
126+ # if using parallel, then we want exactly one timeslice per process
127+ # (serial handles all timeslices)
128+ ntslices = opts_dict ['nmonth' ] * opts_dict ['nyear' ]
129+ if opts_dict ['mpi_enable' ] and me .get_size () != ntslices :
139130 if me .get_rank () == 0 :
140131 print (
141- 'ERROR: more processors requested than the number of months. Recommendation is one processor per month (or fewer).'
132+ 'ERROR: number of processors (' ,
133+ me .get_size (),
134+ ') must equal nmonth*nyear (' ,
135+ ntslices ,
136+ ') => EXITING....' ,
142137 )
143138 sys .exit (2 )
144139
145- # TO DO (verify/simplify )
146- # Partition the input file list (ideally we have one processor per month )
147- in_file_list = me . partition ( in_files , func = EqualStride (), involved = True )
148-
149- # Check the files in the input directory
150- full_in_files = []
151- if me . get_rank () == 0 and opts_dict [ 'verbose' ] :
152- print ( 'VERBOSE: Input files are:' )
153-
154- for onefile in in_file_list :
155- fname = input_dir + '/' + onefile
156- # if opts_dict['verbose']:
157- # print( "my_rank = ", me.get_rank(), " ", fname)
158- if os . path . isfile ( fname ):
159- full_in_files . append ( fname )
160- else :
161- print ( 'ERROR: Could not locate file: ' + fname + ' => EXITING....' )
162- sys . exit ( )
140+ # Partition the timeslices (serial gets all of them, MPI gets one per rank )
141+ my_slices = me . partition ( list ( range ( ntslices )), func = EqualStride (), involved = True )
142+
143+ # Files for timeslice k are in_files[k::ntslices] (assumes the sorted file
144+ # names are member-major: all months for member 0, then member 1, ...)
145+ slice_files = []
146+ for k in my_slices :
147+ full_in_files = []
148+ for onefile in in_files [ k :: ntslices ]:
149+ fname = input_dir + '/' + onefile
150+ if os . path . isfile ( fname ):
151+ full_in_files . append ( fname )
152+ else :
153+ print ( 'ERROR: Could not locate file: ' + fname + ' => EXITING....' )
154+ if opts_dict [ 'mpi_enable' ]:
155+ me . abort ()
156+ sys . exit ( 2 )
157+ slice_files . append ( full_in_files )
163158
164159 # open just the first file (all procs) to get metadata
165- first_file = nc .Dataset (full_in_files [0 ], 'r' )
160+ first_file = nc .Dataset (slice_files [ 0 ] [0 ], 'r' )
166161
167162 # Store dimensions of the input fields
168163 if verbose and me .get_rank () == 0 :
@@ -182,7 +177,7 @@ def main(argv):
182177 for key in input_dims :
183178 if key == 'z_l' :
184179 z_l = len (input_dims ['z_l' ])
185- if key == 'z_i' :
180+ elif key == 'z_i' :
186181 z_i = len (input_dims ['z_i' ])
187182 elif key == 'yq' :
188183 yq = len (input_dims ['yq' ])
@@ -196,7 +191,7 @@ def main(argv):
196191 if z_i == - 1 or z_l == - 1 or xq == - 1 or xh == - 1 or yh == - 1 or yq == - 1 :
197192 if me .get_rank () == 0 :
198193 print ('ERROR: Need dimensions z_i, z_l, xq, xh, yh and yq => EXITING....' )
199- sys .exit ()
194+ sys .exit (2 )
200195
201196 if verbose and me .get_rank () == 0 :
202197 print ('z_i = ' , z_i )
@@ -256,8 +251,8 @@ def main(argv):
256251 v_vars = nc_sumfile .createVariable ('vars' , 'S1' , ('nvars' , 'str_size' ))
257252 v_var_lhh = nc_sumfile .createVariable ('var_lhh' , 'S1' , ('nvars_lhh' , 'str_size' ))
258253 v_var_ihh = nc_sumfile .createVariable ('var_ihh' , 'S1' , ('nvars_ihh' , 'str_size' ))
259- v_var_lhq = nc_sumfile .createVariable ('var_ihq ' , 'S1' , ('nvars_lhq' , 'str_size' ))
260- v_var_lqh = nc_sumfile .createVariable ('var_iqh ' , 'S1' , ('nvars_lqh' , 'str_size' ))
254+ v_var_lhq = nc_sumfile .createVariable ('var_lhq ' , 'S1' , ('nvars_lhq' , 'str_size' ))
255+ v_var_lqh = nc_sumfile .createVariable ('var_lqh ' , 'S1' , ('nvars_lqh' , 'str_size' ))
261256 v_time = nc_sumfile .createVariable ('time' , 'd' , ('time' ,))
262257
263258 v_ens_avg_lhh = nc_sumfile .createVariable (
@@ -376,12 +371,15 @@ def main(argv):
376371 if verbose :
377372 if me .get_rank () == 0 :
378373 print ('VERBOSE: Assigning time variant metadata .....' )
379- vars_dict = first_file .variables
380- time_value = vars_dict ['time' ]
381- time_array = np .array ([time_value ])
374+ # one time value per timeslice (taken from the first file of each slice)
375+ time_array = np .zeros (len (slice_files ), dtype = np .float64 )
376+ for i , full_in_files in enumerate (slice_files ):
377+ with nc .Dataset (full_in_files [0 ], 'r' ) as f :
378+ time_array [i ] = f .variables ['time' ][0 ]
382379
383380 # gather time array to root (0)
384- time_array = pyEnsLib .gather_npArray_pop (time_array , me , (me .get_size (),))
381+ if opts_dict ['mpi_enable' ]:
382+ time_array = pyEnsLib .gather_npArray_pop (time_array , me , (me .get_size (),))
385383 if me .get_rank () == 0 :
386384 v_time [:] = time_array [:]
387385
@@ -426,6 +424,13 @@ def main(argv):
426424 if verbose and me .get_rank () == 0 :
427425 print ('VERBOSE: Calculating RMSZ scores .....' )
428426
427+ # compute each of this rank's timeslices, then stack the results so that
428+ # every array gets a leading time dimension
429+ results = []
430+ for full_in_files in slice_files :
431+ results .append (
432+ pyEnsLib .mom6_calc_rmsz (full_in_files , Var_lhh , Var_ihh , Var_lhq , Var_lqh , opts_dict )
433+ )
429434 (
430435 zscore_lhh ,
431436 zscore_ihh ,
@@ -439,68 +444,69 @@ def main(argv):
439444 ens_stddev_lqh ,
440445 ens_avg_lhq ,
441446 ens_stddev_lhq ,
442- ) = pyEnsLib . mom6_calc_rmsz ( full_in_files , Var_lhh , Var_ihh , Var_lhq , Var_lqh , opts_dict )
447+ ) = [ np . stack ( r , axis = 0 ) for r in zip ( * results )]
443448
444449 if verbose and me .get_rank () == 0 :
445450 print ('VERBOSE: Finished with RMSZ scores .....' )
446451
447- # Collect from all processors
452+ zmall = np .concatenate ((zscore_lhh , zscore_ihh , zscore_lhq , zscore_lhq ), axis = 1 )
453+
454+ # Collect from all processors (serial already has all timeslices)
448455 if opts_dict ['mpi_enable' ]:
449456 # Gather the variable results from all processors to the master processor
450- zmall = np .concatenate ((zscore_lhh , zscore_ihh , zscore_lqh , zscore_lhq ), axis = 0 )
451- print ("1: zmall = " ,zmall .shape )
457+ # (each rank has exactly one timeslice, so pass in index 0)
452458 zmall = pyEnsLib .gather_npArray_pop (
453- zmall ,
459+ zmall [ 0 ] ,
454460 me ,
455461 (
456462 me .get_size (),
457463 n_var_lhh + n_var_ihh + n_var_lqh + n_var_lhq ,
458- len ( full_in_files ) ,
464+ esize ,
459465 nbin ,
460466 ),
461467 )
462- print ("2: zmall = " ,zmall .shape )
463468 ens_avg_lhh = pyEnsLib .gather_npArray_pop (
464- ens_avg_lhh , me , (me .get_size (), n_var_lhh , z_l , yh , xh )
469+ ens_avg_lhh [ 0 ] , me , (me .get_size (), n_var_lhh , z_l , yh , xh )
465470 )
466471 ens_avg_ihh = pyEnsLib .gather_npArray_pop (
467- ens_avg_ihh , me , (me .get_size (), n_var_ihh , z_i , yh , xh )
472+ ens_avg_ihh [ 0 ] , me , (me .get_size (), n_var_ihh , z_i , yh , xh )
468473 )
469474 ens_avg_lhq = pyEnsLib .gather_npArray_pop (
470- ens_avg_lhq , me , (me .get_size (), n_var_lhq , z_l , yh , xq )
475+ ens_avg_lhq [ 0 ] , me , (me .get_size (), n_var_lhq , z_l , yh , xq )
471476 )
472477 ens_avg_lqh = pyEnsLib .gather_npArray_pop (
473- ens_avg_lqh , me , (me .get_size (), n_var_lqh , z_l , yq , xh )
478+ ens_avg_lqh [ 0 ] , me , (me .get_size (), n_var_lqh , z_l , yq , xh )
474479 )
475480
476481 ens_stddev_lhh = pyEnsLib .gather_npArray_pop (
477- ens_stddev_lhh , me , (me .get_size (), n_var_lhh , z_l , yh , xh )
482+ ens_stddev_lhh [ 0 ] , me , (me .get_size (), n_var_lhh , z_l , yh , xh )
478483 )
479484 ens_stddev_ihh = pyEnsLib .gather_npArray_pop (
480- ens_stddev_ihh , me , (me .get_size (), n_var_ihh , z_i , yh , xh )
485+ ens_stddev_ihh [ 0 ] , me , (me .get_size (), n_var_ihh , z_i , yh , xh )
481486 )
482487 ens_stddev_lhq = pyEnsLib .gather_npArray_pop (
483- ens_stddev_lhq , me , (me .get_size (), n_var_lhq , z_l , yh , xq )
488+ ens_stddev_lhq [ 0 ] , me , (me .get_size (), n_var_lhq , z_l , yh , xq )
484489 )
485490 ens_stddev_lqh = pyEnsLib .gather_npArray_pop (
486- ens_stddev_lqh , me , (me .get_size (), n_var_lqh , z_l , yq , xh )
491+ ens_stddev_lqh [ 0 ] , me , (me .get_size (), n_var_lqh , z_l , yq , xh )
487492 )
488493
489- # Assign to summary file:
490- if me .get_rank () == 0 :
491- print ("RMSZ = " , v_RMSZ .shape )
492- v_RMSZ [:, :, :, :] = zmall [:, :, :, :]
493- v_ens_avg_lhh [:, :, :, :, :] = ens_avg_lhh [:, :, :, :, :]
494- v_ens_stddev_lhh [:, :, :, :, :] = ens_stddev_lhh [:, :, :, :, :]
495- v_ens_avg_ihh [:, :, :, :, :] = ens_avg_ihh [:, :, :, :, :]
496- v_ens_stddev_ihh [:, :, :, :, :] = ens_stddev_ihh [:, :, :, :, :]
497- v_ens_avg_lhq [:, :, :, :, :] = ens_avg_lhq [:, :, :, :, :]
498- v_ens_stddev_lhq [:, :, :, :, :] = ens_stddev_lhq [:, :, :, :, :]
499- v_ens_avg_lqh [:, :, :, :, :] = ens_avg_lqh [:, :, :, :, :]
500- v_ens_stddev_lqh [:, :, :, :, :] = ens_stddev_lqh [:, :, :, :, :]
501-
502- print ('STATUS: PyEnsSumMom6 has completed.' )
503- nc_sumfile .close ()
494+
495+ # Assign to summary file:
496+ if me .get_rank () == 0 :
497+ #print("RMSZ = ", v_RMSZ.shape)
498+ v_RMSZ [:, :, :, :] = zmall [:, :, :, :]
499+ v_ens_avg_lhh [:, :, :, :, :] = ens_avg_lhh [:, :, :, :, :]
500+ v_ens_stddev_lhh [:, :, :, :, :] = ens_stddev_lhh [:, :, :, :, :]
501+ v_ens_avg_ihh [:, :, :, :, :] = ens_avg_ihh [:, :, :, :, :]
502+ v_ens_stddev_ihh [:, :, :, :, :] = ens_stddev_ihh [:, :, :, :, :]
503+ v_ens_avg_lhq [:, :, :, :, :] = ens_avg_lhq [:, :, :, :, :]
504+ v_ens_stddev_lhq [:, :, :, :, :] = ens_stddev_lhq [:, :, :, :, :]
505+ v_ens_avg_lqh [:, :, :, :, :] = ens_avg_lqh [:, :, :, :, :]
506+ v_ens_stddev_lqh [:, :, :, :, :] = ens_stddev_lqh [:, :, :, :, :]
507+
508+ print ('STATUS: PyEnsSumMom6 has completed.' )
509+ nc_sumfile .close ()
504510
505511
506512if __name__ == '__main__' :
0 commit comments