forked from lofar-astron/prefactor
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathPre-Facet-Cal.parset
162 lines (150 loc) · 10.3 KB
/
Pre-Facet-Cal.parset
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
# Pre-Facet Calibration Pipeline
#
# Basic Pre-Facet Calibration Pipeline:
# - no demixing but A-team flagging,
# - calibration transfer and averaging of target data in one go.
# - checks frequncies in MSs to group files
# - the new "error_tolerance" option requires LOFAR software version >= 2.15
# (Comment out all lines with "error_tolerance" if you want to use an older version.)
# - using LoSoTo from the pipeline requires latest executable_args node-script
# (available in the current LOFAR trunk, revision 33969 and later)
# (The diff, to do the patching by hand, can be found in the 6th comment at:
# https://github.com/lofar-astron/prefactor/issues/4 )
# - expects shared filesystem, that all nodes can reach all files!
# (E.g. a single workstation or compute cluster with shared filesystem
# doesn't work on multiple nodes on CEP-2 or CEP3.)
# parameters you will need to adjust.
# averaging for the calibrator and target data
! avg_timestep = 2 # averaging step needed to average the data to 4 seconds time resolution
! avg_freqstep = 2 # averaging step needed to average the data to 4 ch/SB frequency resolution
# where to find the calibrator data
! cal_input_path = /data/scratch/username/PathToYourCalibratorData/
! cal_input_pattern = L*.MS
# Path to the skymodels for the Calibrator
! calibrator_path_skymodel = /cep3home/username/Pre-Facet-Cal/skymodels/
# where to find the target data
! target_input_path = /data/scratch/username/PathToYourTargetData/
! target_input_pattern = L*.MS
# path to the skymodel for the phase-only calibration of the target
! target_skymodel = /cep3home/username/Pre-Facet-Cal/skymodels/PleaseProvideTarget.skymodel
# download the phase-only calibration skymodel from TGSS
# "Force" : always download , "True" download if {{ target_skymodel }} does not exist , "False" : never download
! use_tgss_target = True
# how many subbands to average into on frequency band (usually 10 or 12)
! num_SBs_per_group = 10 # make concatenated measurement-sets with that many subbands
# where to put the resulting measurement sets generated by the pipeline
! results_directory = /media/scratch/test/username/WhereYouWantYourProcessedData/
# where to put the inspection plots generated by the pipeline
! inspection_directory = /media/scratch/test/username/WhereYouWantInspectionPlotsEtc/
# where to put the files with the calibration values that are to be transferred to the target
! cal_values_directory = /media/scratch/test/username/WhereYouWantToStoreTheValuesFromTheCalibrator/
# NDPPP-compatible pattern for baselines or stations to be flagged
! flag_baselines = [ CS013HBA* ]
# minimum fraction of unflagged data after RFI flagging and A-team clipping
! min_unflagged_fraction = 0.5
# name of the station that will be used as a reference for the phase-plots
! reference_station = CS001HBA0
## Values needed for RMextract
# the URL of the server where the IONEX files can be downloaded
# leave it at "None" to disable downloads, or set it to:
# ftp://ftp.unibe.ch/aiub/CODE/
# to download from the "standard" server
! ionex_server = None
# the prefix of the IONEX files
! ionex_prefix = CODG
# path where the IONEX files can be stored or are already stored
! ionex_path = /media/scratch/test/username/WhereYouWantToStoreTheValuesFromTheCalibrator/
## pathes to the scripts etc.
# #### ToDo: get the scripts onto CEP3 and adjust the pathes here!
! calibrator_parset = /homea/htb00/htb001/prefactor/Pre-Facet-Calibrator.parset
! target_parset = /homea/htb00/htb001/prefactor/Pre-Facet-Target.parset
! calib_cal_parset = /homea/htb00/htb001/prefactor/parsets/calibcal.parset
! find_skymodel_cal_auto = /homea/htb00/htb001/prefactor/scripts/find_skymodel_cal.py
! ATeam_skymodel = /homea/htb00/htb001/prefactor/skymodels/Ateam_LBA_CC.skymodel
! losoto_importer = /homea/htb00/htb001/prefactor/scripts/losotoImporter.py
! get_ion_script = /homea/htb00/htb001/prefactor/bin/download_IONEX.py
! fitclock_script = /homea/htb00/htb001/prefactor/scripts/fit_clocktec_initialguess_losoto.py
! fitamps_script = /homea/htb00/htb001/prefactor/scripts/amplitudes_losoto_3.py
! plotsols_script = /homea/htb00/htb001/prefactor/scripts/examine_npys.py
! fit_XYoffset_script = /homea/htb00/htb001/prefactor/scripts/find_cal_global_phaseoffset_losoto.py
! transfer_script = /homea/htb00/htb001/prefactor/scripts/transfer_gains_RMextract.py
! ATeam_Clipper = /homea/htb00/htb001/prefactor/scripts/Ateamclipper.py
! get_tgss_skymodel_script = /homea/htb00/htb001/prefactor/scripts/download_tgss_skymodel_target.py
! sortmap_script = /homea/htb00/htb001/prefactor/scripts/sort_times_into_freqGroups.py
! check_flagged_script = /homea/htb00/htb001/prefactor/scripts/check_unflagged_fraction.py
! structurefunction_script = /homea/htb00/htb001/prefactor/scripts/getStructure_from_phases.py
! plotphases_script = /homea/htb00/htb001/prefactor/scripts/plot_solutions_all_stations.py
! losoto_executable = /opt/cep/losoto/current/bin/losoto
! flagging_strategy = /homea/htb00/htb003/lofar_jureca_2.17_stack2016a/share/rfistrategies/HBAdefault
! makesourcedb = /homea/htb00/htb003/lofar_jureca_2.17_stack2016a/bin/makesourcedb
# number of processes to use per step per node
! num_proc_per_node = 10
# number of processes to use per step per node for tasks with high i/o (dppp or cp) or memory (eg calibration)
! num_proc_per_node_limit = 4
# number of threads per process for NDPPP
! max_dppp_threads = 8
# set this to True if you want the pipeline run to continue if single bands fail
! error_tolerance = False
### Stuff that you probably don't need to modify
pipeline.steps = [calibrator_pipeline, target_pipeline]
# run the calibrator pipeline
calibrator_pipeline.control.kind = pipeline
calibrator_pipeline.control.type = {{ calibrator_parset }}
calibrator_pipeline.argument.avg_timestep = {{ avg_timestep }}
calibrator_pipeline.argument.avg_freqstep = {{ avg_freqstep }}
calibrator_pipeline.argument.cal_input_path = {{ cal_input_path }}
calibrator_pipeline.argument.cal_input_pattern = {{ cal_input_pattern }}
calibrator_pipeline.argument.calibrator_path_skymodel = {{ calibrator_path_skymodel }}
calibrator_pipeline.argument.inspection_directory = {{ inspection_directory }}
calibrator_pipeline.argument.cal_values_directory = {{ cal_values_directory }}
calibrator_pipeline.argument.flag_baselines = {{ flag_baselines }}
calibrator_pipeline.argument.reference_station = {{ reference_station }}
calibrator_pipeline.argument.calib_cal_parset = {{ calib_cal_parset }}
calibrator_pipeline.argument.find_skymodel_cal_auto = {{ find_skymodel_cal_auto }}
calibrator_pipeline.argument.losoto_importer = {{ losoto_importer }}
calibrator_pipeline.argument.fitclock_script = {{ fitclock_script }}
calibrator_pipeline.argument.fitamps_script = {{ fitamps_script }}
calibrator_pipeline.argument.plotsols_script = {{ plotsols_script }}
calibrator_pipeline.argument.fit_XYoffset_script = {{ fit_XYoffset_script }}
calibrator_pipeline.argument.plotphases_script = {{ plotphases_script }}
calibrator_pipeline.argument.losoto_executable = {{ losoto_executable }}
calibrator_pipeline.argument.num_proc_per_node = {{ num_proc_per_node }}
calibrator_pipeline.argument.num_proc_per_node_limit = {{ num_proc_per_node_limit }}
calibrator_pipeline.argument.max_dppp_threads = {{ max_dppp_threads }}
calibrator_pipeline.argument.error_tolerance = {{ error_tolerance }}
# run the target pipeline
target_pipeline.control.kind = pipeline
target_pipeline.control.type = {{ target_parset }}
target_pipeline.argument.avg_timestep = {{ avg_timestep }}
target_pipeline.argument.avg_freqstep = {{ avg_freqstep }}
target_pipeline.argument.target_input_path = {{ target_input_path }}
target_pipeline.argument.target_input_pattern = {{ target_input_pattern }}
target_pipeline.argument.target_skymodel = {{ target_skymodel }}
target_pipeline.argument.get_tgss_skymodel_script = {{ get_tgss_skymodel_script }}
target_pipeline.argument.num_SBs_per_group = {{ num_SBs_per_group }}
target_pipeline.argument.inspection_directory = {{ inspection_directory }}
target_pipeline.argument.cal_values_directory = {{ cal_values_directory }}
target_pipeline.argument.results_directory = {{ results_directory }}
target_pipeline.argument.ionex_server = {{ ionex_server }}
target_pipeline.argument.ionex_prefix = {{ ionex_prefix }}
target_pipeline.argument.ionex_path = {{ ionex_path }}
target_pipeline.argument.get_ion_script = {{ get_ion_script }}
target_pipeline.argument.flag_baselines = {{ flag_baselines }}
target_pipeline.argument.min_unflagged_fraction = {{ min_unflagged_fraction }}
target_pipeline.argument.reference_station = {{ reference_station }}
target_pipeline.argument.ATeam_skymodel = {{ ATeam_skymodel }}
target_pipeline.argument.losoto_importer = {{ losoto_importer }}
target_pipeline.argument.transfer_script = {{ transfer_script }}
target_pipeline.argument.ATeam_Clipper = {{ ATeam_Clipper }}
target_pipeline.argument.use_tgss_target = {{ use_tgss_target }}
target_pipeline.argument.sortmap_script = {{ sortmap_script }}
target_pipeline.argument.check_flagged_script = {{ check_flagged_script }}
target_pipeline.argument.structurefunction_script = {{ structurefunction_script }}
target_pipeline.argument.plotphases_script = {{ plotphases_script }}
target_pipeline.argument.losoto_executable = {{ losoto_executable }}
target_pipeline.argument.flagging_strategy = {{ flagging_strategy }}
target_pipeline.argument.makesourcedb = {{ makesourcedb }}
target_pipeline.argument.num_proc_per_node = {{ num_proc_per_node }}
target_pipeline.argument.num_proc_per_node_limit = {{ num_proc_per_node_limit }}
target_pipeline.argument.max_dppp_threads = {{ max_dppp_threads }}
target_pipeline.argument.error_tolerance = {{ error_tolerance }}