forked from lofar-astron/prefactor
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathInitial-Subtract-Deep.parset
618 lines (565 loc) · 41.8 KB
/
Initial-Subtract-Deep.parset
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
# Initial Subtract Deep Pipeline
#
# Adapted from (and shamelesly copied from) Factor
#
# Pipeline to do the initial subtraction of all sources
# - external version of the initsubtract step of Factor
# - will change the input-MSs (adds SUBTRACTED_DATA_HIGH and
# SUBTRACTED_DATA_ALL columns)
# - copies merged skymodels into folder with input data: "data_input_path"
# - requires combined predict-corrupt-subtract step in NDPPP
# (included in LOFAR since release 2.15)
# - in addition to the standard LOFAR software it requires wsclean
# - expects shared filesystem, that all nodes can reach all files!
# (E.g. a single workstation or compute cluster with shared filesystem
# doesn't work on multiple nodes on CEP-2 or CEP3.)
##### For this pipeline to work the data must have:
# - in the DATA column:
# - the average amplitude scale set
# - the average clock offsets removed
# - the LOFAR beam towards the phase center corrected
# - in the CORRECTED_DATA column:
# - the DATA calibrated with the gains in supplied the
# direction-indepndent parmDB
# - inside the measurement sets there has to be a parmDB with the
# direction-indepndent, phase-only calibration values:
# "Gain:0:0:*","Gain:1:1:*"
#
# The Pre-Facet-Cal pipelines will supply exactly this kind of data.
#### Deep imaging version
# - wsclean is used to make a deep image with channels equal to the number of input frequency bands
# - clean components are taken from the channel images/models and subtracted
##### parameters you will need to adjust.
! data_input_path = /data/scratch/username/PathToYourTargetData/
! data_input_pattern = L*.pre-cal.ms
# name of the direction independent parmDBs inside the measurement sets
# needs to include the leading "/" to indicate that the parmDB is a subdirectory of the MS
! direction_indep_parmDBs = /instrument_directionindependent
# where to put the inspection plots generated by the pipeline
! inspection_directory = /media/scratch/test/username/WhereYouWantImagesAndInspectionPlots/
# scratch directory for wsclean (can be local to the processing nodes!)
! local_scratch_dir = /local/username
##### imaging parameters (Feel free to leave them untouched.)
#### specify the image parameters here
# cellsize in degrees, 0.00208 deg is about 7.5 arcsec
! cellsize_highres_deg = 0.00208
! cellsize_lowres_deg = 0.00694
# size of the image is this value times the FWHM of the station beam
! fieldsize_highres = 2.5
! fieldsize_lowres = 6.5
# maximum uv-distance in lambda that will be used for imaging
# 7 klambda is machted to 7.5 arcsec cellsize
! maxlambda_highres = 7000
! maxlambda_lowres = 2000
# How much padding shall we add during the imaging?
! image_padding = 1.4
# How much shall the y-axis be stretched or compressed?
# Used to get rectangular images. Use at your own risk!
! axis_stretch = 1.
##### performance parameters
! max_imagers_per_node = 1
! max_percent_mem_per_img = 92
! max_cpus_per_img = 16
! max_dppp_per_node = 4
! max_dppp_threads = 4
# set this to True if you want the pipeline run to continue if single bands fail
! error_tolerance = False
##### pathes to the scripts etc.
! wsclean_executable = /soft/wsclean/bin/wsclean
! do_magic_script = /home/wwilliams/scripts/git/wprefactor/prefactor/scripts/InitSubtract_deep_sort_and_compute.py
! merge_skymodels_script = /home/wwilliams/scripts/git/wprefactor/prefactor/scripts/merge_skymodels.py
! make_clean_mask_script = /home/wwilliams/scripts/git/wprefactor/prefactor/scripts/make_clean_mask.py
! fits_to_casa_script = /home/wwilliams/scripts/git/wprefactor/prefactor/scripts/convert_fits_image_to_casa.py
! plot_subtract_images_script = /home/wwilliams/scripts/git/wprefactor/prefactor/scripts/plot_subtract_images.py
! casapy2bbs = /soft/lofar-220216/bin/casapy2bbs.py
! makesourcedb = /soft/lofar-220216/bin/makesourcedb
##### the Steps in this pipeline
pipeline.steps = [create_ms_map, combine_mapfile, do_magic, do_magic_maps, create_parmdb_map, combine_mapfile_deep, wsclean_high1_deep, createmap_high1_deep_image, mask_high_deep, wsclean_high2_deep,createmap_high2_deep_image,createmap_high2_deep_image_bands,createmap_high2_deep_model_bands, plot_im_high_deep, mk_inspect_dir, copy_mask_high_deep, fits_to_image_high_deep, casa_to_bbs_high_deep, make_sourcedb_high_deep, expand_sourcedb_high_deep, subtract_high, correct_high, combine_mapfile_low_deep, wsclean_low1_deep, createmap_low1_deep_image, mask_low_deep, wsclean_low2_deep,createmap_low2_deep_image,createmap_low2_deep_image_bands,createmap_low2_deep_model_bands, plot_im_low_deep, copy_mask_low_deep, fits_to_image_low_deep, casa_to_bbs_low_deep, make_sourcedb_low_deep, expand_sourcedb_low_deep, subtract_low, merge, copy_skymodels, createmap_plots, move_high2_deep, move_low2_deep, move_plots]
# create a mapfile with all MSs, length = nfiles
create_ms_map.control.kind = plugin
create_ms_map.control.type = createMapfile
create_ms_map.control.method = mapfile_from_folder
create_ms_map.control.mapfile_dir = input.output.mapfile_dir
create_ms_map.control.filename = create_ms_map.mapfile
create_ms_map.control.folder = {{ data_input_path }}
create_ms_map.control.pattern = {{ data_input_pattern }}
# generate a mapfile with all files in a single entry, length = 1
combine_mapfile.control.kind = plugin
combine_mapfile.control.type = createMapfile
combine_mapfile.control.method = mapfile_all_to_one
combine_mapfile.control.mapfile_in = create_ms_map.output.mapfile
combine_mapfile.control.mapfile_dir = input.output.mapfile_dir
combine_mapfile.control.filename = combine_mapfile.mapfile
# compute frequency groupings, image sizes, averaging values, etc., len = different
# the deep version of the do_magic_script should return mapfiles sorted by increasing frequency
do_magic.control.type = pythonplugin
do_magic.control.executable = {{ do_magic_script }}
do_magic.argument.flags = [combine_mapfile.output.mapfile]
do_magic.argument.outmapname = do_magic.datamap
do_magic.argument.mapfile_dir = input.output.mapfile_dir
do_magic.argument.cellsize_highres_deg = {{ cellsize_highres_deg }}
do_magic.argument.cellsize_lowres_deg = {{ cellsize_lowres_deg }}
do_magic.argument.fieldsize_highres = {{ fieldsize_highres }}
do_magic.argument.fieldsize_lowres = {{ fieldsize_lowres }}
do_magic.argument.image_padding = {{ image_padding }}
do_magic.argument.y_axis_stretch = {{ axis_stretch }}
# convert the output of do_magic into usable mapfiles,len = 1 / different
do_magic_maps.control.kind = plugin
do_magic_maps.control.type = mapfilenamesFromMapfiles
do_magic_maps.control.mapfile_nbands_map = do_magic.output.nbands.mapfile
do_magic_maps.control.mapfile_nchansout_clean1_map = do_magic.output.nchansout_clean1.mapfile
do_magic_maps.control.mapfile_groupmap = do_magic.output.groupmap.mapfile
do_magic_maps.control.mapfile_single_map = do_magic.output.single_mapfile.mapfile
do_magic_maps.control.mapfile_high_size_map = do_magic.output.high_size_mapfile.mapfile
do_magic_maps.control.mapfile_low_size_map = do_magic.output.low_size_mapfile.mapfile
do_magic_maps.control.mapfile_freqstep_map = do_magic.output.freqstep.mapfile
do_magic_maps.control.mapfile_timestep_map = do_magic.output.timestep.mapfile
do_magic_maps.control.mapfile_high_padsize_map = do_magic.output.high_padsize_mapfile.mapfile
do_magic_maps.control.mapfile_low_padsize_map = do_magic.output.low_padsize_mapfile.mapfile
do_magic_maps.control.mapfile_deep_high_size_map = do_magic.output.deep_high_size_mapfile.mapfile
do_magic_maps.control.mapfile_deep_low_size_map = do_magic.output.deep_low_size_mapfile.mapfile
do_magic_maps.control.mapfile_deep_high_padsize_map = do_magic.output.deep_high_padsize_mapfile.mapfile
do_magic_maps.control.mapfile_deep_low_padsize_map = do_magic.output.deep_low_padsize_mapfile.mapfile
do_magic_maps.control.mapfile_nwavelengths_high_map = do_magic.output.nwavelengths_high_mapfile.mapfile
do_magic_maps.control.mapfile_nwavelengths_low_map = do_magic.output.nwavelengths_low_mapfile.mapfile
# create a mapfile with the direction independent parmDBs, length = nfiles
create_parmdb_map.control.kind = plugin
create_parmdb_map.control.type = createMapfile
create_parmdb_map.control.method = add_suffix_to_file
create_parmdb_map.control.mapfile_in = do_magic_maps.output.single_map
create_parmdb_map.control.add_suffix_to_file = {{ direction_indep_parmDBs }}
create_parmdb_map.control.mapfile_dir = input.output.mapfile_dir
create_parmdb_map.control.filename = dir_indep_instrument_parmdbs.mapfile
# generate a mapfile with all files in a single entry, length = 1
combine_mapfile_deep.control.kind = plugin
combine_mapfile_deep.control.type = createMapfile
combine_mapfile_deep.control.method = mapfile_all_to_one
combine_mapfile_deep.control.mapfile_in = do_magic_maps.output.single_map
combine_mapfile_deep.control.mapfile_dir = input.output.mapfile_dir
combine_mapfile_deep.control.filename = combine_mapfile_deep.mapfile
# first high-res imaging, length = nbands
wsclean_high1_deep.control.kind = recipe
wsclean_high1_deep.control.type = executable_args
wsclean_high1_deep.control.executable = {{ wsclean_executable }}
wsclean_high1_deep.control.outputsuffixes = [-MFS-image.fits,-MFS-model.fits]
wsclean_high1_deep.control.outputkey = name
wsclean_high1_deep.control.args_format = wsclean
wsclean_high1_deep.control.max_per_node = 1
wsclean_high1_deep.control.error_tolerance = {{ error_tolerance }}
wsclean_high1_deep.control.mapfiles_in = [combine_mapfile_deep.output.mapfile,do_magic_maps.output.deep_high_padsize_map,do_magic_maps.output.deep_high_size_map,do_magic_maps.output.nchansout_clean1_map,do_magic_maps.output.nwavelengths_high_map]
wsclean_high1_deep.control.inputkeys = [msfile,paddedsize,outputsize,nchansout_clean1,nwavelengths]
wsclean_high1_deep.argument.flags = [-no-update-model-required,-reorder,-fitbeam,-joinchannels,msfile]
wsclean_high1_deep.argument.size = paddedsize
wsclean_high1_deep.argument.trim = outputsize
wsclean_high1_deep.argument.channelsout = nchansout_clean1
wsclean_high1_deep.argument.niter = 400000
wsclean_high1_deep.argument.threshold = 0.0
wsclean_high1_deep.argument.pol = I
wsclean_high1_deep.argument.weight = briggs 0.0
wsclean_high1_deep.argument.mgain = 0.65
wsclean_high1_deep.argument.baseline-averaging = nwavelengths
wsclean_high1_deep.argument.minuv-l = 80
wsclean_high1_deep.argument.maxuv-l = {{ maxlambda_highres }}
wsclean_high1_deep.argument.scale = {{ cellsize_highres_deg }}
wsclean_high1_deep.argument.mem = {{ max_percent_mem_per_img }}
wsclean_high1_deep.argument.j = {{ max_cpus_per_img }}
wsclean_high1_deep.argument.weighting-rank-filter = 3
wsclean_high1_deep.argument.tempdir = {{ local_scratch_dir }}
# create a map with the wsclean MFS image only
createmap_high1_deep_image.control.kind = plugin
createmap_high1_deep_image.control.type = createMapfile
createmap_high1_deep_image.control.method = mapfile_from_folder
createmap_high1_deep_image.control.mapfile_dir = input.output.mapfile_dir
createmap_high1_deep_image.control.filename = wsclean_high1_deep_image.mapfile
createmap_high1_deep_image.control.folder = input.output.working_directory/input.output.job_name
createmap_high1_deep_image.control.pattern = *wsclean_high1_deep-MFS-image.fits
# make masks for the high-res images, length = nbands
mask_high_deep.control.type = pythonplugin
mask_high_deep.control.executable = {{ make_clean_mask_script }}
mask_high_deep.control.max_per_node = {{ max_imagers_per_node }}
mask_high_deep.control.error_tolerance = {{ error_tolerance }}
mask_high_deep.control.mapfile_in = createmap_high1_deep_image.output.mapfile
mask_high_deep.control.inputkey = imagefile
mask_high_deep.control.outputkey = maskfile
mask_high_deep.argument.flags = [imagefile,maskfile]
mask_high_deep.argument.threshisl = 3.0
mask_high_deep.argument.threshpix = 5.0
mask_high_deep.argument.atrous_do = True
mask_high_deep.argument.rmsbox = (60,20)
mask_high_deep.argument.adaptive_rmsbox = True
mask_high_deep.argument.img_format = fits
mask_high_deep.argument.atrous_jmax = 3
mask_high_deep.argument.trim_by = 0.1
# second high-res imaging, length = nbands
wsclean_high2_deep.control.kind = recipe
wsclean_high2_deep.control.type = executable_args
wsclean_high2_deep.control.executable = {{ wsclean_executable }}
wsclean_high2_deep.control.outputsuffixes = [-MFS-image.fits,-MFS-model.fits]
wsclean_high2_deep.control.outputkey = name
wsclean_high2_deep.control.args_format = wsclean
wsclean_high2_deep.control.max_per_node = {{ max_imagers_per_node }}
wsclean_high2_deep.control.error_tolerance = {{ error_tolerance }}
wsclean_high2_deep.control.mapfiles_in = [combine_mapfile_deep.output.mapfile,mask_high_deep.output.mapfile,do_magic_maps.output.deep_high_padsize_map,do_magic_maps.output.deep_high_size_map,do_magic_maps.output.nbands_map,do_magic_maps.output.nwavelengths_high_map,mask_high_deep.output.threshold_5sig.mapfile]
wsclean_high2_deep.control.inputkeys = [msfile,fitsmask,paddedsize,outputsize,nbands,nwavelengths,threshold]
wsclean_high2_deep.argument.flags = [-no-update-model-required,-reorder,-fitbeam,-joinchannels,msfile]
wsclean_high2_deep.argument.fitsmask = fitsmask
wsclean_high2_deep.argument.size = paddedsize
wsclean_high2_deep.argument.trim = outputsize
wsclean_high2_deep.argument.channelsout = nbands
wsclean_high2_deep.argument.niter = 400000
wsclean_high2_deep.argument.threshold = threshold
wsclean_high2_deep.argument.pol = I
wsclean_high2_deep.argument.weight = briggs 0.0
wsclean_high2_deep.argument.mgain = 0.65
wsclean_high2_deep.argument.baseline-averaging = nwavelengths
wsclean_high2_deep.argument.minuv-l = 80
wsclean_high2_deep.argument.maxuv-l = {{ maxlambda_highres }}
wsclean_high2_deep.argument.scale = {{ cellsize_highres_deg }}
wsclean_high2_deep.argument.mem = {{ max_percent_mem_per_img }}
wsclean_high2_deep.argument.j = {{ max_cpus_per_img }}
wsclean_high2_deep.argument.weighting-rank-filter = 3
wsclean_high2_deep.argument.tempdir = {{ local_scratch_dir }}
# create a map with the wsclean MFS image only
createmap_high2_deep_image.control.kind = plugin
createmap_high2_deep_image.control.type = createMapfile
createmap_high2_deep_image.control.method = mapfile_from_folder
createmap_high2_deep_image.control.mapfile_dir = input.output.mapfile_dir
createmap_high2_deep_image.control.filename = wsclean_high2_deep_image.mapfile
createmap_high2_deep_image.control.folder = input.output.working_directory/input.output.job_name
createmap_high2_deep_image.control.pattern = *wsclean_high2_deep-MFS-image.fits
# create a map with wsclean image bands
createmap_high2_deep_image_bands.control.kind = plugin
createmap_high2_deep_image_bands.control.type = createMapfile
createmap_high2_deep_image_bands.control.method = mapfile_from_folder
createmap_high2_deep_image_bands.control.mapfile_dir = input.output.mapfile_dir
createmap_high2_deep_image_bands.control.filename = high2_deep_image_bands.mapfile
createmap_high2_deep_image_bands.control.folder = input.output.working_directory/input.output.job_name
createmap_high2_deep_image_bands.control.pattern = *wsclean_high2_deep-0*-image.fits
# create a map with the wsclean model bands
# should return a mapfile sorted by increasing channel number (i.e. frequency) to match the do_magic output
createmap_high2_deep_model_bands.control.kind = plugin
createmap_high2_deep_model_bands.control.type = createMapfile
createmap_high2_deep_model_bands.control.method = mapfile_from_folder
createmap_high2_deep_model_bands.control.mapfile_dir = input.output.mapfile_dir
createmap_high2_deep_model_bands.control.filename = high2_deep_model_bands.mapfile
createmap_high2_deep_model_bands.control.folder = input.output.working_directory/input.output.job_name
createmap_high2_deep_model_bands.control.pattern = *wsclean_high2_deep-0*-model.fits
# plot the high-res image and mask, length = nbands
plot_im_high_deep.control.type = pythonplugin
plot_im_high_deep.control.executable = {{ plot_subtract_images_script }}
plot_im_high_deep.control.error_tolerance = {{ error_tolerance }}
plot_im_high_deep.control.mapfiles_in = [createmap_high2_deep_image.output.mapfile,mask_high_deep.output.mapfile]
plot_im_high_deep.control.inputkeys = [imhigh,maskhigh]
plot_im_high_deep.control.outputkey = imname
plot_im_high_deep.argument.flags = [imhigh,maskhigh,imname]
# create the inspection_directory if needed
mk_inspect_dir.control.kind = plugin
mk_inspect_dir.control.type = makeDirectory
mk_inspect_dir.control.directory = {{ inspection_directory }}
# copy the mask images to where we want them
copy_mask_high_deep.control.kind = recipe
copy_mask_high_deep.control.type = executable_args
copy_mask_high_deep.control.executable = /bin/cp
copy_mask_high_deep.control.max_per_node = 10 #not much use to have too many of those
copy_mask_high_deep.control.mapfile_in = mask_high_deep.output.mapfile
copy_mask_high_deep.control.inputkey = source
copy_mask_high_deep.control.arguments = [source,{{ inspection_directory }}]
# convert high-res images to casa images, length = nbands
fits_to_image_high_deep.control.type = pythonplugin
fits_to_image_high_deep.control.executable = {{ fits_to_casa_script }}
fits_to_image_high_deep.control.error_tolerance = {{ error_tolerance }}
fits_to_image_high_deep.control.mapfile_in = createmap_high2_deep_model_bands.output.mapfile
fits_to_image_high_deep.control.inputkey = fitsfile
fits_to_image_high_deep.control.outputkey = imagefile
fits_to_image_high_deep.argument.flags = [fitsfile,imagefile]
fits_to_image_high_deep.argument.force_stokes_i = True
# convert high-res casa images to skymodel files, length = nbands
casa_to_bbs_high_deep.control.kind = recipe
casa_to_bbs_high_deep.control.type = executable_args
casa_to_bbs_high_deep.control.executable = {{ casapy2bbs }}
casa_to_bbs_high_deep.control.error_tolerance = {{ error_tolerance }}
casa_to_bbs_high_deep.control.mapfile_in = fits_to_image_high_deep.output.mapfile
casa_to_bbs_high_deep.control.inputkey = inputmodel
casa_to_bbs_high_deep.control.outputkey = outfile
casa_to_bbs_high_deep.control.arguments = [-n,inputmodel,outfile]
# make sourcedbs from the high-res skymodels, length = nbands
# can use outtype=blob because we'll use NDPPP
# also NDPPP doesn't wait for exclusive lock on blob-type sourceDBs
make_sourcedb_high_deep.control.kind = recipe
make_sourcedb_high_deep.control.type = executable_args
make_sourcedb_high_deep.control.executable = {{ makesourcedb }}
make_sourcedb_high_deep.control.error_tolerance = {{ error_tolerance }}
make_sourcedb_high_deep.control.args_format = lofar
make_sourcedb_high_deep.control.outputkey = out
make_sourcedb_high_deep.control.mapfile_in = casa_to_bbs_high_deep.output.mapfile
make_sourcedb_high_deep.control.inputkey = in
make_sourcedb_high_deep.argument.format = <
make_sourcedb_high_deep.argument.outtype = blob
# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles
expand_sourcedb_high_deep.control.kind = plugin
expand_sourcedb_high_deep.control.type = mapfileSingleToGroup
expand_sourcedb_high_deep.control.mapfile_in = make_sourcedb_high_deep.output.mapfile
expand_sourcedb_high_deep.control.mapfile_groups = do_magic_maps.output.groupmap
expand_sourcedb_high_deep.control.mapfile_dir = input.output.mapfile_dir
expand_sourcedb_high_deep.control.filename = expand_sourcedb_high_deep.datamap
# Predict, corrupt, and subtract the high-resolution model, length = nfiles
# The low-resolution model will be subtracted from the
# output column (SUBTRACTED_DATA_HIGH) later
subtract_high.control.type = dppp
subtract_high.control.mapfiles_in = [do_magic_maps.output.single_map,expand_sourcedb_high_deep.output.mapfile,create_parmdb_map.output.mapfile]
subtract_high.control.inputkeys = [msin,sourcedb,parmdb]
subtract_high.control.inplace = True
subtract_high.control.max_per_node = {{ max_dppp_per_node }}
subtract_high.argument.numthreads = {{ max_dppp_threads }}
subtract_high.control.error_tolerance = {{ error_tolerance }}
subtract_high.argument.msin.datacolumn = DATA
subtract_high.argument.msout = .
subtract_high.argument.msout.datacolumn = SUBTRACTED_DATA_HIGH
subtract_high.argument.steps = [subtract]
subtract_high.argument.subtract.type = predict
subtract_high.argument.subtract.sourcedb = sourcedb
subtract_high.argument.subtract.operation = subtract
subtract_high.argument.subtract.applycal.parmdb = parmdb
# Correct the SUBTRACTED_DATA_HIGH column and average to prepare for imaging, length = nfiles
correct_high.control.type = dppp
correct_high.control.mapfiles_in = [do_magic_maps.output.single_map,create_parmdb_map.output.mapfile,do_magic_maps.output.freqstep_map,do_magic_maps.output.timestep_map]
correct_high.control.inputkeys = [msin,parmdb,avg.freqstep,avg.timestep]
correct_high.control.max_per_node = {{ max_dppp_per_node }}
correct_high.argument.numthreads = {{ max_dppp_threads }}
correct_high.control.error_tolerance = {{ error_tolerance }}
correct_high.argument.msin.datacolumn = SUBTRACTED_DATA_HIGH
correct_high.argument.msout.overwrite = True
correct_high.argument.msout.writefullresflag = False
correct_high.argument.steps = [correct,avg]
correct_high.argument.correct.type = applycal
correct_high.argument.correct.parmdb = parmdb
correct_high.argument.correct.invert = True
correct_high.argument.avg.type = squash
# generate a mapfile with all files in a single entry, length = 1
combine_mapfile_low_deep.control.kind = plugin
combine_mapfile_low_deep.control.type = createMapfile
combine_mapfile_low_deep.control.method = mapfile_all_to_one
combine_mapfile_low_deep.control.mapfile_in = correct_high.output.mapfile
combine_mapfile_low_deep.control.mapfile_dir = input.output.mapfile_dir
combine_mapfile_low_deep.control.filename = combine_mapfile_low_deep.mapfile
# first low-res imaging, length = nbands
wsclean_low1_deep.control.kind = recipe
wsclean_low1_deep.control.type = executable_args
wsclean_low1_deep.control.executable = {{ wsclean_executable }}
wsclean_low1_deep.control.outputsuffixes = [-image.fits,-model.fits]
wsclean_low1_deep.control.outputkey = name
wsclean_low1_deep.control.args_format = wsclean
wsclean_low1_deep.control.max_per_node = {{ max_imagers_per_node }}
wsclean_low1_deep.control.error_tolerance = {{ error_tolerance }}
wsclean_low1_deep.control.mapfiles_in = [combine_mapfile_low_deep.output.mapfile,do_magic_maps.output.low_padsize_map,do_magic_maps.output.low_size_map,do_magic_maps.output.nchansout_clean1_map,do_magic_maps.output.nwavelengths_low_map]
wsclean_low1_deep.control.inputkeys = [msfile,paddedsize,outputsize,nchansout_clean1,nwavelengths]
wsclean_low1_deep.argument.flags = [-no-update-model-required,-reorder,-fitbeam,-joinchannels,msfile]
wsclean_low1_deep.argument.size = paddedsize
wsclean_low1_deep.argument.trim = outputsize
wsclean_low1_deep.argument.channelsout = nchansout_clean1
wsclean_low1_deep.argument.niter = 20000
wsclean_low1_deep.argument.threshold = 0.0
wsclean_low1_deep.argument.pol = I
wsclean_low1_deep.argument.weight = briggs 0.0
wsclean_low1_deep.argument.mgain = 0.65
wsclean_low1_deep.argument.baseline-averaging = nwavelengths
wsclean_low1_deep.argument.minuv-l = 80
wsclean_low1_deep.argument.maxuv-l = {{ maxlambda_lowres }}
wsclean_low1_deep.argument.scale = {{ cellsize_lowres_deg }}
wsclean_low1_deep.argument.mem = {{ max_percent_mem_per_img }}
wsclean_low1_deep.argument.j = {{ max_cpus_per_img }}
wsclean_low1_deep.argument.weighting-rank-filter = 3
wsclean_low1_deep.argument.tempdir = {{ local_scratch_dir }}
# create a map with the wsclean MFS image only
createmap_low1_deep_image.control.kind = plugin
createmap_low1_deep_image.control.type = createMapfile
createmap_low1_deep_image.control.method = mapfile_from_folder
createmap_low1_deep_image.control.mapfile_dir = input.output.mapfile_dir
createmap_low1_deep_image.control.filename = wsclean_low1_deep_image.mapfile
createmap_low1_deep_image.control.folder = input.output.working_directory/input.output.job_name
createmap_low1_deep_image.control.pattern = *wsclean_low1_deep-MFS-image.fits
# make masks for the low-res images, length = nbands
mask_low_deep.control.type = pythonplugin
mask_low_deep.control.executable = {{ make_clean_mask_script }}
mask_low_deep.control.max_per_node = {{ max_imagers_per_node }}
mask_low_deep.control.error_tolerance = {{ error_tolerance }}
mask_low_deep.control.mapfile_in = createmap_low1_deep_image.output.mapfile
mask_low_deep.control.inputkey = imagefile
mask_low_deep.control.outputkey = maskfile
mask_low_deep.argument.flags = [imagefile,maskfile]
mask_low_deep.argument.threshisl = 4
mask_low_deep.argument.threshpix = 5
mask_low_deep.argument.atrous_do = True
mask_low_deep.argument.rmsbox = (60,20)
mask_low_deep.argument.adaptive_rmsbox = True
mask_low_deep.argument.img_format = fits
mask_low_deep.argument.atrous_jmax = 3
mask_low_deep.argument.trim_by = 0.1
# second low-res imaging, length = nbands
wsclean_low2_deep.control.kind = recipe
wsclean_low2_deep.control.type = executable_args
wsclean_low2_deep.control.executable = {{ wsclean_executable }}
wsclean_low2_deep.control.outputsuffixes = [-image.fits,-model.fits]
wsclean_low2_deep.control.outputkey = name
wsclean_low2_deep.control.args_format = wsclean
wsclean_low2_deep.control.max_per_node = {{ max_imagers_per_node }}
wsclean_low2_deep.control.error_tolerance = {{ error_tolerance }}
wsclean_low2_deep.control.mapfiles_in = [combine_mapfile_low_deep.output.mapfile,mask_low_deep.output.mapfile,do_magic_maps.output.low_padsize_map,do_magic_maps.output.low_size_map,do_magic_maps.output.nbands_map,do_magic_maps.output.nwavelengths_low_map,mask_low_deep.output.threshold_5sig.mapfile]
wsclean_low2_deep.control.inputkeys = [msfile,fitsmask,paddedsize,outputsize,nbands,nwavelengths,threshold]
wsclean_low2_deep.argument.flags = [-no-update-model-required,-reorder,-fitbeam,-joinchannels,msfile]
wsclean_low2_deep.argument.fitsmask = fitsmask
wsclean_low2_deep.argument.size = paddedsize
wsclean_low2_deep.argument.trim = outputsize
wsclean_low2_deep.argument.channelsout = nbands
wsclean_low2_deep.argument.niter = 20000
wsclean_low2_deep.argument.threshold = threshold
wsclean_low2_deep.argument.pol = I
wsclean_low2_deep.argument.weight = briggs 0.0
wsclean_low2_deep.argument.mgain = 0.65
wsclean_low2_deep.argument.baseline-averaging = nwavelengths
wsclean_low2_deep.argument.minuv-l = 80
wsclean_low2_deep.argument.maxuv-l = {{ maxlambda_lowres }}
wsclean_low2_deep.argument.scale = {{ cellsize_lowres_deg }}
wsclean_low2_deep.argument.mem = {{ max_percent_mem_per_img }}
wsclean_low2_deep.argument.j = {{ max_cpus_per_img }}
wsclean_low2_deep.argument.weighting-rank-filter = 3
wsclean_low2_deep.argument.tempdir = {{ local_scratch_dir }}
# create a map with the wsclean MFS image only
createmap_low2_deep_image.control.kind = plugin
createmap_low2_deep_image.control.type = createMapfile
createmap_low2_deep_image.control.method = mapfile_from_folder
createmap_low2_deep_image.control.mapfile_dir = input.output.mapfile_dir
createmap_low2_deep_image.control.filename = wsclean_low2_deep_image.mapfile
createmap_low2_deep_image.control.folder = input.output.working_directory/input.output.job_name
createmap_low2_deep_image.control.pattern = *wsclean_low2_deep-MFS-image.fits
# create a map with wsclean image bands
createmap_low2_deep_image_bands.control.kind = plugin
createmap_low2_deep_image_bands.control.type = createMapfile
createmap_low2_deep_image_bands.control.method = mapfile_from_folder
createmap_low2_deep_image_bands.control.mapfile_dir = input.output.mapfile_dir
createmap_low2_deep_image_bands.control.filename = low2_deep_image_bands.mapfile
createmap_low2_deep_image_bands.control.folder = input.output.working_directory/input.output.job_name
createmap_low2_deep_image_bands.control.pattern = *wsclean_low2_deep-0*-image.fits
# create a map with the wsclean model bands
createmap_low2_deep_model_bands.control.kind = plugin
createmap_low2_deep_model_bands.control.type = createMapfile
createmap_low2_deep_model_bands.control.method = mapfile_from_folder
createmap_low2_deep_model_bands.control.mapfile_dir = input.output.mapfile_dir
createmap_low2_deep_model_bands.control.filename = low2_deep_model_bands.mapfile
createmap_low2_deep_model_bands.control.folder = input.output.working_directory/input.output.job_name
createmap_low2_deep_model_bands.control.pattern = *wsclean_low2_deep-0*-model.fits
# plot the low-res image and mask, length = nbands
plot_im_low_deep.control.type = pythonplugin
plot_im_low_deep.control.executable = {{ plot_subtract_images_script }}
plot_im_low_deep.control.error_tolerance = {{ error_tolerance }}
plot_im_low_deep.control.mapfiles_in = [createmap_low2_deep_image.output.mapfile,mask_low_deep.output.mapfile]
plot_im_low_deep.control.inputkeys = [imlow,masklow]
plot_im_low_deep.control.outputkey = imname
plot_im_low_deep.argument.flags = [imlow,masklow,imname]
# copy the mask images to where we want them
copy_mask_low_deep.control.kind = recipe
copy_mask_low_deep.control.type = executable_args
copy_mask_low_deep.control.executable = /bin/cp
copy_mask_low_deep.control.max_per_node = 10 #not much use to have too many of those
copy_mask_low_deep.control.mapfile_in = mask_low_deep.output.mapfile
copy_mask_low_deep.control.inputkey = source
copy_mask_low_deep.control.arguments = [source,{{ inspection_directory }}]
# convert low-res images to casa images, length = nbands
fits_to_image_low_deep.control.type = pythonplugin
fits_to_image_low_deep.control.executable = {{ fits_to_casa_script }}
fits_to_image_low_deep.control.error_tolerance = {{ error_tolerance }}
fits_to_image_low_deep.control.mapfile_in = createmap_low2_deep_model_bands.output.mapfile
fits_to_image_low_deep.control.inputkey = fitsfile
fits_to_image_low_deep.control.outputkey = imagefile
fits_to_image_low_deep.argument.flags = [fitsfile,imagefile]
fits_to_image_low_deep.argument.force_stokes_i = True
# convert low-res casa images to skymodel files, length = nbands
casa_to_bbs_low_deep.control.kind = recipe
casa_to_bbs_low_deep.control.type = executable_args
casa_to_bbs_low_deep.control.executable = {{ casapy2bbs }}
casa_to_bbs_low_deep.control.error_tolerance = {{ error_tolerance }}
casa_to_bbs_low_deep.control.mapfile_in = fits_to_image_low_deep.output.mapfile
casa_to_bbs_low_deep.control.inputkey = inputmodel
casa_to_bbs_low_deep.control.outputkey = outfile
casa_to_bbs_low_deep.control.arguments = [-n,inputmodel,outfile]
# make sourcedbs from the low-res skymodels, length = nbands
# outtype = blob for the same reasons as for sourcedb_high
make_sourcedb_low_deep.control.kind = recipe
make_sourcedb_low_deep.control.type = executable_args
make_sourcedb_low_deep.control.executable = {{ makesourcedb }}
make_sourcedb_low_deep.control.error_tolerance = {{ error_tolerance }}
make_sourcedb_low_deep.control.args_format = lofar
make_sourcedb_low_deep.control.outputkey = out
make_sourcedb_low_deep.control.mapfile_in = casa_to_bbs_low_deep.output.mapfile
make_sourcedb_low_deep.control.inputkey = in
make_sourcedb_low_deep.argument.format = <
make_sourcedb_low_deep.argument.outtype = blob
# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles
expand_sourcedb_low_deep.control.kind = plugin
expand_sourcedb_low_deep.control.type = mapfileSingleToGroup
expand_sourcedb_low_deep.control.mapfile_in = make_sourcedb_low_deep.output.mapfile
expand_sourcedb_low_deep.control.mapfile_groups = do_magic_maps.output.groupmap
expand_sourcedb_low_deep.control.mapfile_dir = input.output.mapfile_dir
expand_sourcedb_low_deep.control.filename = expand_sourcedb_low_deep.datamap
# Predict, corrupt, and subtract the low-resolution model, length = nfiles
subtract_low.control.type = dppp
subtract_low.control.mapfiles_in = [do_magic_maps.output.single_map,expand_sourcedb_low_deep.output.mapfile,create_parmdb_map.output.mapfile]
subtract_low.control.inputkeys = [msin,sourcedb,parmdb]
subtract_low.control.inplace = True
subtract_low.control.max_per_node = {{ max_dppp_per_node }}
subtract_low.argument.numthreads = {{ max_dppp_threads }}
subtract_low.control.error_tolerance = {{ error_tolerance }}
subtract_low.argument.msin.datacolumn = SUBTRACTED_DATA_HIGH
subtract_low.argument.msout = .
subtract_low.argument.msout.datacolumn = SUBTRACTED_DATA_ALL
subtract_low.argument.steps = [subtract]
subtract_low.argument.subtract.type = predict
subtract_low.argument.subtract.sourcedb = sourcedb
subtract_low.argument.subtract.operation = subtract
subtract_low.argument.subtract.applycal.parmdb = parmdb
# merge the high-res and low-res skymodels, length = nbands
merge.control.type = pythonplugin
merge.control.executable = {{ merge_skymodels_script }}
merge.control.error_tolerance = {{ error_tolerance }}
merge.control.mapfiles_in = [casa_to_bbs_low_deep.output.mapfile,casa_to_bbs_high_deep.output.mapfile]
merge.control.inputkeys = [modellow,modelhigh]
merge.control.outputkey = modelmerged
merge.argument.flags = [modellow,modelhigh,modelmerged]
# copy the merged skymodels to the directory with the input data, length = nbands
copy_skymodels.control.kind = recipe
copy_skymodels.control.type = executable_args
copy_skymodels.control.executable = /bin/cp
copy_skymodels.control.mapfile_in = merge.output.mapfile
copy_skymodels.control.inputkey = source
copy_skymodels.control.arguments = [source,{{ data_input_path }}]
# create a map with the generated plots
createmap_plots.control.kind = plugin
createmap_plots.control.type = createMapfile
createmap_plots.control.method = mapfile_from_folder
createmap_plots.control.mapfile_dir = input.output.mapfile_dir
createmap_plots.control.filename = diagnostic_plots.mapfile
createmap_plots.control.folder = input.output.working_directory/input.output.job_name
createmap_plots.control.pattern = *.png
# move the high2 images to where we want them
move_high2_deep.control.kind = recipe
move_high2_deep.control.type = executable_args
move_high2_deep.control.executable = /bin/mv
move_high2_deep.control.max_per_node = 10 #not much use to have too many of those
move_high2_deep.control.mapfile_in = createmap_high2_deep_image.output.mapfile
move_high2_deep.control.inputkey = source
move_high2_deep.control.arguments = [source,{{ inspection_directory }}]
# move the low2 images to where we want them
move_low2_deep.control.kind = recipe
move_low2_deep.control.type = executable_args
move_low2_deep.control.executable = /bin/mv
move_low2_deep.control.max_per_node = 10 #not much use to have too many of those
move_low2_deep.control.mapfile_in = createmap_low2_deep_image.output.mapfile
move_low2_deep.control.inputkey = source
move_low2_deep.control.arguments = [source,{{ inspection_directory }}]
# move the plots to the directory with the input data, length = 2x nbands
move_plots.control.kind = recipe
move_plots.control.type = executable_args
move_plots.control.executable = /bin/mv
move_plots.control.mapfile_in = createmap_plots.output.mapfile
move_plots.control.inputkey = source
move_plots.control.arguments = [source,{{ inspection_directory }}]