forked from lofar-astron/prefactor
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathInitial-Subtract-Fast.parset
664 lines (602 loc) · 41.4 KB
/
Initial-Subtract-Fast.parset
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
# Initial Subtract Fast Pipeline
#
#
# Pipeline to do the initial subtraction of all sources
# - will change the input-MSs (adds SUBTRACTED_DATA_HIGH and
# SUBTRACTED_DATA_ALL columns)
# - copies merged skymodels into folder with input data: "data_input_path"
# - requires combined predict-corrupt-subtract step in NDPPP
# (included in LOFAR since release 2.15)
# - in addition to the standard LOFAR software it requires wsclean
# - expects shared filesystem, that all nodes can reach all files!
# (E.g. a single workstation or compute cluster with shared filesystem;
# doesn't work on multiple nodes on CEP3.)
##### For this pipeline to work the data must have:
# - in the DATA column:
# - the average amplitude scale set
# - the average clock offsets removed
# - the LOFAR beam towards the phase center corrected
# - in the CORRECTED_DATA column:
# - the DATA calibrated with the gains in supplied the
# direction-indepndent parmDB
# - inside the measurement sets there has to be a parmDB with the
# direction-indepndent, phase-only calibration values:
# "Gain:0:0:*","Gain:1:1:*"
#
# The Pre-Facet-Cal pipelines will supply exactly this kind of data.
#### Fast version
# - ~ 6 times faster than the normal Initial-Subtract pipeline with default
# parameters
# - only a part of the full bandwidth is imaged and model components are
# interpolated to all frequencies
# - only one high-res and one low-res image is made for each band imaged, with
# a mask used to filter the clean components so that they are (mostly) real
# sources
# - WSClean's baseline-dependent averaging is used to speed up imaging
# - WSClean's auto masking is used to reduce the number of artifacts included
# in the subtracted model
# - Rectangular images are made to speed up imaging at lower declinations (where
# the primary beam is elongated N-S)
##### parameters you will need to adjust.
! data_input_path = /data/scratch/username/PathToYourTargetData/
! data_input_pattern = L*.pre-cal.ms
# name of the direction independent parmDBs inside the measurement sets
# needs to include the leading "/" to indicate that the parmDB is a subdirectory of the MS
! direction_indep_parmDBs = /instrument_directionindependent
# where to put the inspection plots generated by the pipeline
! inspection_directory = /media/scratch/test/username/WhereYouWantImagesAndInspectionPlots/
# scratch directory for wsclean (can be local to the processing nodes!)
! local_scratch_dir = /local/username
##### imaging parameters (Feel free to leave them untouched.)
#### specify the image parameters here
# cellsize in degrees, 0.00208 deg is about 7.5 arcsec
! cellsize_highres_deg = 0.00208
! cellsize_lowres_deg = 0.00694
# size of the image is this value times the FWHM of mean semi-major axis of the station beam
! fieldsize_highres = 2.5
! fieldsize_lowres = 6.5
# maximum uv-distance in lambda that will be used for imaging
# 7 klambda is machted to 7.5 arcsec cellsize
! maxlambda_highres = 7000
! maxlambda_lowres = 2000
# How much padding shall we add during the imaging?
! image_padding = 1.4
# Number of bands to image (spread over the full bandwidth). Larger values
# result in better subtraction but longer runtimes
! nbands_image = 6
# Minimum flux density in Jy of clean components from the high-resolution
# imaging to include in subtract_high step (note that all components from the
# low-resolution imaging are included in the subtract_low step). Clean
# components below this flux density will not be subtracted. Lower values result
# in better subtraction but longer runtimes
! min_flux_jy = 0.005
##### performance parameters
! max_imagers_per_node = 2
! max_percent_mem_per_img = 45
! max_cpus_per_img = 20
! max_dppp_per_node = 6
! max_dppp_threads = 8
# set this to True if you want the pipeline run to continue if single bands fail
! error_tolerance = False
##### pathes to the scripts etc.
! wsclean_executable = /homea/htb00/htb003/local_jureca/bin/wsclean
! do_magic_script = /homea/htb00/htb001/prefactor/scripts/InitSubtract_sort_and_compute.py
! merge_skymodels_script = /homea/htb00/htb001/prefactor/scripts/merge_skymodels.py
! make_clean_mask_script = /homea/htb00/htb001/prefactor/scripts/make_clean_mask.py
! plot_subtract_images_script = /homea/htb00/htb001/prefactor/scripts/plot_subtract_images.py
! fits2sky_script = /homea/htb00/htb001/prefactor/scripts/fits2sky.py
! pad_image_script = /homea/htb00/htb001/prefactor/scripts/pad_image.py
! makesourcedb = /homea/htb00/htb003/lofar_jureca_2-15/bin/makesourcedb
# the Steps in this pipeline
pipeline.steps = [create_ms_map, combine_mapfile, do_magic, do_magic_maps, create_parmdb_map, select_imaging_bands, select_high_padsize, select_high_size, select_high_nwavelengths, wsclean_high, mask_high, mk_inspect_dir, copy_mask, plot_im_high, move_high, create_maxsize_high_map, pad_model_high, pad_mask_high, combine_model_high_mapfile, expand_model_high, combine_mask_high_mapfile, expand_mask_high, fits_to_bbs_high, make_sourcedb_high, expand_sourcedb_high, subtract_high, select_correct_high_files, select_parmdb, select_freqstep, select_timestep, correct_high, regroup_map, select_low_padsize, select_low_size, select_low_nwavelengths, wsclean_low, mask_low, plot_im_low, move_low, create_maxsize_low_map, pad_model_low, pad_mask_low, combine_model_low_mapfile, expand_model_low, combine_mask_low_mapfile, expand_mask_low, fits_to_bbs_low, make_sourcedb_low, expand_sourcedb_low, subtract_low, merge, copy_skymodels, createmap_plots, move_plots]
# create a mapfile with all MSs, length = nfiles
create_ms_map.control.kind = plugin
create_ms_map.control.type = createMapfile
create_ms_map.control.method = mapfile_from_folder
create_ms_map.control.mapfile_dir = input.output.mapfile_dir
create_ms_map.control.filename = create_ms_map.mapfile
create_ms_map.control.folder = {{ data_input_path }}
create_ms_map.control.pattern = {{ data_input_pattern }}
# generate a mapfile with all files in a single entry, length = 1
combine_mapfile.control.kind = plugin
combine_mapfile.control.type = createMapfile
combine_mapfile.control.method = mapfile_all_to_one
combine_mapfile.control.mapfile_in = create_ms_map.output.mapfile
combine_mapfile.control.mapfile_dir = input.output.mapfile_dir
combine_mapfile.control.filename = combine_mapfile.mapfile
# compute frequency groupings, image sizes, averaging values, etc., len = different
do_magic.control.type = pythonplugin
do_magic.control.executable = {{ do_magic_script }}
do_magic.argument.flags = [combine_mapfile.output.mapfile]
do_magic.argument.outmapname = do_magic.datamap
do_magic.argument.mapfile_dir = input.output.mapfile_dir
do_magic.argument.cellsize_highres_deg = {{ cellsize_highres_deg }}
do_magic.argument.cellsize_lowres_deg = {{ cellsize_lowres_deg }}
do_magic.argument.fieldsize_highres = {{ fieldsize_highres }}
do_magic.argument.fieldsize_lowres = {{ fieldsize_lowres }}
do_magic.argument.image_padding = {{ image_padding }}
do_magic.argument.calc_y_axis_stretch = True
# convert the output of do_magic into usable mapfiles, len = 1 / different
do_magic_maps.control.kind = plugin
do_magic_maps.control.type = mapfilenamesFromMapfiles
do_magic_maps.control.mapfile_groupmap = do_magic.output.groupmap.mapfile
do_magic_maps.control.mapfile_single_map = do_magic.output.single_mapfile.mapfile
do_magic_maps.control.mapfile_high_size_map = do_magic.output.high_size_mapfile.mapfile
do_magic_maps.control.mapfile_low_size_map = do_magic.output.low_size_mapfile.mapfile
do_magic_maps.control.mapfile_freqstep_map = do_magic.output.freqstep.mapfile
do_magic_maps.control.mapfile_timestep_map = do_magic.output.timestep.mapfile
do_magic_maps.control.mapfile_high_padsize_map = do_magic.output.high_padsize_mapfile.mapfile
do_magic_maps.control.mapfile_low_padsize_map = do_magic.output.low_padsize_mapfile.mapfile
do_magic_maps.control.mapfile_nwavelengths_high_map = do_magic.output.nwavelengths_high_mapfile.mapfile
do_magic_maps.control.mapfile_nwavelengths_low_map = do_magic.output.nwavelengths_low_mapfile.mapfile
# create a mapfile with the direction independent parmDBs, length = nfiles
create_parmdb_map.control.kind = plugin
create_parmdb_map.control.type = createMapfile
create_parmdb_map.control.method = add_suffix_to_file
create_parmdb_map.control.mapfile_in = do_magic_maps.output.single_map
create_parmdb_map.control.add_suffix_to_file = {{ direction_indep_parmDBs }}
create_parmdb_map.control.mapfile_dir = input.output.mapfile_dir
create_parmdb_map.control.filename = dir_indep_instrument_parmdbs.mapfile
# select bands spread over the full bandwidth for imaging, length = nbands_image
select_imaging_bands.control.kind = plugin
select_imaging_bands.control.type = selectDistFreqs
select_imaging_bands.control.mapfile_in = do_magic_maps.output.groupmap
select_imaging_bands.control.num = {{ nbands_image }}
select_imaging_bands.control.mapfile_dir = input.output.mapfile_dir
select_imaging_bands.control.filename = imaging_high_bands.mapfile
# adjust the high_padsize mapfile to match the selected bands, length = nbands_image
select_high_padsize.control.kind = plugin
select_high_padsize.control.type = matchDistFreqs
select_high_padsize.control.mapfile_in = do_magic_maps.output.high_padsize_map
select_high_padsize.control.mapfile_dist = select_imaging_bands.output.mapfile
select_high_padsize.control.mapfile_full = do_magic_maps.output.groupmap
select_high_padsize.control.mapfile_dir = input.output.mapfile_dir
select_high_padsize.control.filename = imaging_high_padsize.mapfile
# adjust the high_size mapfile to match the selected bands, length = nbands_image
select_high_size.control.kind = plugin
select_high_size.control.type = matchDistFreqs
select_high_size.control.mapfile_in = do_magic_maps.output.high_size_map
select_high_size.control.mapfile_dist = select_imaging_bands.output.mapfile
select_high_size.control.mapfile_full = do_magic_maps.output.groupmap
select_high_size.control.mapfile_dir = input.output.mapfile_dir
select_high_size.control.filename = imaging_high_size.mapfile
# adjust the nwavelengths mapfile to match the selected bands, length = nbands_image
select_high_nwavelengths.control.kind = plugin
select_high_nwavelengths.control.type = matchDistFreqs
select_high_nwavelengths.control.mapfile_in = do_magic_maps.output.nwavelengths_high_map
select_high_nwavelengths.control.mapfile_dist = select_imaging_bands.output.mapfile
select_high_nwavelengths.control.mapfile_full = do_magic_maps.output.groupmap
select_high_nwavelengths.control.mapfile_dir = input.output.mapfile_dir
select_high_nwavelengths.control.filename = imaging_high_nwavelengths.mapfile
# high-res imaging, length = nbands_image
wsclean_high.control.kind = recipe
wsclean_high.control.type = executable_args
wsclean_high.control.executable = {{ wsclean_executable }}
wsclean_high.control.outputsuffixes = [-image.fits,-model.fits]
wsclean_high.control.outputkey = name
wsclean_high.control.args_format = wsclean
wsclean_high.control.max_per_node = {{ max_imagers_per_node }}
wsclean_high.control.error_tolerance = {{ error_tolerance }}
wsclean_high.control.mapfiles_in = [select_imaging_bands.output.mapfile,select_high_padsize.output.mapfile,select_high_size.output.mapfile,select_high_nwavelengths.output.mapfile]
wsclean_high.control.inputkeys = [msfile,paddedsize,outputsize,nwavelengths]
wsclean_high.argument.flags = [-no-update-model-required,-reorder,-local-rms,-fitbeam,msfile]
wsclean_high.argument.size = paddedsize
wsclean_high.argument.trim = outputsize
wsclean_high.argument.niter = 50000
wsclean_high.argument.threshold = 0.0
wsclean_high.argument.pol = I
wsclean_high.argument.weight = briggs 0.0
wsclean_high.argument.mgain = 0.8
wsclean_high.argument.baseline-averaging = nwavelengths
wsclean_high.argument.maxuv-l = {{ maxlambda_highres }}
wsclean_high.argument.scale = {{ cellsize_highres_deg }}
wsclean_high.argument.mem = {{ max_percent_mem_per_img }}
wsclean_high.argument.j = {{ max_cpus_per_img }}
wsclean_high.argument.tempdir = {{ local_scratch_dir }}
wsclean_high.argument.weighting-rank-filter = 3
wsclean_high.argument.auto-mask = 3.3
wsclean_high.argument.auto-threshold = 0.5
wsclean_high.argument.local-rms-window = 50
wsclean_high.argument.local-rms-method = rms-with-min
# make masks for the high-res images, length = nbands_image
mask_high.control.type = pythonplugin
mask_high.control.executable = {{ make_clean_mask_script }}
mask_high.control.max_per_node = {{ max_imagers_per_node }}
mask_high.control.error_tolerance = {{ error_tolerance }}
mask_high.control.mapfile_in = wsclean_high.output.wsclean_high-image.fits.mapfile
mask_high.control.inputkey = imagefile
mask_high.control.outputkey = maskfile
mask_high.argument.flags = [imagefile,maskfile]
mask_high.argument.threshisl = 3.0
mask_high.argument.threshpix = 5.0
mask_high.argument.atrous_do = True
mask_high.argument.rmsbox = (60,20)
mask_high.argument.adaptive_rmsbox = True
mask_high.argument.img_format = fits
mask_high.argument.atrous_jmax = 3
mask_high.argument.trim_by = 0.1
# create the inspection_directory if needed
mk_inspect_dir.control.kind = plugin
mk_inspect_dir.control.type = makeDirectory
mk_inspect_dir.control.directory = {{ inspection_directory }}
# copy the mask images to where we want them
copy_mask.control.kind = recipe
copy_mask.control.type = executable_args
copy_mask.control.executable = /bin/cp
copy_mask.control.max_per_node = 10 #not much use to have too many of those
copy_mask.control.mapfile_in = mask_high.output.mapfile
copy_mask.control.inputkey = source
copy_mask.control.arguments = [source,{{ inspection_directory }}]
# plot the high-res image and mask, length = nbands_image
plot_im_high.control.type = pythonplugin
plot_im_high.control.executable = {{ plot_subtract_images_script }}
plot_im_high.control.error_tolerance = {{ error_tolerance }}
plot_im_high.control.mapfiles_in = [wsclean_high.output.wsclean_high-image.fits.mapfile,mask_high.output.mapfile]
plot_im_high.control.inputkeys = [imhigh,maskhigh]
plot_im_high.control.outputkey = imname
plot_im_high.argument.flags = [imhigh,maskhigh,imname]
# move the high-res images to where we want them
move_high.control.kind = recipe
move_high.control.type = executable_args
move_high.control.executable = /bin/mv
move_high.control.max_per_node = 10 #not much use to have too many of those
move_high.control.mapfile_in = wsclean_high.output.wsclean_high-image.fits.mapfile
move_high.control.inputkey = source
move_high.control.arguments = [source,{{ inspection_directory }}]
# make a mapfile with maximum image size, length = nbands_image
create_maxsize_high_map.control.kind = plugin
create_maxsize_high_map.control.type = mapfileMaxSize
create_maxsize_high_map.control.mapfile_in = select_high_size.output.mapfile
create_maxsize_high_map.control.mapfile_dir = input.output.mapfile_dir
create_maxsize_high_map.control.filename = maxsize_high.datamap
# Pad the model images to a uniform size, length = nbands_image
pad_model_high.control.type = pythonplugin
pad_model_high.control.executable = {{ pad_image_script }}
pad_model_high.control.error_tolerance = {{ error_tolerance }}
pad_model_high.control.mapfiles_in = [wsclean_high.output.wsclean_high-model.fits.mapfile,create_maxsize_high_map.output.mapfile]
pad_model_high.control.inputkeys = [imagefile,xypadsize]
pad_model_high.argument.flags = [imagefile,xypadsize]
# Pad the mask images to a uniform size, length = nbands_image
pad_mask_high.control.type = pythonplugin
pad_mask_high.control.executable = {{ pad_image_script }}
pad_mask_high.control.error_tolerance = {{ error_tolerance }}
pad_mask_high.control.mapfiles_in = [mask_high.output.mapfile,create_maxsize_high_map.output.mapfile]
pad_mask_high.control.inputkeys = [imagefile,xypadsize]
pad_mask_high.argument.flags = [imagefile,xypadsize]
# compress the model_high mapfile, length = 1
combine_model_high_mapfile.control.kind = plugin
combine_model_high_mapfile.control.type = compressMapfile
combine_model_high_mapfile.control.mapfile_in = wsclean_high.output.wsclean_high-model.fits.mapfile
combine_model_high_mapfile.control.mapfile_dir = input.output.mapfile_dir
combine_model_high_mapfile.control.filename = combine_model_high_mapfile.mapfile
# expand the model_high mapfile so that there is one entry for every band, length = nbands
expand_model_high.control.kind = plugin
expand_model_high.control.type = expandMapfile
expand_model_high.control.mapfile_in = combine_model_high_mapfile.output.mapfile
expand_model_high.control.mapfile_to_match = do_magic_maps.output.groupmap
expand_model_high.control.mapfile_dir = input.output.mapfile_dir
expand_model_high.control.filename = expand_model_high.datamap
# compress the mask_high mapfile, length = 1
combine_mask_high_mapfile.control.kind = plugin
combine_mask_high_mapfile.control.type = compressMapfile
combine_mask_high_mapfile.control.mapfile_in = mask_high.output.mapfile
combine_mask_high_mapfile.control.mapfile_dir = input.output.mapfile_dir
combine_mask_high_mapfile.control.filename = combine_mask_high_mapfile.mapfile
# expand the mask high mapfile so that there is one entry for every band, length = nbands
expand_mask_high.control.kind = plugin
expand_mask_high.control.type = expandMapfile
expand_mask_high.control.mapfile_in = combine_mask_high_mapfile.output.mapfile
expand_mask_high.control.mapfile_to_match = do_magic_maps.output.groupmap
expand_mask_high.control.mapfile_dir = input.output.mapfile_dir
expand_mask_high.control.filename = expand_mask_high.datamap
# convert high-res model images to sky models, length = nbands
fits_to_bbs_high.control.type = pythonplugin
fits_to_bbs_high.control.executable = {{ fits2sky_script }}
fits_to_bbs_high.control.error_tolerance = {{ error_tolerance }}
fits_to_bbs_high.control.max_per_node = 1 # can use a lot of memory, so don't do too many at once
fits_to_bbs_high.control.mapfiles_in = [do_magic_maps.output.groupmap,expand_model_high.output.mapfile,expand_mask_high.output.mapfile]
fits_to_bbs_high.control.inputkeys = [msfile,rootname,mask]
fits_to_bbs_high.control.outputkey = skymodel
fits_to_bbs_high.argument.flags = [rootname,msfile,skymodel,mask]
fits_to_bbs_high.argument.min_flux_jy = {{ min_flux_jy }}
# make sourcedbs from the high-res skymodels, length = nbands
# can use outtype=blob because we'll use NDPPP
# also NDPPP doesn't wait for exclusive lock on blob-type sourceDBs
make_sourcedb_high.control.kind = recipe
make_sourcedb_high.control.type = executable_args
make_sourcedb_high.control.executable = {{ makesourcedb }}
make_sourcedb_high.control.error_tolerance = {{ error_tolerance }}
make_sourcedb_high.control.args_format = lofar
make_sourcedb_high.control.outputkey = out
make_sourcedb_high.control.mapfile_in = fits_to_bbs_high.output.mapfile
make_sourcedb_high.control.inputkey = in
make_sourcedb_high.argument.format = <
make_sourcedb_high.argument.outtype = blob
make_sourcedb_high.argument.append = False
# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles
expand_sourcedb_high.control.kind = plugin
expand_sourcedb_high.control.type = mapfileSingleToGroup
expand_sourcedb_high.control.mapfile_in = make_sourcedb_high.output.mapfile
expand_sourcedb_high.control.mapfile_groups = do_magic_maps.output.groupmap
expand_sourcedb_high.control.mapfile_dir = input.output.mapfile_dir
expand_sourcedb_high.control.filename = expand_sourcedb_high.datamap
# Predict, corrupt, and subtract the high-resolution model, length = nfiles
# The low-resolution model will be subtracted from the
# output column (SUBTRACTED_DATA_HIGH) later
subtract_high.control.type = dppp
subtract_high.control.mapfiles_in = [do_magic_maps.output.single_map,expand_sourcedb_high.output.mapfile,create_parmdb_map.output.mapfile]
subtract_high.control.inputkeys = [msin,sourcedb,parmdb]
subtract_high.control.inplace = True
subtract_high.control.max_per_node = {{ max_dppp_per_node }}
subtract_high.argument.numthreads = {{ max_dppp_threads }}
subtract_high.control.error_tolerance = {{ error_tolerance }}
subtract_high.argument.msin.datacolumn = DATA
subtract_high.argument.msout = .
subtract_high.argument.msout.datacolumn = SUBTRACTED_DATA_HIGH
subtract_high.argument.steps = [subtract]
subtract_high.argument.subtract.type = predict
subtract_high.argument.subtract.sourcedb = sourcedb
subtract_high.argument.subtract.operation = subtract
subtract_high.argument.subtract.applycal.parmdb = parmdb
# select files spread over the full bandwidth for imaging, length = nbands_image * ntimes
select_correct_high_files.control.kind = plugin
select_correct_high_files.control.type = selectDistFreqs
select_correct_high_files.control.mapfile_in = do_magic_maps.output.single_map
select_correct_high_files.control.num = {{ nbands_image }}
select_correct_high_files.control.mapfile_dir = input.output.mapfile_dir
select_correct_high_files.control.filename = select_correct_high_files.mapfile
# adjust the dir-indep parmdb mapfile to match the selected bands, length = nbands_image * ntimes
select_parmdb.control.kind = plugin
select_parmdb.control.type = matchDistFreqs
select_parmdb.control.mapfile_in = create_parmdb_map.output.mapfile
select_parmdb.control.mapfile_dist = select_correct_high_files.output.mapfile
select_parmdb.control.mapfile_full = do_magic_maps.output.single_map
select_parmdb.control.mapfile_dir = input.output.mapfile_dir
select_parmdb.control.filename = parmdb.mapfile
# adjust the freqstep mapfile to match the selected bands, length = nbands_image * ntimes
select_freqstep.control.kind = plugin
select_freqstep.control.type = matchDistFreqs
select_freqstep.control.mapfile_in = do_magic_maps.output.freqstep_map
select_freqstep.control.mapfile_dist = select_correct_high_files.output.mapfile
select_freqstep.control.mapfile_full = do_magic_maps.output.single_map
select_freqstep.control.mapfile_dir = input.output.mapfile_dir
select_freqstep.control.filename = avg_freqstep.mapfile
# adjust the timestep mapfile to match the selected bands, length = nbands_image * ntimes
select_timestep.control.kind = plugin
select_timestep.control.type = matchDistFreqs
select_timestep.control.mapfile_in = do_magic_maps.output.timestep_map
select_timestep.control.mapfile_dist = select_correct_high_files.output.mapfile
select_timestep.control.mapfile_full = do_magic_maps.output.single_map
select_timestep.control.mapfile_dir = input.output.mapfile_dir
select_timestep.control.filename = avg_timestep.mapfile
# Correct the SUBTRACTED_DATA_HIGH column and average to prepare for imaging, length = nbands_image * ntimes
correct_high.control.type = dppp
correct_high.control.mapfiles_in = [select_correct_high_files.output.mapfile,select_parmdb.output.mapfile,select_freqstep.output.mapfile,select_timestep.output.mapfile]
correct_high.control.inputkeys = [msin,parmdb,avg.freqstep,avg.timestep]
correct_high.control.max_per_node = {{ max_dppp_per_node }}
correct_high.argument.numthreads = {{ max_dppp_threads }}
correct_high.control.error_tolerance = {{ error_tolerance }}
correct_high.argument.msin.datacolumn = SUBTRACTED_DATA_HIGH
correct_high.argument.msout.overwrite = True
correct_high.argument.msout.writefullresflag = False
correct_high.argument.steps = [correct,avg]
correct_high.argument.correct.type = applycal
correct_high.argument.correct.parmdb = parmdb
correct_high.argument.correct.invert = True
correct_high.argument.avg.type = squash
# re-group mapfile to have one group per band, length = nbands_image
regroup_map.control.kind = plugin
regroup_map.control.type = reGroupMapfile
regroup_map.control.mapfile_in = correct_high.output.mapfile
regroup_map.control.mapfile_groups = select_imaging_bands.output.mapfile
regroup_map.control.mapfile_dir = input.output.mapfile_dir
regroup_map.control.filename = re_grouped.datamap
# adjust the low_padsize mapfile to match the selected bands, length = nbands_image
select_low_padsize.control.kind = plugin
select_low_padsize.control.type = matchDistFreqs
select_low_padsize.control.mapfile_in = do_magic_maps.output.low_padsize_map
select_low_padsize.control.mapfile_dist = select_imaging_bands.output.mapfile
select_low_padsize.control.mapfile_full = do_magic_maps.output.groupmap
select_low_padsize.control.mapfile_dir = input.output.mapfile_dir
select_low_padsize.control.filename = imaging_low_padsize.mapfile
# adjust the low size mapfile to match the selected bands, length = nbands_image
select_low_size.control.kind = plugin
select_low_size.control.type = matchDistFreqs
select_low_size.control.mapfile_in = do_magic_maps.output.low_size_map
select_low_size.control.mapfile_dist = select_imaging_bands.output.mapfile
select_low_size.control.mapfile_full = do_magic_maps.output.groupmap
select_low_size.control.mapfile_dir = input.output.mapfile_dir
select_low_size.control.filename = imaging_low_size.mapfile
# adjust the low nwavelengths mapfile to match the selected bands, length = nbands_image
select_low_nwavelengths.control.kind = plugin
select_low_nwavelengths.control.type = matchDistFreqs
select_low_nwavelengths.control.mapfile_in = do_magic_maps.output.nwavelengths_low_map
select_low_nwavelengths.control.mapfile_dist = select_imaging_bands.output.mapfile
select_low_nwavelengths.control.mapfile_full = do_magic_maps.output.groupmap
select_low_nwavelengths.control.mapfile_dir = input.output.mapfile_dir
select_low_nwavelengths.control.filename = imaging_low_nwavelengths.mapfile
# first low-res imaging, length = nbands_image
wsclean_low.control.kind = recipe
wsclean_low.control.type = executable_args
wsclean_low.control.executable = {{ wsclean_executable }}
wsclean_low.control.outputsuffixes = [-image.fits,-model.fits]
wsclean_low.control.outputkey = name
wsclean_low.control.args_format = wsclean
wsclean_low.control.max_per_node = {{ max_imagers_per_node }}
wsclean_low.control.error_tolerance = {{ error_tolerance }}
wsclean_low.control.mapfiles_in = [regroup_map.output.mapfile,select_low_padsize.output.mapfile,select_low_size.output.mapfile,select_low_nwavelengths.output.mapfile]
wsclean_low.control.inputkeys = [msfile,paddedsize,outputsize,nwavelengths]
wsclean_low.argument.flags = [-no-update-model-required,-reorder,-local-rms,-fitbeam,msfile]
wsclean_low.argument.size = paddedsize
wsclean_low.argument.trim = outputsize
wsclean_low.argument.niter = 25000
wsclean_low.argument.threshold = 0.0
wsclean_low.argument.pol = I
wsclean_low.argument.weight = briggs 0.0
wsclean_low.argument.mgain = 0.8
wsclean_low.argument.baseline-averaging = nwavelengths
wsclean_low.argument.maxuv-l = {{ maxlambda_lowres }}
wsclean_low.argument.scale = {{ cellsize_lowres_deg }}
wsclean_low.argument.mem = {{ max_percent_mem_per_img }}
wsclean_low.argument.j = {{ max_cpus_per_img }}
wsclean_low.argument.tempdir = {{ local_scratch_dir }}
wsclean_low.argument.weighting-rank-filter = 3
wsclean_low.argument.auto-mask = 3.3
wsclean_low.argument.auto-threshold = 0.5
wsclean_low.argument.local-rms-window = 50
wsclean_low.argument.local-rms-method = rms-with-min
# make masks for the low-res images, length = nbands_image
mask_low.control.type = pythonplugin
mask_low.control.executable = {{ make_clean_mask_script }}
mask_low.control.max_per_node = {{ max_imagers_per_node }}
mask_low.control.error_tolerance = {{ error_tolerance }}
mask_low.control.mapfile_in = wsclean_low.output.wsclean_low-image.fits.mapfile
mask_low.control.inputkey = imagefile
mask_low.control.outputkey = maskfile
mask_low.argument.flags = [imagefile,maskfile]
mask_low.argument.threshisl = 4
mask_low.argument.threshpix = 5
mask_low.argument.atrous_do = True
mask_low.argument.rmsbox = (60,20)
mask_low.argument.adaptive_rmsbox = True
mask_low.argument.img_format = fits
mask_low.argument.atrous_jmax = 3
mask_low.argument.trim_by = 0.1
# plot the low-res image and mask, length = nbands_image
plot_im_low.control.type = pythonplugin
plot_im_low.control.executable = {{ plot_subtract_images_script }}
plot_im_low.control.error_tolerance = {{ error_tolerance }}
plot_im_low.control.mapfiles_in = [wsclean_low.output.wsclean_low-image.fits.mapfile,mask_low.output.mapfile]
plot_im_low.control.inputkeys = [imlow,masklow]
plot_im_low.control.outputkey = imname
plot_im_low.argument.flags = [imlow,masklow,imname]
# move the low-res images to where we want them
move_low.control.kind = recipe
move_low.control.type = executable_args
move_low.control.executable = /bin/mv
move_low.control.max_per_node = 10 #not much use to have too many of those
move_low.control.mapfile_in = wsclean_low.output.wsclean_low-image.fits.mapfile
move_low.control.inputkey = source
move_low.control.arguments = [source,{{ inspection_directory }}]
# make a mapfile with maximum image size, length = nbands_image
create_maxsize_low_map.control.kind = plugin
create_maxsize_low_map.control.type = mapfileMaxSize
create_maxsize_low_map.control.mapfile_in = select_low_size.output.mapfile
create_maxsize_low_map.control.mapfile_dir = input.output.mapfile_dir
create_maxsize_low_map.control.filename = maxsize_low.datamap
# Pad the model images to a uniform size, length = nbands_image
pad_model_low.control.type = pythonplugin
pad_model_low.control.executable = {{ pad_image_script }}
pad_model_low.control.error_tolerance = {{ error_tolerance }}
pad_model_low.control.mapfiles_in = [wsclean_low.output.wsclean_low-model.fits.mapfile,create_maxsize_low_map.output.mapfile]
pad_model_low.control.inputkeys = [imagefile,xypadsize]
pad_model_low.argument.flags = [imagefile,xypadsize]
# Pad the mask images to a uniform size, length = nbands_image
pad_mask_low.control.type = pythonplugin
pad_mask_low.control.executable = {{ pad_image_script }}
pad_mask_low.control.error_tolerance = {{ error_tolerance }}
pad_mask_low.control.mapfiles_in = [mask_low.output.mapfile,create_maxsize_low_map.output.mapfile]
pad_mask_low.control.inputkeys = [imagefile,xypadsize]
pad_mask_low.argument.flags = [imagefile,xypadsize]
# compress the model_low mapfile, length = 1
combine_model_low_mapfile.control.kind = plugin
combine_model_low_mapfile.control.type = compressMapfile
combine_model_low_mapfile.control.mapfile_in = wsclean_low.output.wsclean_low-model.fits.mapfile
combine_model_low_mapfile.control.mapfile_dir = input.output.mapfile_dir
combine_model_low_mapfile.control.filename = combine_model_low_mapfile.mapfile
# expand the model_low mapfile so that there is one entry for every band, length = nbands
expand_model_low.control.kind = plugin
expand_model_low.control.type = expandMapfile
expand_model_low.control.mapfile_in = combine_model_low_mapfile.output.mapfile
expand_model_low.control.mapfile_to_match = do_magic_maps.output.groupmap
expand_model_low.control.mapfile_dir = input.output.mapfile_dir
expand_model_low.control.filename = expand_model_low.datamap
# compress the mask_low mapfile, length = 1
combine_mask_low_mapfile.control.kind = plugin
combine_mask_low_mapfile.control.type = compressMapfile
combine_mask_low_mapfile.control.mapfile_in = mask_low.output.mapfile
combine_mask_low_mapfile.control.mapfile_dir = input.output.mapfile_dir
combine_mask_low_mapfile.control.filename = combine_mask_low_mapfile.mapfile
# expand the mask low mapfile so that there is one entry for every band, length = nbands
expand_mask_low.control.kind = plugin
expand_mask_low.control.type = expandMapfile
expand_mask_low.control.mapfile_in = combine_mask_low_mapfile.output.mapfile
expand_mask_low.control.mapfile_to_match = do_magic_maps.output.groupmap
expand_mask_low.control.mapfile_dir = input.output.mapfile_dir
expand_mask_low.control.filename = expand_mask_low.datamap
# convert low-res model images to sky models, length = nbands
fits_to_bbs_low.control.type = pythonplugin
fits_to_bbs_low.control.executable = {{ fits2sky_script }}
fits_to_bbs_low.control.error_tolerance = {{ error_tolerance }}
fits_to_bbs_low.control.max_per_node = 4 # can use a lot of memory, so don't do too many at once
fits_to_bbs_low.control.mapfiles_in = [do_magic_maps.output.groupmap,expand_model_low.output.mapfile,expand_mask_low.output.mapfile]
fits_to_bbs_low.control.inputkeys = [msfile,rootname,mask]
fits_to_bbs_low.control.outputkey = skymodel
fits_to_bbs_low.argument.flags = [rootname,msfile,skymodel,mask]
fits_to_bbs_low.argument.min_flux_jy = 0.0 # keep all clean components, as there typically aren't too many
# make sourcedbs from the low-res skymodels, length = nbands
# outtype = blob for the same reasons as for sourcedb_high
make_sourcedb_low.control.kind = recipe
make_sourcedb_low.control.type = executable_args
make_sourcedb_low.control.executable = {{ makesourcedb }}
make_sourcedb_low.control.error_tolerance = {{ error_tolerance }}
make_sourcedb_low.control.args_format = lofar
make_sourcedb_low.control.outputkey = out
make_sourcedb_low.control.mapfile_in = fits_to_bbs_low.output.mapfile
make_sourcedb_low.control.inputkey = in
make_sourcedb_low.argument.format = <
make_sourcedb_low.argument.outtype = blob
make_sourcedb_low.argument.append = False
# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles
expand_sourcedb_low.control.kind = plugin
expand_sourcedb_low.control.type = mapfileSingleToGroup
expand_sourcedb_low.control.mapfile_in = make_sourcedb_low.output.mapfile
expand_sourcedb_low.control.mapfile_groups = do_magic_maps.output.groupmap
expand_sourcedb_low.control.mapfile_dir = input.output.mapfile_dir
expand_sourcedb_low.control.filename = expand_sourcedb_low.datamap
# Predict, corrupt, and subtract the low-resolution model, length = nfiles
subtract_low.control.type = dppp
subtract_low.control.mapfiles_in = [do_magic_maps.output.single_map,expand_sourcedb_low.output.mapfile,create_parmdb_map.output.mapfile]
subtract_low.control.inputkeys = [msin,sourcedb,parmdb]
subtract_low.control.inplace = True
subtract_low.control.max_per_node = {{ max_dppp_per_node }}
subtract_low.argument.numthreads = {{ max_dppp_threads }}
subtract_low.control.error_tolerance = {{ error_tolerance }}
subtract_low.argument.msin.datacolumn = SUBTRACTED_DATA_HIGH
subtract_low.argument.msout = .
subtract_low.argument.msout.datacolumn = SUBTRACTED_DATA_ALL
subtract_low.argument.steps = [subtract]
subtract_low.argument.subtract.type = predict
subtract_low.argument.subtract.sourcedb = sourcedb
subtract_low.argument.subtract.operation = subtract
subtract_low.argument.subtract.applycal.parmdb = parmdb
# merge the high-res and low-res skymodels, length = nbands
merge.control.type = pythonplugin
merge.control.executable = {{ merge_skymodels_script }}
merge.control.error_tolerance = {{ error_tolerance }}
merge.control.mapfiles_in = [fits_to_bbs_low.output.mapfile,fits_to_bbs_high.output.mapfile]
merge.control.inputkeys = [modellow,modelhigh]
merge.control.outputkey = modelmerged
merge.argument.flags = [modellow,modelhigh,modelmerged]
# copy the merged skymodels to the directory with the input data, length = nbands
copy_skymodels.control.kind = recipe
copy_skymodels.control.type = executable_args
copy_skymodels.control.executable = /bin/cp
copy_skymodels.control.mapfile_in = merge.output.mapfile
copy_skymodels.control.inputkey = source
copy_skymodels.control.arguments = [source,{{ data_input_path }}]
# create a map with the generated plots
createmap_plots.control.kind = plugin
createmap_plots.control.type = createMapfile
createmap_plots.control.method = mapfile_from_folder
createmap_plots.control.mapfile_dir = input.output.mapfile_dir
createmap_plots.control.filename = diagnostic_plots.mapfile
createmap_plots.control.folder = input.output.working_directory/input.output.job_name
createmap_plots.control.pattern = *.png
# move the plots to the inpection directory, length = 2x nbands
move_plots.control.kind = recipe
move_plots.control.type = executable_args
move_plots.control.executable = /bin/mv
move_plots.control.mapfile_in = createmap_plots.output.mapfile
move_plots.control.inputkey = source
move_plots.control.arguments = [source,{{ inspection_directory }}]