Line data Source code
1 : /* -*- mode: c++ -*- */
2 : //# ParallelImagerMixin.h: Main class for parallel imaging
3 : //# Copyright (C) 2016
4 : //# Associated Universities, Inc. Washington DC, USA.
5 : //#
6 : //# This library is free software; you can redistribute it and/or modify it
7 : //# under the terms of the GNU Library General Public License as published by
8 : //# the Free Software Foundation; either version 2 of the License, or (at your
9 : //# option) any later version.
10 : //#
11 : //# This library is distributed in the hope that it will be useful, but WITHOUT
12 : //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
14 : //# License for more details.
15 : //#
16 : //# You should have received a copy of the GNU Library General Public License
17 : //# along with this library; if not, write to the Free Software Foundation,
18 : //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
19 : //#
20 : //# Correspondence concerning AIPS++ should be addressed as follows:
21 : //# Internet email: aips2-request@nrao.edu.
22 : //# Postal address: AIPS++ Project Office
23 : //# National Radio Astronomy Observatory
24 : //# 520 Edgemont Road
25 : //# Charlottesville, VA 22903-2475 USA
26 : //#
27 : #ifndef PARALLEL_IMAGER_MIXIN_H_
28 : #define PARALLEL_IMAGER_MIXIN_H_
29 :
30 : #include <string>
31 : #include <vector>
32 :
33 : #include <synthesis/ImagerObjects/MPIGlue.h>
34 : #include <casacore/casa/Containers/Record.h>
35 : #include <synthesis/ImagerObjects/ParallelImagerParams.h>
36 : #include <synthesis/ImagerObjects/ParamFieldIterator.h>
37 : #include <synthesis/ImagerObjects/SynthesisUtilMethods.h>
38 : #include <synthesis/ImagerObjects/SynthesisDeconvolverMixin.h>
39 : #include <synthesis/ImagerObjects/SynthesisNormalizerMixin.h>
40 : #include <synthesis/ImagerObjects/SynthesisImagerMixin.h>
41 : #include <synthesis/ImagerObjects/IterationControl.h>
42 : #include <synthesis/ImagerObjects/ContinuumPartitionMixin.h>
43 : #include <synthesis/ImagerObjects/CubePartitionMixin.h>
44 : #include <synthesis/ImagerObjects/SerialPartitionMixin.h>
45 :
46 : namespace casa {
47 :
48 : /**
49 : * ParallelImagerMixin
50 : *
51 : * This class ties together mixin classes for imager, normalizer, deconvolver,
52 : * iteration control and data partitioning to provide the high level
53 : * functionality of synthesis imaging. The functionality embodied by this class,
54 : * together with its parent mixin classes, replaces all of the functionality in
55 : * task_tclean.py, refimagerhelper.py, synthesisimager_cmpt.cc,
56 : * synthesisnormalizer_cmpt.cc, synthesisdeconvolver_cmpt.cc (and maybe
57 : * others). Some rearrangement of the method calls spread across task_tclean.py
58 : * and refimagerhelper.py was necessary to create a single class embodying only
59 : * the highest level algorithmic structure common to the three cases of serial,
60 : * continuum parallel and cube parallel imaging.
61 : *
62 : * Various implementation classes are defined via typedefs at the end of this
63 : * file. The ParallelImagerMixin implementation classes are all defined using
64 : * other mixin classes that simply wrap pre-existing, non-mixin imager,
65 : * normalizer, and deconvolver component instances for use in this
66 : * framework. [We refer to classes such as SynthesisNormalizer as "component"
67 : * classes, and the mixin classes that wrap such components as "component mixin"
68 : * classes]. These component mixin classes are intended to provide a sufficient
69 : * design that will support changes in the component classes transparently; if
70 : * this proves insufficient, new implementations of the component mixin classes
71 : * could be supported with minimal refactoring.
72 : */
73 : template <class T>
74 : class ParallelImagerMixin
75 : : public T {
76 :
77 : public:
78 :
79 : // ParallelImagerMixin constructor
80 : //
81 : // The various MPI communicators passed to the constructor are intended to
82 : // represent the following process groups:
83 : //
84 : // * worker_comm: all processes used for significant computation (likely
85 : // just excluding a "front-end" or user-facing process).
86 : //
87 : // * imaging/normalization/deconvolution_comm: the groups of processes for
88 : // each of these components. Note that the scope of each of these
89 : // communicators defines the set of communicating processes that comprise
90 : // the component; for example, in cube imaging each imaging_comm comprises
91 : // a single process, whereas in continuum imaging a single imaging_comm
92 : // comprises all (worker) processes. Such differences are determined by
93 : // design decisions, and do not reflect inherent features of the
94 : // framework.
95 : //
96 : // * iteration_comm: processes taking part in iteration control, should be
97 : // those processes in worker_comm with the possible addition of a process
98 : // for the tclean "front-end".
99 : //
100 : // Note that there may be overlap in the process groups for each of these
101 : // communicators. The mixin classes in this framework support such usage,
102 : // although the viability of that support depends on the usage of these
103 : // communicators by the wrapped components in addition to the mixin
104 : // classes. The current component classes make no use of these
105 : // communicators, so overlapping process groups are supported. When
106 : // component classes are using their respective communicators, it is
107 : // sufficient for safely calling MPI routines from component code to ensure
108 : // that concurrent access to a communicator by this framework and any
109 : // component threads is avoided. [One way this can be achieved is by
110 : // limiting calls to MPI routines on the provided communicator only to
111 : // methods directly called by this framework.]
112 : //
113 0 : ParallelImagerMixin(MPI_Comm worker_comm,
114 : MPI_Comm imaging_comm,
115 : MPI_Comm normalization_comm,
116 : MPI_Comm deconvolution_comm,
117 : MPI_Comm iteration_comm,
118 : int niter,
119 : bool calculate_psf,
120 : bool calculate_residual,
121 : string save_model,
122 : ParallelImagerParams ¶ms)
123 : : niter(niter)
124 : , calculate_psf(calculate_psf)
125 : , calculate_residual(calculate_residual)
126 : , save_model(save_model)
127 : , worker_comm(worker_comm)
128 : , imaging_comm(imaging_comm)
129 : , deconvolution_comm(deconvolution_comm)
130 : , normalization_comm(normalization_comm)
131 0 : , iteration_comm(iteration_comm) {
132 : // Get parameters for this process
133 0 : ParallelImagerParams my_params = T::get_params(worker_comm, params);
134 :
135 : // Convert parameters to other formats used by synthesis imaging
136 : // components, putting them into vectors by field index (not field key
137 : // as used by the Records).
138 0 : auto to_synthesis_params_select = [] (const casacore::Record &r) {
139 0 : SynthesisParamsSelect pars;
140 0 : pars.fromRecord(r);
141 0 : return pars;
142 : };
143 0 : std::vector<SynthesisParamsSelect> selection_params =
144 : transformed_by_field<SynthesisParamsSelect>(
145 : my_params.selection, to_synthesis_params_select, "ms");
146 :
147 0 : auto to_synthesis_params_image = [] (const casacore::Record &r) {
148 0 : SynthesisParamsImage pars;
149 0 : pars.fromRecord(r);
150 0 : return pars;
151 : };
152 0 : std::vector<SynthesisParamsImage> image_params =
153 : transformed_by_field<SynthesisParamsImage>(
154 : my_params.image, to_synthesis_params_image);
155 :
156 0 : auto to_synthesis_params_grid = [] (const casacore::Record &r) {
157 0 : SynthesisParamsGrid pars;
158 0 : pars.fromRecord(r);
159 0 : return pars;
160 : };
161 0 : std::vector<SynthesisParamsGrid> grid_params =
162 : transformed_by_field<SynthesisParamsGrid>(
163 : my_params.grid, to_synthesis_params_grid);
164 :
165 0 : auto to_synthesis_params_deconv = [] (const casacore::Record &r) {
166 0 : SynthesisParamsDeconv pars;
167 0 : pars.fromRecord(r);
168 0 : return pars;
169 : };
170 0 : std::vector<SynthesisParamsDeconv> deconvolution_params =
171 : transformed_by_field<SynthesisParamsDeconv>(
172 : my_params.deconvolution, to_synthesis_params_deconv);
173 :
174 0 : auto to_vector_params = [] (const casacore::Record &r) {
175 0 : casacore::Record result = r;
176 0 : return result;
177 : };
178 0 : std::vector<casacore::Record> normalization_params =
179 : transformed_by_field<casacore::Record>(my_params.normalization, to_vector_params);
180 :
181 : // Configure components
182 0 : T::setup_imager(imaging_comm, selection_params, image_params,
183 : grid_params, my_params.weight);
184 0 : T::setup_normalizer(normalization_comm, normalization_params);
185 0 : T::setup_deconvolver(deconvolution_comm, deconvolution_params);
186 : // don't initialize iteration control on any rank until all workers have
187 : // completed initialization (need second barrier for case in which there
188 : // are processes in iteration_comm that are not in worker_comm)
189 0 : MPI_Barrier(worker_comm);
190 0 : MPI_Barrier(iteration_comm);
191 0 : T::setup_iteration_controller(iteration_comm, my_params.iteration);
192 0 : }
193 :
194 : ~ParallelImagerMixin() {
195 : T::teardown_imager();
196 : T::teardown_normalizer();
197 : T::teardown_deconvolver();
198 : T::teardown_iteration_controller();
199 : auto free_comm = [](MPI_Comm *comm) {
200 : if (*comm != MPI_COMM_NULL
201 : && *comm != MPI_COMM_SELF
202 : && *comm != MPI_COMM_WORLD)
203 : MPI_Comm_free(comm);
204 : };
205 : free_comm(&worker_comm);
206 : free_comm(&imaging_comm);
207 : free_comm(&normalization_comm);
208 : free_comm(&deconvolution_comm);
209 : free_comm(&iteration_comm);
210 : }
211 :
212 : // Top level imaging method. Note that differences in parallel continuum,
213 : // parallel cube, and serial imaging are not apparent at this level.
214 0 : casacore::Record clean() {
215 0 : if (calculate_psf) {
216 0 : T::make_psf();
217 0 : T::normalize_psf();
218 : }
219 0 : if (niter >= 0) {
220 0 : if (calculate_residual) {
221 0 : run_major_cycle();
222 : }
223 0 : else if (niter == 0 && save_model != "none") {
224 0 : T::normalize_model();
225 0 : T::predict_model();
226 : }
227 0 : if (niter > 0)
228 0 : while (run_minor_cycle())
229 0 : run_major_cycle();
230 : }
231 0 : T::restore_images();
232 0 : T::concat_images("virtualnomove");
233 0 : casacore::Record result = T::get_summary(); // includes plot_report
234 0 : MPI_Barrier(worker_comm);
235 0 : return result;
236 : }
237 :
238 : protected:
239 :
240 0 : void run_major_cycle() {
241 0 : T::normalize_model();
242 0 : T::execute_major_cycle();
243 0 : T::normalize_residual();
244 0 : T::denormalize_model();
245 0 : }
246 :
247 0 : bool run_minor_cycle() {
248 0 : T::initialize_minor_cycle();
249 0 : bool result = !T::is_clean_complete();
250 0 : if (result) T::execute_minor_cycle();
251 0 : return result;
252 : }
253 :
254 : int niter;
255 :
256 : bool calculate_psf;
257 :
258 : bool calculate_residual;
259 :
260 : string save_model;
261 :
262 : MPI_Comm worker_comm;
263 :
264 : MPI_Comm imaging_comm;
265 :
266 : MPI_Comm deconvolution_comm;
267 :
268 : MPI_Comm normalization_comm;
269 :
270 : MPI_Comm iteration_comm;
271 :
272 : // Convenience function for transforming input parameter casacore::Record fields.
273 : template<class T1>
274 0 : static std::vector<T1> transformed_by_field(casacore::Record &rec,
275 : T1 (*fn)(const casacore::Record &),
276 : const string &prefix = "") {
277 0 : std::vector<T1> result;
278 0 : auto add_to_result = [&](const casacore::Record &rec) {
279 0 : result.push_back(fn(rec));
280 : };
281 0 : std::for_each(ParamFieldIterator::begin(&rec, prefix),
282 : ParamFieldIterator::end(&rec, prefix),
283 : add_to_result);
284 0 : return result;
285 : };
286 : };
287 :
288 : // Parallel continuum imager class
289 : typedef ParallelImagerMixin<
290 : ContinuumPartitionMixin<
291 : SynthesisImagerMixin<
292 : SynthesisNormalizerMixin<
293 : SynthesisDeconvolverMixin<
294 : IterationControl> > > > >
295 : ContinuumParallelImagerImpl;
296 :
297 : // Parallel cube imager class
298 : typedef ParallelImagerMixin<
299 : CubePartitionMixin<
300 : SynthesisImagerMixin<
301 : SynthesisNormalizerMixin<
302 : SynthesisDeconvolverMixin<
303 : IterationControl> > > > >
304 : CubeParallelImagerImpl;
305 :
306 : // Serial (non-MPI) imager class -- allows ParallelImagerMixin type to be used
307 : // regardless of serial vs parallel CASA. However, the naming is
308 : // unfortunate...suggestions are welcome!
309 : typedef ParallelImagerMixin<
310 : SerialPartitionMixin<
311 : SynthesisImagerMixin<
312 : SynthesisNormalizerMixin<
313 : SynthesisDeconvolverMixin<
314 : IterationControl> > > > >
315 : SerialParallelImagerImpl;
316 :
317 : } // namespace casa
318 :
319 : #endif // PARALLEL_IMAGER_H_
|
