"""
Created on 11 Sep 2014
@author: sjw
"""
import collections
import os
import pipeline.infrastructure.filenamer as filenamer
import pipeline.infrastructure.logging as logging
import pipeline.infrastructure.renderer.basetemplates as basetemplates
import pipeline.infrastructure.utils as utils
from pipeline.h.tasks.common.displays import bandpass as bandpass
LOG = logging.get_logger(__name__)
BandpassApplication = collections.namedtuple('BandpassApplication',
'ms bandtype solint intent spw gaintable')
PhaseupApplication = collections.namedtuple('PhaseupApplication',
'ms calmode solint minblperant minsnr flagged phaseupbw')
[docs]class T2_4MDetailsBandpassRenderer(basetemplates.T2_4MDetailsDefaultRenderer):
"""
T2_4MDetailsBandpassRenderer generates the detailed T2_4M-level plots and
output specific to the bandpass calibration task.
"""
def __init__(self, uri='bandpass.mako',
description='Bandpass calibration',
always_rerender=False):
super(T2_4MDetailsBandpassRenderer, self).__init__(uri=uri,
description=description,
always_rerender=always_rerender)
"""
Get the Mako context appropriate to the results created by a Bandpass
task.
This routine triggers the creation of the plots specific to the bandpass
task, creating thumbnail pages as necessary. The returned dictionary
:param context: the pipeline Context
:type context: :class:`~pipeline.infrastructure.launcher.Context`
:param results: the bandpass results to describe
:type results:
:class:`~pipeline.infrastructure.tasks.bandpass.common.BandpassResults`
:rtype a dictionary that can be passed to the matching bandpass Mako
template
"""
[docs] def update_mako_context(self, ctx, context, results):
stage_dir = os.path.join(context.report_dir, 'stage%d' % results.stage_number)
if not os.path.exists(stage_dir):
os.mkdir(stage_dir)
# generate the bandpass-specific plots, collecting the Plot objects
# returned by the plot generator
bandpass_table_rows = []
phaseup_applications = []
amp_refant = {}
amp_mode = {}
amp_details = {}
amp_vs_time_subpages = {}
phase_refant = {}
phase_mode = {}
phase_details = {}
phase_vs_time_subpages = {}
no_cal_results = [r for r in results if r.applies_adopted]
with_cal_results = [r for r in results if not r.applies_adopted and r.final]
# we want table entries for all results, but plots only for those with a caltable
for result in results:
vis = os.path.basename(result.inputs['vis'])
ms = context.observing_run.get_ms(vis)
bandpass_table_rows.extend(self.get_bandpass_table(context, result, ms))
phaseup_applications.extend(self.get_phaseup_applications(context, result, ms))
for result in with_cal_results:
vis = os.path.basename(result.inputs['vis'])
ms = context.observing_run.get_ms(vis)
ms_refant = ms.reference_antenna.split(',')[0]
# need two summary plots: one for the refant, one for the mode
plotter = bandpass.BandpassAmpVsFreqSummaryChart(context, result)
summaries = plotter.plot()
for_refant = [p for p in summaries
if p.parameters['ant'] == ms_refant]
amp_refant[vis] = for_refant[0] if for_refant else None
# replace mode with first non-refant plot until we have scores
LOG.todo('Replace bp summary plot with mode when scores are in place')
non_refants = [p for p in summaries
if p.parameters['ant'] != ms_refant]
amp_mode[vis] = non_refants[0] if non_refants else None
# need two summary plots: one for the refant, one for the mode
plotter = bandpass.BandpassPhaseVsFreqSummaryChart(context, result)
summaries = plotter.plot()
for_refant = [p for p in summaries
if p.parameters['ant'] == ms_refant]
phase_refant[vis] = for_refant[0] if for_refant else None
# use the same typical antenna as for amp vs frequency
non_refants = [p for p in summaries
if p.parameters['ant'] == amp_mode[vis].parameters['ant']]
phase_mode[vis] = non_refants[0] if non_refants else None
# make phase vs freq plots for all data
plotter = bandpass.BandpassPhaseVsFreqDetailChart(context, result)
phase_details[vis] = plotter.plot()
renderer = BandpassPhaseVsFreqPlotRenderer(context, result,
phase_details[vis])
with renderer.get_file() as fileobj:
fileobj.write(renderer.render())
outfile = os.path.basename(renderer.path)
phase_vs_time_subpages[ms.basename] = outfile
plotter = bandpass.BandpassAmpVsFreqDetailChart(context, result)
amp_details[vis] = plotter.plot()
renderer = BandpassAmpVsFreqPlotRenderer(context, result,
amp_details[vis])
with renderer.get_file() as fileobj:
fileobj.write(renderer.render())
# the filename is sanitised - the MS name is not. We need to
# map MS to sanitised filename for link construction.
outfile = os.path.basename(renderer.path)
amp_vs_time_subpages[ms.basename] = outfile
# render plots for all EBs in one page
for d, plotter_cls, subpages in (
(phase_details, BandpassPhaseVsFreqPlotRenderer, phase_vs_time_subpages),
(amp_details, BandpassAmpVsFreqPlotRenderer, amp_vs_time_subpages)):
if d:
all_plots = list(utils.flatten([v for v in d.values()]))
renderer = plotter_cls(context, with_cal_results, all_plots)
with renderer.get_file() as fileobj:
fileobj.write(renderer.render())
# redirect the subpages to the master page
for vis in subpages:
subpages[vis] = renderer.path
bandpass_table_rows = utils.merge_td_columns(bandpass_table_rows)
adopted_table_rows = make_adopted_table(context, results)
# add the PlotGroups to the Mako context. The Mako template will parse
# these objects in order to create links to the thumbnail pages we
# just created
ctx.update({'bandpass_table_rows': bandpass_table_rows,
'adopted_table': adopted_table_rows,
'phaseup_applications': phaseup_applications,
'amp_mode': amp_mode,
'amp_refant': amp_refant,
'phase_mode': phase_mode,
'phase_refant': phase_refant,
'amp_subpages': amp_vs_time_subpages,
'phase_subpages': phase_vs_time_subpages,
'dirname': stage_dir})
return ctx
[docs] def get_phaseup_applications(self, context, result, ms):
# return early if phase-up was not activated
if not result.inputs.get('phaseup', False):
return []
calmode_map = {'p': 'Phase only',
'a': 'Amplitude only',
'ap': 'Phase and amplitude'}
# identify phaseup from 'preceding' list attached to result
phaseup_calapps = []
for previous_result in result.preceding:
for calapp in previous_result:
l = [cf for cf in calapp.calfrom if cf.caltype == 'gaincal']
if l and calapp not in phaseup_calapps:
phaseup_calapps.append(calapp)
applications = []
for calapp in phaseup_calapps:
solint = utils.get_origin_input_arg(calapp, 'solint')
if solint == 'inf':
solint = 'Infinite'
# Convert solint=int to a real integration time.
# solint is spw dependent; science windows usually have the same
# integration time, though that's not guaranteed by the MS.
if solint == 'int':
in_secs = ['%0.2fs' % (dt.seconds + dt.microseconds * 1e-6)
for dt in utils.get_intervals(context, calapp)]
solint = 'Per integration (%s)' % utils.commafy(in_secs, quotes=False, conjunction='or')
assert(len(calapp.origin) is 1)
origin = calapp.origin[0]
calmode = origin.inputs.get('calmode', 'N/A')
calmode = calmode_map.get(calmode, calmode)
minblperant = origin.inputs.get('minblperant', 'N/A')
minsnr = origin.inputs.get('minsnr', 'N/A')
flagged = 'TODO'
phaseupbw = result.inputs.get('phaseupbw', 'N/A')
a = PhaseupApplication(ms.basename, calmode, solint, minblperant,
minsnr, flagged, phaseupbw)
applications.append(a)
return applications
[docs] def get_bandpass_table(self, context, result, ms):
applications = []
bandtype_map = {'B': 'Channel',
'BPOLY': 'Polynomial'}
for calapp in result.final:
gaintable = os.path.basename(calapp.gaintable)
to_intent = ', '.join(calapp.intent.split(','))
if to_intent == '':
to_intent = 'ALL'
for origin in calapp.origins:
spws = origin.inputs['spw'].split(',')
solint = origin.inputs['solint']
if solint == 'inf':
solint = 'Infinite'
# Convert solint=int to a real integration time.
# solint is spw dependent; science windows usually have the same
# integration time, though that's not guaranteed by the MS.
if solint == 'int':
in_secs = ['%0.2fs' % (dt.seconds + dt.microseconds * 1e-6)
for dt in utils.get_intervals(context, calapp, set(spws))]
solint = 'Per integration (%s)' % utils.commafy(in_secs, quotes=False, conjunction='or')
# TODO get this from the calapp rather than the top-level
# inputs?
bandtype = origin.inputs['bandtype']
bandtype = bandtype_map.get(bandtype, bandtype)
a = BandpassApplication(ms.basename, bandtype, solint,
to_intent, ', '.join(spws), gaintable)
applications.append(a)
return applications
[docs]class BaseBandpassPlotRenderer(basetemplates.JsonPlotRenderer):
def __init__(self, uri, context, results, plots, title, outfile,
score_types):
# wrap singular lists so the code works the same for scalars and vectors
if not isinstance(results, collections.Iterable):
results = [results]
self._ms = {}
self._num_pols = {}
self._qa_data = {}
for r in results:
vis = os.path.basename(r.inputs['vis'])
self._ms[vis] = context.observing_run.get_ms(vis)
caltable = r.final[0].gaintable
self._num_pols[vis] = utils.get_num_caltable_polarizations(caltable)
self._qa_data[vis] = r.qa.rawdata
self._score_types = score_types
super(BaseBandpassPlotRenderer, self).__init__(
uri, context, results, plots, title, outfile)
[docs] def update_json_dict(self, json_dict, plot):
spw = int(plot.parameters['spw'])
vis = plot.parameters['vis']
ms = self._ms[vis]
dd = ms.get_data_description(spw=spw)
if dd is None:
return
antennas = ms.get_antenna(plot.parameters['ant'])
assert len(antennas) is 1, 'plot antennas != 1'
antenna = antennas[0]
num_pols = self._num_pols[vis]
qa_data = self._qa_data[vis]
scores_dict = collections.defaultdict(dict)
for corr_axis in dd.corr_axis:
pol_id = dd.get_polarization_id(corr_axis)
# QA dictionary keys are a function of both antenna and feed.
qa_id = int(antenna.id) * num_pols + pol_id
qa_str = str(qa_id)
for score_type in self._score_types:
if 'QASCORES' in qa_data:
try:
score = qa_data['QASCORES'][score_type][str(spw)][qa_str]
except KeyError:
LOG.error('Could not get %s score for %s (%s) spw %s '
'pol %s (id=%s)', score_type, antenna.name,
antenna.id, spw, corr_axis, qa_str)
score = None
else:
score = 1.0
scores_dict[score_type][corr_axis] = score
json_dict.update(scores_dict)
plot.scores = scores_dict
# def update_mako_context(self, mako_context):
# super(BaseBandpassPlotRenderer, self).update_mako_context(mako_context)
# mako_context['vis'] = self._vis
[docs]class BandpassAmpVsFreqPlotRenderer(BaseBandpassPlotRenderer):
def __init__(self, context, results, plots):
vis = utils.get_vis_from_plots(plots)
title = 'Calibrated amplitude vs frequency for %s' % vis
outfile = filenamer.sanitize('amp_vs_freq-%s.html' % vis)
score_types = frozenset(['AMPLITUDE_SCORE_DD',
'AMPLITUDE_SCORE_FN',
'AMPLITUDE_SCORE_SNR'])
super(BandpassAmpVsFreqPlotRenderer, self).__init__(
'bandpass-amp_vs_freq_plots.mako', context,
results, plots, title, outfile, score_types)
[docs]class BandpassPhaseVsFreqPlotRenderer(BaseBandpassPlotRenderer):
def __init__(self, context, results, plots):
vis = utils.get_vis_from_plots(plots)
title = 'Calibrated phase vs frequency for %s' % vis
outfile = filenamer.sanitize('phase_vs_freq-%s.html' % vis)
score_types = frozenset(['PHASE_SCORE_DD',
'PHASE_SCORE_FN',
'PHASE_SCORE_RMS'])
super(BandpassPhaseVsFreqPlotRenderer, self).__init__(
'bandpass-phase_vs_freq_plots.mako', context,
results, plots, title, outfile, score_types)
AdoptedTR = collections.namedtuple('AdoptedTR', 'vis gaintable')
[docs]def make_adopted_table(context, results):
# will hold all the flux stat table rows for the results
rows = []
for adopted_result in [r for r in results if r.applies_adopted]:
assert (len(adopted_result.final) == 1)
calapp = adopted_result.final[0]
vis_cell = os.path.basename(calapp.vis)
gaintable_cell = os.path.basename(calapp.gaintable)
tr = AdoptedTR(vis_cell, gaintable_cell)
rows.append(tr)
return utils.merge_td_columns(rows)