Source code for pipeline.hifa.tasks.wvrgcalflag.renderer

"""
Created on 10 Sep 2014

@author: sjw
"""

import collections
import os

#import pipeline.infrastructure.displays.image as image
import pipeline.h.tasks.common.displays.image as image
import pipeline.infrastructure.filenamer as filenamer
import pipeline.infrastructure.logging as logging
import pipeline.infrastructure.renderer.basetemplates as basetemplates
import pipeline.infrastructure.utils as utils
from pipeline.hifa.tasks.wvrgcalflag import display as display


LOG = logging.get_logger(__name__)

WvrApplication = collections.namedtuple('WvrApplication', 
                                        'ms gaintable interpolated applied') 


[docs]class T2_4MDetailsWvrgcalflagRenderer(basetemplates.T2_4MDetailsDefaultRenderer): """ T2_4MDetailsWvrgcalflagRenderer generates the detailed T2_4M-level plots and output specific to the wvrgcalflag task. """ def __init__(self, uri='wvrgcalflag.mako', description='Calculate and flag WVR calibration', always_rerender=False): super(T2_4MDetailsWvrgcalflagRenderer, self).__init__( uri=uri, description=description, always_rerender=always_rerender) """ Update the Mako context appropriate to the results created by a Wvrgcalflag task. :param context: the pipeline Context :type context: :class:`~pipeline.infrastructure.launcher.Context` :param results: the bandpass results to describe :type results: :class:`~pipeline.infrastructure.tasks.wvrgcalflag.resultobjects.WvrgcalflagResults` :rtype a dictionary that can be passed to the matching Mako template """
[docs] def update_mako_context(self, ctx, context, results): plots_dir = os.path.join(context.report_dir, 'stage%d' % results.stage_number) if not os.path.exists(plots_dir): os.mkdir(plots_dir) applications = [] flag_plots = {} metric_plots = {} phase_offset_summary_plots = {} phase_offset_detail_plots = {} baseline_summary_plots = {} baseline_detail_plots = {} wvrinfos = {} ms_non12 = [] ms_toofew12 = [] for result in results: vis = os.path.basename(result.inputs['vis']) ms = context.observing_run.get_ms(vis) # If there's no WVR data (empty dataresult) if not result.flaggerresult.dataresult: # Check if this MS only contains data for non-WVR antennas. if len([a for a in ms.antennas if a.diameter != 12.0]) == len(ms.antennas): ms_non12.append(os.path.basename(vis)) elif result.too_few_wvr: ms_toofew12.append(os.path.basename(vis)) continue applications.extend(self.get_wvr_applications(result.flaggerresult.dataresult)) try: wvrinfos[vis] = result.flaggerresult.dataresult.wvr_infos except AttributeError: pass # collect flagging metric plots from result if result.flaggerresult.dataresult.qa_wvr.view: plotter = image.ImageDisplay() plots = plotter.plot(context, result.flaggerresult.dataresult.qa_wvr, reportdir=plots_dir, prefix='qa', change='WVR') # sort plots by spw metric_plots[vis] = sorted(plots, key=lambda plot: int(plot.parameters['spw'])) if result.flaggerresult.view: flag_plotter = image.ImageDisplay() plots = flag_plotter.plot(context, result.flaggerresult, reportdir=plots_dir, prefix='flag', change='Flagging') # sort plots by spw flag_plots[vis] = sorted(plots, key=lambda plot: int(plot.parameters['spw'])) if result.flaggerresult.dataresult.qa_wvr.nowvr_result: # generate the phase offset summary plots phase_offset_summary_plotter = display.WVRPhaseOffsetSummaryPlot(context, result.flaggerresult) phase_offset_summary_plots[vis] = phase_offset_summary_plotter.plot() # generate the per-antenna phase offset plots phase_offset_plotter = display.WVRPhaseOffsetPlot(context, result.flaggerresult) phase_offset_detail_plots[vis] = phase_offset_plotter.plot() baseline_plotter = display.WVRPhaseVsBaselineChart(context, result.flaggerresult) baseline_detail_plots[vis] = baseline_plotter.plot() # get the first scan for the QA intent(s) qa_intent = set(result.flaggerresult.dataresult.inputs['qa_intent'].split(',')) qa_scan = sorted([scan.id for scan in ms.scans if not qa_intent.isdisjoint(scan.intents)])[0] # scan parameter on plot is comma-separated string qa_scan = str(qa_scan) LOG.trace('Using scan %s for phase vs baseline summary plots', qa_scan) baseline_summary_plots[vis] = [p for p in baseline_detail_plots[vis] if qa_scan in set(p.parameters['scan'].split(','))] weblog_dir = os.path.join(context.report_dir, 'stage%s' % results.stage_number) # render plots for all EBs in one page phase_offset_subpages = {} baseline_summary_subpages = {} metric_subpages = {} flag_subpages = {} for d, plotter_cls, subpages in ( (phase_offset_detail_plots, WvrgcalflagPhaseOffsetPlotRenderer, phase_offset_subpages), (baseline_detail_plots, WvrgcalflagPhaseOffsetVsBaselinePlotRenderer, baseline_summary_subpages), (metric_plots, WvrcalflagMetricPlotsRenderer, metric_subpages), (flag_plots, WvrgcalflagFlagPlotRenderer, flag_subpages)): if d: all_plots = list(utils.flatten([v for v in d.values()])) renderer = plotter_cls(context, results, all_plots) with renderer.get_file() as fileobj: fileobj.write(renderer.render()) # redirect the subpages to the master page for vis in d: subpages[vis] = renderer.path # add the PlotGroups to the Mako context. The Mako template will parse # these objects in order to create links to the thumbnail pages we # just created ctx.update({ 'applications': applications, 'wvrinfos': wvrinfos, 'flag_plots': flag_plots, 'flag_subpages': flag_subpages, 'metric_plots': metric_plots, 'metric_subpages': metric_subpages, 'phase_offset_summary_plots': phase_offset_summary_plots, 'phase_offset_subpages': phase_offset_subpages, 'baseline_summary_plots': baseline_summary_plots, 'baseline_summary_subpages': baseline_summary_subpages, 'dirname': weblog_dir }) # tell the user not to panic for non-12m dishes missing WVR if ms_non12: msg = ('WVR data are not expected for %s, which %s not observed ' 'using 12m antennas.' '' % (utils.commafy(ms_non12, quotes=False, conjunction='or'), 'was' if len(ms_non12) is 1 else 'were')) ctx['alerts_info'] = [msg] elif ms_toofew12: msg = ("No WVR correction computed for {}, which {} observed" " with too few 12m antennas." "".format(utils.commafy(ms_toofew12, quotes=False, conjunction='or'), 'was' if len(ms_toofew12) is 1 else 'were')) ctx['alerts_info'] = [msg]
# Phase vs time for the overview plot should be for the widest window # at the highest frequency # spws = sorted(ms.spectral_windows, # key=operator.attrgetter('bandwidth', 'centre_frequency')) # overview_spw = spws[-1]
[docs] @staticmethod def get_wvr_applications(result): applications = [] interpolated = utils.commafy(result.wvrflag, False) # define a closure that adds a wvrapplication for each calapplication # unless the applications list already contains one for that # ms/caltable combination def collect(calapps, accept): for calapp in calapps: ms = os.path.basename(calapp.vis) gaintable = os.path.basename(calapp.gaintable) a = WvrApplication(ms, gaintable, interpolated, accept) if not any([r for r in applications if r.ms == ms and r.gaintable == gaintable]): applications.append(a) collect(result.final, True) collect(result.pool, False) return applications
[docs]class WvrcalflagMetricPlotsRenderer(basetemplates.JsonPlotRenderer): def __init__(self, context, result, plots): vis = utils.get_vis_from_plots(plots) title = 'RMS phase improvement view for %s' % vis outfile = filenamer.sanitize('rms_phase_improvement-%s.html' % vis) super(WvrcalflagMetricPlotsRenderer, self).__init__( 'generic_x_vs_y_spw_intent_plots.mako', context, result, plots, title, outfile)
[docs] def update_json_dict(self, d, plot): d['intent'] = plot.parameters['intent']
[docs]class WvrgcalflagPhaseOffsetPlotRenderer(basetemplates.JsonPlotRenderer): def __init__(self, context, result, plots): vis = utils.get_vis_from_plots(plots) title = 'WVR Phase Offset Plots for %s' % vis outfile = filenamer.sanitize('phase_offsets-%s.html' % vis) # HACK! ant parameter is a list, but for these plots we know there's # only one plot. for plot in [p for p in plots if isinstance(p.parameters['ant'], list)]: plot.parameters['ant'] = plot.parameters['ant'][0] super(WvrgcalflagPhaseOffsetPlotRenderer, self).__init__( 'wvrgcalflag_phase_offset_plots.mako', context, result, plots, title, outfile)
[docs] def update_json_dict(self, d, plot): if plot.qa_score <= 0.0: d['ratio'] = 0.0 else: d['ratio'] = 1.0 / plot.qa_score plot.score = d['ratio']
[docs]class WvrgcalflagPhaseOffsetVsBaselinePlotRenderer(basetemplates.JsonPlotRenderer): def __init__(self, context, result, plots): vis = utils.get_vis_from_plots(plots) title = 'Phase offset vs average baseline for %s' % vis outfile = filenamer.sanitize('phase_offsets_vs_baseline-%s.html' % vis) super(WvrgcalflagPhaseOffsetVsBaselinePlotRenderer, self).__init__( 'generic_x_vs_y_spw_scan_plots.mako', context, result, plots, title, outfile)
[docs]class WvrgcalflagFlagPlotRenderer(basetemplates.JsonPlotRenderer): def __init__(self, context, result, plots): vis = utils.get_vis_from_plots(plots) title = 'Flagging metric view for %s' % vis outfile = filenamer.sanitize('flagging_antenna_vs_time-%s.html' % vis) super(WvrgcalflagFlagPlotRenderer, self).__init__( 'generic_x_vs_y_spw_intent_plots.mako', context, result, plots, title, outfile)