"""
This file implements the algorithm to determine continuum channel ranges to
use from an ALMA image cube (dirty or clean) in CASA format. All dependencies
on what were originally analysisUtils functions have been pasted into this
file for convenience. This function is meant to be run inside CASA.
Simple usage:
import findContinuum as fc
fc.findContinuum('my_dirty_cube.image')
This file can be found in a typical pipeline distribution directory, e.g.:
/lustre/naasc/sciops/comm/rindebet/pipeline/branches/trunk/pipeline/extern
As of March 7, 2019 (version 3.36), it is compatible with both python 2 and 3.
Code changes for Pipeline2020: (as of August 9, 2020)
0) fix for PIPE-554 (bug found in cube imaging of calibrators by ARI-L project)
1) fix for PIPE-525 (divide by zero in a 130-target MOUS)
2) new feature PIPE-702 (expand mask if mom8fc image has emission outside it)
3) Add warning messages for too little bandwidth or too little spread.
4) if outdir is specified and does not exist, stop and print error message.
5) Modified the is_binary() function to work in both python 2 and 3.
6) Add LSRK frequency ranges in a new final line of the .dat file
7) Increase to 2 significant digits on sigmaFC in the png name
8) Allow manually specified narrow value to be used everywhere instead of 'auto'
9) Print Final selection message to casa log
10) fix bug in convertSelectionIntoChannelList (only relevant if avoidance used)
11) allow single value in convertSelection
12) fix bug in avoidance range when gaps were trimmed (not relevant to PL)
13) remove warning about no beam in cube header
14) Add totalChannels & medianWidthOfRanges parameters+logic to pickAutoTrimChannels()
15) Insure that channel ranges are in increasing order if a second range is added to a solo range
16) add imstatListit=False to all imstat calls
17) add minor ticks to lower x-axis (channel number) of the plot
18) added 21 new control parameters to findContinuum (default values listed) for a total of 98:
* returnWarnings=False (PL 2020 might set this to True if Dirk has time)
* sigmaFindContinuumMode='auto' (split from sigmaFindContinuum)
* returnSigmaFindContinuum=False
* enableRejectNarrowInnerWindows=True (to be able to disable it manually)
* avoidExtremaInNoiseCalcForJointMask=False
* buildMom8fc=True
* momentdir=''
* amendMaskIterations='auto'
* skipchan=1
* checkIfMaskWouldHaveBeenAmended=False
* fontsize=10
* thresholdForSame=0.01 (to control the new 4-letter codes)
* keepIntermediatePngs=True
* enableOnlyExtraMask=True
* useMomentDiff=True
* smallBandwidthFraction=0.05
* smallSpreadFraction=0.33
* skipAmendMask=False (for manual usage)
* useAnnulus=True
* cubeSigmaThreshold=7.5
* npixThreshold=7 (for onlyExtraMask)
19) Added 27 new functions:
* round_half_up (to support the same rounding in python 3 as 2)
* byteDecode (to convert bytes to string for python 3)
* imageSNR
* imageSNRAnnulus
* plotChannelSelections
* compute4LetterCodeAndUpdateLegend
* compute4LetterCode
* cubeNoiseLevel
* updateChannelRangesOnPlot
* computeNpixCubeMedian
* computeNpixMom8Median
* computeNpixMom8MedianBadAtm
* replaceLineFullRangesWithNoise
* gatherWarnings
* tooLittleBandwidth
* tooLittleSpread
* computeSpread
* amendMaskYesOrNo
* extraMaskYesOrNo
* onlyExtraMaskYesOrNo
* invertChannelRanges
* robustMADofContinuumRanges
* findWidestContiguousListInChannelRange
* imagePercentileNoMask (not used by PL because avoidExtremaInNoiseCalcForJointMask=False)
* combineContDat (not used by pipeline)
* getSpwFromPipelineImageName (not used by pipeline)
* getFieldnameFromPipelineImageName (not used by pipeline)
-Todd Hunter
"""
from __future__ import print_function # prevents adding old-style print statements
import os
import decimal
try:
# pyfits is only needed to support reading spectra from FITS tables, which
# is not a use case exercised by the ALMA pipeline, but rather manual users.
import pyfits
except:
#print('WARNING: pyfits not available! You will not be able to read spectra from FITS tables (an uncommon use case).')
pass
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.ticker
import time as timeUtilities
# Check if this is CASA6 CASA 6
try:
import casalith
casaVersion = casalith.version_string()
except:
import casadef
if casadef.casa_version >= '5.0.0':
import casa as mycasa
if 'cutool' in dir(mycasa):
cu = mycasa.cutool()
casaVersion = '.'.join([str(i) for i in cu.version()[:-1]]) + '-' + str(cu.version()[-1])
else:
casaVersion = mycasa.casa['build']['version'].split()[0]
else:
casaVersion = casadef.casa_version
print("casaVersion = ", casaVersion)
try:
from taskinit import *
# print("imported casatasks and tools using taskinit *")
except:
# The following makes CASA 6 look like CASA 5 to a script like this.
from casatasks import casalog
from casatasks import immath
from casatasks import imregrid
from casatasks import imsmooth
from casatasks import imhead
from casatasks import immoments
from casatasks import makemask
from casatasks import imstat
from casatasks import imsubimage
from casatasks import imcollapse
# Tools
from casatools import measures as metool
from casatools import table as tbtool
from casatools import atmosphere as attool
from casatools import msmetadata as msmdtool
from casatools import image as iatool
from casatools import ms as mstool
from casatools import quanta as qatool
# print("imported casatasks and casatools individually")
if casaVersion < '5.9.9':
synthesismaskhandler = casac.synthesismaskhandler
from immath_cli import immath_cli as immath # only used if pbcube is not passed and no emission is found
from imhead_cli import imhead_cli as imhead
from imregrid_cli import imregrid_cli as imregrid
from imsmooth_cli import imsmooth_cli as imsmooth
from immoments_cli import immoments_cli as immoments
from makemask_cli import makemask_cli as makemask
from imsubimage_cli import imsubimage_cli as imsubimage
from imcollapse_cli import imcollapse_cli as imcollapse
from imstat_cli import imstat_cli as imstat # used by computeMadSpectrum
else:
from casatools import synthesismaskhandler
import warnings
import subprocess
import scipy
import glob
import shutil
import random
from scipy.stats import scoreatpercentile, percentileofscore
from scipy.ndimage.filters import gaussian_filter
casaVersionString = casaVersion
casaMajorVersion = int(casaVersion[0])
if casaMajorVersion < 5:
from scipy.stats import nanmean as scipy_nanmean
import casadef # This still works in CASA 5.0, but might not someday.
else:
# scipy.nanmean still exists, but is deprecated in favor of numpy's version
from numpy import nanmean as scipy_nanmean
[docs]def version(showfile=True):
"""
Returns the CVS revision number.
"""
myversion = "$Id: findContinuumCycle8.py,v 4.96 2020/08/11 13:23:15 we Exp $"
if (showfile):
print("Loaded from %s" % (__file__))
return myversion
[docs]def casalogPost(mystring, debug=True):
"""
Generates an INFO message prepended with the version number of findContinuum.
"""
if (debug): print(mystring)
token = version(False).split()
origin = token[1].replace(',','_') + token[2]
casalog.post(mystring.replace('\n',''), origin=origin)
#if casaVersion >= '5.9' and plt.get_backend()=='TkAgg':
# casalogPost('Setting classic style matplotlib due to TkAgg')
# plt.style.use('classic') # it is supposed to preserve ratio of fontsize to plot size
SFC_FACTOR_WHEN_MOM8MASK_EXISTS = 0.8 # was 0.6 on Jan 26; 0.5 was too low on 2015.1.00131.S G09_0847+0045 spw 17
# 0.8 was too high on 2018.1.00828.S
ALL_CONTINUUM_CRITERION = 0.925 # was 0.94 for 2019 Jan 17 test
DYNAMIC_RANGE_LIMIT_PLOT = 800
AMENDMASK_PIXEL_RATIO_EXCEEDED = -1
AMENDMASK_PIXELS_ABOVE_THRESHOLD_EXCEEDED = -2
maxTrimDefault = 20
dpiDefault = 150
imstatListit = False
imstatVerbose = False
meanSpectrumMethods = ['mom0mom8jointMask', 'peakOverMad', 'peakOverRms',
'meanAboveThreshold', 'meanAboveThresholdOverRms',
'meanAboveThresholdOverMad', 'auto']
[docs]def help(match='', debug=False):
"""
Print an alphabetized list of all the defined functions at the top level in
this python file.
match: limit the list to those functions containing this string (case insensitive)
-- Todd Hunter
"""
myfile = __file__
if (myfile[-1] == 'c'):
if (debug): print("au loaded from .pyc file, looking at .py file instead")
myfile = myfile[:-1]
aufile = open(myfile,'r')
lines = aufile.readlines()
if (debug): print("Read %d lines from %s" % (len(lines), __file__))
aufile.close()
commands = []
for line in lines:
if (line.find('def ') == 0):
commandline = line.split('def ')[1]
tokens = commandline.split('(')
if (len(tokens) > 1):
command = tokens[0]
else:
command = commandline.strip('\n\r').strip('\r\n')
if (match == '' or command.lower().find(match.lower()) >= 0):
commands.append(command)
commands.sort()
for command in commands:
print(command)
[docs]def round_half_up(x):
return float(decimal.Decimal(float(x)).to_integral_value(rounding=decimal.ROUND_HALF_UP))
[docs]def is_binary(filename):
"""
This function is called by runFindContinuum.
Return true if the given filename appears to be binary.
Works in python 2.7 and 3
"""
f = os.popen('file %s'%filename, 'r')
result = f.read().find('text')
if result == -1:
return True
else:
return False
# Old method: fails in python 3
# File is considered to be binary if it contains a NULL byte.
# This approach returns True for .fits files, but
# incorrectly reports UTF-16 as binary.
# with open(filename, 'rb') as f:
# for block in f:
# if '\0' in block:
# return True
# return False
[docs]def getMemorySize():
"""
This function is called by findContinuum and runFindContinuum.
"""
try:
return(os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES'))
except ValueError:
# SC_PHYS_PAGES can be missing on OS X
return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']).strip())
MOM8MINSNR_DEFAULT = 4
MOM0MINSNR_DEFAULT = 5
[docs]def removeIfNecessary(imgname):
if os.path.exists(imgname):
shutil.rmtree(imgname)
if os.path.exists(imgname):
print("WARNING: shutil.rmtree failed for %s" % imgname)
[docs]def findContinuum(img='', pbcube=None, psfcube=None, minbeamfrac=0.3, spw='', transition='',
baselineModeA='min', baselineModeB='min', sigmaCube=3,
nBaselineChannels=0.19, sigmaFindContinuum='auto', sigmaFindContinuumMode='auto',
verbose=False, png='', pngBasename=False, nanBufferChannels=1,
source='', useAbsoluteValue=True, trimChannels='auto',
percentile=20, continuumThreshold=None, narrow='auto',
separator=';', overwrite=True, titleText='', # 25
maxTrim=maxTrimDefault, maxTrimFraction=1.0,
meanSpectrumFile='', centralArcsec='auto', alternateDirectory='.',
plotAtmosphere='transmission', airmass=1.5, pwv=1.0,
channelFractionForSlopeRemoval=0.75, mask='',
invert=False, meanSpectrumMethod='mom0mom8jointMask', peakFilterFWHM=10, # 38
skyTempThreshold=1.2, # was 1.5 in C4R2, reduced after slope removal added
skyTransmissionThreshold=0.08, maxGroupsForSkyThreshold=5,
minBandwidthFractionForSkyThreshold=0.2, regressionTest=False,
quadraticFit=True, triangleFraction=0.83, maxMemory=-1, # 46
tdmSkyTempThreshold=0.65, negativeThresholdFactor=1.15,
vis='', singleContinuum=False, applyMaskToMask=False,
plotBaselinePoints=True, dropBaselineChannels=2.0,
madRatioUpperLimit=1.5, madRatioLowerLimit=1.15,
projectCode='', useIAGetProfile=True, useThresholdWithMask=False,
dpi=dpiDefault, normalizeByMAD='auto', returnSnrs=False, # 61
overwriteMoments=False,minPeakOverMadForSFCAdjustment=[19,19],
minPixelsInJointMask=3, userJointMask='', signalRatioTier1=0.967,
snrThreshold=23, mom0minsnr=MOM0MINSNR_DEFAULT, mom8minsnr=MOM8MINSNR_DEFAULT, overwriteMasks=True,
rmStatContQuadratic=False, quadraticNsigma=1.8, bidirectionalMaskPhase2=False,
makeMovie=False, returnAllContinuumBoolean=True, outdir='', allowBluePruning=True,
avoidance='', returnSigmaFindContinuum=False,
enableRejectNarrowInnerWindows=True, avoidExtremaInNoiseCalcForJointMask=False, # 81
buildMom8fc=True, momentdir='',
amendMaskIterations='auto', skipchan=1, # 85
checkIfMaskWouldHaveBeenAmended=False, fontsize=10, thresholdForSame=0.01, # 88
keepIntermediatePngs=True, returnWarnings=False, enableOnlyExtraMask=True,
useMomentDiff=True, smallBandwidthFraction=0.05, smallSpreadFraction=0.33,
skipAmendMask=False, useAnnulus=True, cubeSigmaThreshold=7.5,
npixThreshold=7): # 98
"""
This function calls functions to:
1) compute a representative 'mean' spectrum of a dirty cube
2) find the continuum channels
3) plot the results and writes a text file with channel ranges and frequency ranges
It calls runFindContinuum up to 2 times.
If meanSpectrumMethod != 'mom0mom8jointMask', then it runs it a second
time with a reduced field size if it finds only one range of continuum
channels.
Please note that the channel ranges do not necessarily map back to
the visibility spectra in the measurement set(s). In order to produce
channel ranges that are guaranteed to map back correctly, then you must
supply a list of all desired measurement sets to the vis parameter. In
this case, the final line in the .dat file created will contain the
channel list in the correct frequency frame and order.
Returns:
--------
* A channel selection string suitable for the spw parameter of clean.
* The name of the final png produced.
* The aggregate bandwidth in continuum in GHz.
If returnAllContinuumBoolean = True, then it also returns a Boolean desribing
if it thinks the whole spectrum is continuum (True) or not (False)
Cycle 7 pipeline usage should set returnAllContinuumBoolean = True
If returnWarnings = True, then it also returns a list of warning strings,
which may be empty [], for tooLittleBandwidth and/or tooLittleSpread
If returnSnrs = True, then it also returns two more lists: mom0snrs and mom8snrs,
and the max baseline (floating point meters) inferred from the psf image
if returnSigmaFindContinuum = True (new in PL2020), then it also returns 7 more things:
1) the final value used for sigmaFindContinuum
2) the Boolean intersectionOfSelections
3) number of central ranges that were (or would have been) dropped, summed
over all runs of findContinuumChannels by the final run of runFindContinuum.
4) the result of amendMaskYesOrNo (if it was run; False otherwise)
5) the result of extraMaskYesOrNo (if it was run; False otherwise)
6) the result of extraMaskYesOrNo test 2 (if it was run; False otherwise)
7) the name of the finalMomDiff image (if amendMask=True) else the final mom8fc image
8) the path code (for amendMask)
9) the 4-letter code (for amendMask)
Required Inputs:
----------------
img: the image cube to operate upon (as opposed to specifying a
pre-existing meanSpectrumFile from a previous run)
--or--
meanSpectrumFile: an alternative pre-existing ASCII text file to use
for the mean spectrum, instead of operating on the img to create one.
Note: if you specify meanSpectrumFile and vis, you must also specify img, but
only its metadata will be used.
Parameters relevant only to meanSpectrumFile:
* invert: if True, then invert the sign of intensity and add the minimum
Optional inputs (see also General Parameters below):
---------------------------------------------------
spw: the spw name or number (integer or string integer) to put in the x-axis label;
also, it will be used to select the spw for which to generate topo channels for the
*.dat file if vis is also specified; if vis is specified and spw is
not, then it will search the name of the image for an spw number
and use it for generating the topo channel list.
transition: the name of the spectral transition (for the plot title)
avoidance: a CASA channel selection string to avoid selecting (for manual use)
meanSpectrumMethod: 'auto', 'mom0mom8jointMask', 'peakOverMad', 'peakOverRms',
'meanAboveThreshold', 'meanAboveThresholdOverRms', 'meanAboveThresholdOverMad',
where 'peak' refers to the peak in a spatially-smoothed version of cube
'auto' invokes 'meanAboveThreshold' unless sky temperature range
(max-min) from the atmospheric model is more than skyTempThreshold
in which case it invokes 'peakOverMad' instead
ALMA Cycle 5 used 'auto', while ALMA Cycle 6 will use 'mom0mom8jointMask'
Parameters relevant only to 'mom0mom8jointMask' (ALMA Cycle 6 onward)
---------------------------------------------------------------------
* minPeakOverMadForSFCAdjustment: a list of two values, first is for spws with
channels > 2500, second is for spws with fewer channels
* pbcube: the primary beam response for img, but only used by
meanSpectrumMethod='mom0mom8jointMask'. If not specified,
it will be searched for by changing file suffix from .residual to .pb
* psfcube: the psf response for img, but only used by
meanSpectrumMethod='mom0mom8jointMask'. If not specified,
it will be searched for by changing file suffix from .residual to .psf,
if still not found, then pruneMask will default to 6 pixels
* minbeamfrac: only used if psfcube is specified and meanSpectrumMethod='mom0mom8jointMask'
floating point value or 'auto' for 0.3 for 12m and 0.1 for 7m (maxBaseline<60m)
Set to zero to prevent pruning of the joint mask entirely; otherwise pruneMask
will use np.max([4,minbeamfrac*pixelsPerBeam])
* minPixelsInJointMask: if fewer than these pixels are found (after pruning), then
use all pixels above pb=0.3 (or equivalent higher value on mitigated images).
This option was essentially obseleted by the addition of pruneMask with the
knowledge of beamsize from psfcube. To avoid having any effect, it should be set
to < minbeamfrac*minPixelsPerBeam, which is typically 0.5*7=3.5; so use 3 or less.
* userJointMask: if specified, use this joint mask instead of computing one
(this option is meant for test purposes only, typically *.mask2_bi)
* normalizeByMAD: can be True, False or 'auto' (default);
if 'auto', then if atmospheric transmission varies enough across the spw,
set to True, otherwise set to False.
If True, then request normalization of the resulting mean spectrum by the
xmadm spectrum using an outer annulus and outside the joint mask. The
nominal annulus is the 0.2-0.3 pb response, but the range will
automatically be scaled upward with minimum value in the pb cube,
in order to preserve the fractional number of pixels contained in
a single field's 0.2-0.3 annulus.
* mom0minsnr: sets the threshold for the mom0 image (i.e. median + mom0minsnr*scaledMAD)
* mom8minsnr: sets the threshold for the mom8 image (i.e. median + mom8minsnr*scaledMAD)
* snrThreshold: if SNR is higher than this, then run a phase 2 mask calculation
* bidirectionalMaskPhase2: True=extend mask to negative values beyond threshold; False=Cycle6 behavior
(True was tried but gives worse results in some cases)
* rmStatContQuadratic: if True, then fit and remove a quadratic using the subset of channels that lack signal
* makeMovie: if True, then make a png for each quadratic fit as channels are
removed one-by-one (in the rmStatContQuadratic option)
* quadraticNsigma: the stopping threshold to use when ignoring outliers prior to
fitting the quadratic (in the rmStatContQuadratic option). Tto more quickly test
different values, be sure to set the meanSpectrumFile option and overwrite=False)
* allowBluePruning: use continuum range pruning heuristic added in Cycle 6 (default=True)
* amendMaskIterations (new in PL2020): if 'auto', then choose 0 for TDM and 3 for FDM;
if 0, then do not attempt any amendMask (i.e. mimic the Cycle 7 pipeline release)
if >=1, then when finished the initial findContinuum trial, create a mom8fc image and
mom0fc image scaled by nchan and chanwidth in kms, and subtract them to create momDiff
image. If the momDiff image has sufficient signal remaining in it, then expand the
joint mask, and repeat the findContinuum process with this expanded mask as the
userJointMask; if >=2, and there is still sufficient signal in the new momDiff image from
the second run of findContinuum, then check if further analysis of the extra portion of the
mask is necessary. If so, first check if intersecting the ranges of the first two rounds
eliminates the excess, otherwise create a new spectrum from it and pass it into a third
round of findContinuum with this as the meanSpectrum file. If >=3, and there is stil
sufficient signal in the new momDiff, image then set sigmaFindContinuumMode='autolower'
and sigmaFindContinuum=most recent value, and run a final round of findContinuum.
* skipAmendMask: if True, then skip the test for amend mask, and move on to onlyExtraMask
* useAnnulus: if amendMaskIterations>0, sets whether median/MAD in imageSNR be assessed in
annulus but outside the jointmask (True) or simply outside the joint mask (False)
* enableOnlyExtraMask: if False, then do not do the extraMask or onlyExtraMask iterations
* useMomentDiff: if False, use the mom8fc image to assess AmendMask decision instead of momDiff
* checkIfMaskWouldHaveBeenAmended (new in PL2020): if True, then if amendMaskIterations=0,
then it will still check to see if the mask would have been amended and return the
result (assuming that returnSigmaFindContinuum==True) -- for test purposes only
Parameters relevant only to 'auto' and 'mom0mom8jointMask':
-----------------------------------------------------------
* skyTempThreshold: rms value in Kelvin (for FDM spectra<1000MHz), above which
the meanSpectrumMethod is changed in 'auto' mode to peakOverMad; and above
which normalizeByMAD is set True in meanSpectrumMethod='mom0mom8jointMask'
when normalizeByMAD='auto' mode
* tdmSkyTempThreshold: rms value in Kelvin (for TDM and 1875MHz FDM spectra)
* skyTransmissionThreshold: threshold on (max-min)/mean, above which the
meanSpectrumMethod is changed in 'auto' mode to peakOverMad; and above
which normalizeByMAD is set True in meanSpectrumMethod='mom0mom8jointMask'
when normalizeByMAD='auto' mode
* overwriteMoments: if True, then overwrite any existing moment0 or moment8 image
* overwriteMasks: if True, then overwrite any existing moment0 or moment8 mask image
* rmStatContQuadratic: if True, then fit and remove quadratic after removing outliers
* outdir: directory to write the .mom0 and .mom8 images, .png, .dat and meanSpectrum
(default is same directory as the cube)
* avoidExtremaInNoiseCalcForJointMask (new in PL2020): experimental Boolean to avoid
pixels <5%ile and >95%ile in the chauvenet imstat of mom0 and mom8 images
Parameters relevant only to 'auto':
-----------------------------------
* triangleFraction: remove a triangle shape if the MAD of the dual-linfit residual
is less than this fraction*originalMAD; set this parameter to zero to turn it off.
Parameters relevant only to 'peakOverMad' or 'meanAboveThreshold*':
-------------------------------------------------------------------
* maxMemory: behave as if the machine has this many GB of memory
* nanBufferChannels: when removing or replacing NaNs, do this many extra channels
beyond their extent
* channelFractionForSlopeRemoval: if this many channels are initially selected,
then fit and remove a linear slope and re-identify continuum channels
Set to 1 to turn off.
* quadraticFit: if True, fit quadratic polynomial to the noise regions when
deemed appropriate; Otherwise, fit only a linear slope
* centralArcsec: radius of central box within which to compute the mean spectrum
default='auto' means start with whole field, then reduce to 1/10 if only
one window is found (unless mask is specified); or 'fwhm' for the central square
with the same radius as the PB FWHM; or a floating point value in arcseconds
* mask: a mask image preferably equal in shape to the parent image that is used to determine
the region to compute the noise (outside the mask, i.e. mask=0) and
(in the 'meanAboveThresholdMethod') the mean spectrum (inside the mask, i.e. mask=1).
The spatial union of all masked pixels in all channels is used as the mask for each channel.
Option 'auto' will look for the <filename>.mask that matches the <filename>.image
and if it finds it, it will use it; otherwise it will use no mask.
If the mask does not match in shape but is multi-channel, it will be regridded to match
and written out as <filename>.mask.regrid.
If it matches spatially but is single-channel, this channel will be used for all.
To convert a .crtf file to a mask, use:
makemask(mode='copy', inpimage=img, inpmask='chariklo.crtf', output=img+'.mask')
* applyMaskToMask: if True, apply the mask inside the user mask image to set its masked pixels to 0
Parameters relevant only to 'peakOverRms' and 'peakOverMad':
------------------------------------------------------------
* peakFilterFWHM: value (in pixels) to presmooth each plane with a Gaussian kernel of
this width. Set to 1 to not do any smoothing.
Default = 10 which is typically about 2 synthesized beams.
Parameters relevant only to 'meanAboveThreshold':
-------------------------------------------------
* useAbsoluteValue: passed to meanSpectrum, then avgOverCube -- take absolute value of
the cube before producing mean spectrum
* useIAGetProfile: if True, then for baselineMode='min', use ia.getprofile instead of
ia.getregion and subsequent arithmetic (faster, less memory exhaustive)
* continuumThreshold: if specified, begin by using only pixels above this intensity level
* sigmaCube: multiply this value by the MAD to get the threshold above
which a voxel is included in the mean spectrum.
* baselineModeA: 'min' or 'edge', method to define the baseline in meanSpectrum()
Parameters relevant to 'min':
-----------------------------
* percentile: control parameter for 'min'
Parameters relevant to 'edge':
* nBaselineChannels: if integer, then the number of channels to use in:
a) computing the mean spectrum with the 'meanAboveThreshold' methods.
b) computing the MAD of the lowest/highest channels in findContinuumChannels
if float, then the fraction of channels to use (i.e. the percentile)
default = 0.19, which is 24 channels (i.e. 12 on each side) of a TDM window
Parameters relevant only when amendMaskIterations > 0 (new for PL2020)
----------------------------------------------------------------------
* keepIntermediatePngs: if True, keep changing the png name to avoid overwriting them; only
the final one is returned
* thresholdForSame: fractional value used to determine if the residual in the mom8fc image
improved ('L'), worsened ('H') or stayed the same ('S')
* cubeSigmaThreshold: minimum cube SNR for onlyExtraMask to trigger
* npixThreshold: minimum momDiff pixels for onlyExtraMask to trigger
Parameters for function findContinuumChannels (which is called by all meanSpectrum heuristics):
-----------------------------------------------------------------------------------------------
baselineModeB: 'min' (default) or 'edge', method to define the baseline
Parameters only relevant if baselineModeB='min':
* dropBaselineChannels: percentage of extreme values to drop
nBaselineChannels: if integer, then the number of channels to use in computing
standard deviation/MAD of the baseline channels (i.e. the blue points in the plot)
if float, then it is the fraction of channels to use (i.e. the percentile)
sigmaFindContinuum: starting value for sigmaFindContinuum; passed to findContinuumChannels,
'auto' starts with value:
TDM: singleContinuum: 9, meanAboveThreshold: 4.5, peakOverMAD: 6.5,
mom0mom8jointMask: 7.2
FDM: meanAboveThreshold: 3.5, peakOverMAD: 6.0,
mom0mom8jointMask: nchan<750: 4.2 (strong lines) or 4.5 (weak lines)
nchan>=759: 2.6 (strong lines or 3.2 (weak lines)
and adjusts it depending on results
sigmaFindContinuumMode (new in PL2020): 'auto', 'autolower', 'fixed'
trimChannels: after doing best job of finding continuum, remove this many
channels from each edge of each block of channels found (for margin of safety)
If it is a float between 0..1, then trim this fraction of channels in each
group (rounding up). If it is 'auto' (default), use 0.1 but not more than
maxTrim channels, and not more than maxTrimFraction
narrow: the minimum number of channels that a group of channels must have to survive
if 0<narrow<1, then it is interpreted as the fraction of all
channels within identified blocks
if 'auto', then use int(ceil(log10(nchan)))
maxTrim: in trimChannels='auto', this is the max channels to trim per group for
TDM spws; it is automatically scaled upward for FDM spws.
maxTrimFraction: in trimChannels='auto', the max fraction of channels to trim per group
singleContinuum: if True, treat the cube as having come from a Single_Continuum setup;
For testing purpose. This option is overridden by the contents of vis (if specified).
negativeThresholdFactor: scale the nominal negative threshold by this factor (to
adjust the sensitivity to absorption features: smaller values=more sensitive)
signalRatioTier1: threshold for signalRatio, above which we desensitize the level to
consider line emission in order to avoid small differences in noise levels from
affecting the result (e.g. that occur between serial and parallel tclean cubes)
signalRatio=1 means: no lines seen, while closer to zero: more lines seen
enableRejectNarrowInnerWindows (new in PL2020): if True, then remove any inner groups
of channels that are narrower than both edge windows (when there are 3-15 groups)
General parameters:
-------------------
verbose: if True, then print additional information during processing
momentdir (new in PL2020): alternative directory to look for (and write) the moment 0&8 images
png: the name of the png to produce ('' yields default name)
pngBasename: if True, then remove the directory from img name before generating png name
source: the name of the source, to be shown in the title of the spectrum.
if None, then use the filename, up to the first underscore.
overwrite: if True, or ASCII file does not exist, then recalculate the mean spectrum
writing it to <img>.meanSpectrum
if False, then read the mean spectrum from the existing ASCII file
separator: the character to use to separate groups of channels in the string returned
titleText: default is img name and transition and the control parameter values
plotAtmosphere: '', 'tsky', or 'transmission'
airmass: for plot of atmospheric transmission
skipchan (newly-exposed in PL2020): used in making the spectral plot, the number of highest
and lowest intensity channels to avoid when constructing the y-axis range
pwv: in mm (for plot of atmospheric transmission)
vis: comma-delimited list or python list of measurement sets to use to convert channel
ranges to topocentric frequency ranges (for use in uvcontfit or uvcontsub)
plotBaselinePoints: if True, then plot the baseline-defining points as black dots
fontsize (newly-exposed in PL2020): size to use for channel ranges, axis labels and plot legend
dpi: dots per inch to use in writing the png (106 produces 861x649 pixels)
150 produces 1218x918
projectCode: a string to prepend to the title of the plot (useful for regression testing
where you can put the page number from the full PDF when running only a subset)
buildMom8fc (new in PL2020): if True, then when finished, generate "mom8fc" and "mom0fc"
images using selected channels
smallBandwidthFraction: threshold for returning a warning for too little bandwidth found
smallSpreadFraction: threshold for returning a warning for too little bandwidth spread found
"""
executionTimeSeconds = timeUtilities.time()
if meanSpectrumMethod not in meanSpectrumMethods:
print("Unrecognized option for meanSpectrumMethod: %s" % meanSpectrumMethod)
print("Available options: %s " % meanSpectrumMethods)
return
if img == '' and meanSpectrumFile != '' and vis != '':
print("If you specify meanSpectrumFile and vis, then you must also specify img (needed only to retrieve the osbserving date and direction).")
return
if userJointMask != '' and amendMaskIterations > 0:
print("You cannot set userJointMask at the same time as amendMaskIterations>0")
return
img = img.rstrip('/')
if type(centralArcsec) == str:
if centralArcsec.isdigit():
centralArcsecValue = centralArcsec
centralArcsec = float(centralArcsec)
else:
centralArcsecValue = "'"+centralArcsec+"'"
else:
centralArcsecValue = str(centralArcsec)
if len(outdir) > 0:
if not os.path.exists(outdir):
print("Requested outdir does not exist.")
return
if meanSpectrumMethod == 'mom0mom8jointMask':
if mask != '':
print("The mask parameter is not relevant for meanSpectrumMethod='mom0mom8jointMask'. Use the userJointMask parameter instead.")
return
casalogPost("\n BEGINNING: %s findContinuum.findContinuum('%s', overwriteMoments=%s, sigmaFindContinuum='%s', sigmaFindContinuumMode='%s', meanSpectrumMethod='%s', meanSpectrumFile='%s', singleContinuum=%s, outdir='%s', userJointMask='%s', momentdir='%s', amendMaskIterations=%s)" % (version().split()[2], img, overwriteMoments, str(sigmaFindContinuum), sigmaFindContinuumMode, meanSpectrumMethod, meanSpectrumFile, singleContinuum, outdir, userJointMask, momentdir, str(amendMaskIterations)))
else:
casalogPost("\n BEGINNING: %s findContinuum.findContinuum('%s', centralArcsec=%s, mask='%s', overwrite=%s, sigmaFindContinuum='%s', sigmaFindContinuumMode='%s', meanSpectrumMethod='%s', peakFilterFWHM=%.0f, meanSpectrumFile='%s', triangleFraction=%.2f, singleContinuum=%s, useIAGetProfile=%s, outdir='%s', mask='%s')" % (version().split()[2], img, centralArcsecValue, mask, overwrite, str(sigmaFindContinuum), sigmaFindContinuumMode, meanSpectrumMethod, peakFilterFWHM, meanSpectrumFile, triangleFraction, singleContinuum, useIAGetProfile, outdir, mask))
img = img.rstrip('/')
imageInfo = [] # information returned from getImageInfo
if (len(vis) > 0):
# vis is a non-blank list or non-blank string
if (type(vis) == str):
vis = vis.split(',')
# vis is now assured to be a non-blank list
for v in vis:
if not os.path.exists(v):
print("Could not find measurement set: ", v)
return
if (img != ''):
if meanSpectrumFile != '' and overwrite:
casalogPost("Setting overwrite to False because both img and meanSpectrumFile were specified!")
overwrite = False
if (not os.path.exists(img)):
casalogPost("Could not find image = %s" % (img))
return
imageInfo = getImageInfo(img)
if (imageInfo is None):
return
bmaj, bmin, bpa, cdelt1, cdelt2, naxis1, naxis2, freq, imgShape, crval1, crval2, maxBaselineIgnore = imageInfo
chanInfo = numberOfChannelsInCube(img, returnChannelWidth=True, returnFreqs=True)
nchan,firstFreq,lastFreq,channelWidth = chanInfo
if (nchan < 2):
casalogPost("You must have more than one channel in the image.")
return
channelWidth = abs(channelWidth)
else:
# we need to define chanInfo for the tooLittleBandwidth function to work later
avgspectrum, avgSpectrumNansReplaced, threshold, edgesUsed, nchan, nanmin, firstFreq, lastFreq, centralArcsec, mask, percentagePixelsNotMasked = readPreviousMeanSpectrum(meanSpectrumFile)
channelWidth = (lastFreq-firstFreq)/(nchan-1)
chanInfo = [nchan,firstFreq,lastFreq,channelWidth]
print("Set chanInfo to ", chanInfo)
meanFreqHz = np.mean([firstFreq,lastFreq])
# Look for the .pb image if it was not specified
if pbcube == '' or pbcube is None:
pbcube = locatePBCube(img)
if pbcube is not None:
if os.path.exists(pbcube):
casalogPost("Found PB cube: %s" % (pbcube))
elif amendMaskIterations > 0:
casalogPost("No PB cube found, cannot use amendMask option.")
return
# Look for the .psf image if it was not specified
if psfcube == '' or psfcube is None:
if img.find('.residual') >= 0:
if os.path.islink(img):
psfcube = os.readlink(img).replace('.residual','.psf')
else:
psfcube = img.replace('.residual','.psf')
if not os.path.exists(psfcube):
psfcube = None
else:
casalogPost("Found PSF cube: %s" % (psfcube))
if psfcube == '' or psfcube is None:
maxBaseline = 0 # i.e. it is not known if no image is passed
else:
maxBaseline = getImageInfo(psfcube)[11]
casalogPost('Inferred max baseline = %f m' % (maxBaseline))
# copy to the dirty cube for later use in the plot legend in runFindContinuum
imageInfo[11] = maxBaseline
# channelWidth, nchan, and maxBaseline are now defined, so we can define amendMaskIterations
if amendMaskIterations == 'auto':
if tdmSpectrum(channelWidth,nchan):
if maxBaseline > 400:
amendMaskIterations = 0
casalogPost('TDM spectrum detected with maxBaseline>400m detected: setting amendMaskIterations=%d' % (amendMaskIterations))
else:
amendMaskIterations = 3
casalogPost('TDM spectrum detected with maxBaseline<=400m: setting amendMaskIterations=%d' % (amendMaskIterations))
else:
amendMaskIterations = 3
casalogPost('FDM spectrum detected: setting amendMaskIterations=%d' % (amendMaskIterations))
if (mask == 'auto'):
mask = img.rstrip('.image') + '.mask'
if (not os.path.exists(mask)):
mask = ''
else:
maskInfo = getImageInfo(mask)
maskShape = maskInfo[8]
if (maskShape == imgShape).all():
centralArcsec = -1
else:
casalogPost("Shape of mask (%s) does not match the image (%s)." % (maskShape,imgShape))
casalogPost("If you want to automatically regrid the mask, then set its name explicitly.")
return
else:
if mask == False:
mask = ''
if (len(mask) > 0):
if (not os.path.exists(mask)):
casalogPost("Could not find image mask = %s" % (mask))
return
maskInfo = getImageInfo(mask)
maskShape = maskInfo[8]
if (maskShape == imgShape).all():
casalogPost("Shape of mask matches the image.")
else:
casalogPost("Shape of mask (%s) does not match the image (%s)." % (maskShape,imgShape))
# check if the spatial dimension matches. If so, then simply add channels
if (maskShape[0] == imgShape[0] and
maskShape[1] == imgShape[1]):
myia = iatool()
myia.open(img)
axis = findSpectralAxis(myia) # assume img & mask have same spectral axis number
myia.close()
if (imgShape[axis] != 1):
if (os.path.exists(mask+'.regrid') and not overwrite):
casalogPost("Regridded mask already exists, so I will use it.")
else:
casalogPost("Regridding the spectral axis of the mask with replicate.")
imregrid(mask, output=mask+'.regrid', template=img,
axes=[axis], replicate=True,
interpolation='nearest',
overwrite=overwrite)
else:
casalogPost("This single plane mask will be used for all channels.")
else:
casalogPost("Regridding the mask spatially and spectrally.")
imregrid(mask, output=mask+'.regrid', template=img, asvelocity=False, interpolation='nearest')
mask = mask+'.regrid'
maskInfo = getImageInfo(mask)
maskShape = maskInfo[8]
bytes = getMemorySize()
if meanSpectrumMethod == 'mom0mom8jointMask':
minGroupsForSFCAdjustment = 7 # was 8 on april 3, 2018
else:
minGroupsForSFCAdjustment = 10
try:
hostname = os.getenv('HOST')
except:
hostname = "?"
casalogPost("Total memory on %s = %.3f GB" % (hostname,bytes/(1024.**3)))
if (maxMemory > 0 and maxMemory < bytes/(1024.**3)):
bytes = maxMemory*(1024.**3)
casalogPost("but behaving as if it has only %.3f GB" % (bytes/(1024.**3)))
meanSpectrumMethodRequested = meanSpectrumMethod
meanSpectrumMethodMessage = ''
if img != '':
npixels = float(nchan)*naxis1*naxis2
else:
npixels = 1
triangularPatternSeen = False
badAtmosphere = None
if (meanSpectrumMethod.find('auto') >= 0):
# Cycle 4+5 Heuristic
meanSpectrumMethod = 'meanAboveThreshold'
if (img != ''):
a,b,c,d,e = atmosphereVariation(img, imageInfo, chanInfo, airmass, pwv)
if (b > skyTransmissionThreshold or e > skyTempThreshold):
meanSpectrumMethod = 'peakOverMad'
badAtmosphere = True
meanSpectrumMethodMessage = "Set meanSpectrumMethod='%s' since atmos. variation %.2f>%.2f or %.3f>%.1fK." % (meanSpectrumMethod,b,skyTransmissionThreshold,e,skyTempThreshold)
elif (e > tdmSkyTempThreshold and abs(channelWidth*nchan) > 1e9): # tdmSpectrum(channelWidth,nchan)):
meanSpectrumMethod = 'peakOverMad'
meanSpectrumMethodMessage = "Set meanSpectrumMethod='%s' since atmos. variation %.2f>%.2f or %.3f>%.2fK." % (meanSpectrumMethod,b,skyTransmissionThreshold,e,tdmSkyTempThreshold)
badAtmosphere = True
else:
badAtmosphere = False
# Maybe comment this out once thresholds are stable
if abs(channelWidth*nchan > 1e9): # tdmSpectrum(channelWidth,nchan):
myThreshold = tdmSkyTempThreshold
else:
myThreshold = skyTempThreshold
triangularPatternSeen, value = checkForTriangularWavePattern(img,triangleFraction)
if (triangularPatternSeen):
meanSpectrumMethod = 'peakOverMad'
meanSpectrumMethodMessage = "Set meanSpectrumMethod='%s' since triangular pattern was seen (%.2f<%.2f)." % (meanSpectrumMethod, value, triangleFraction)
else:
if value == False:
triangleMsg = 'slopeTest=F'
else:
triangleMsg = '%.2f>%.2f' % (value,triangleFraction)
meanSpectrumMethodMessage = "Did not change meanSpectrumMethod since atmos. variation %.2f<%.2f & %.3f<%.1fK (%s)." % (b,skyTransmissionThreshold,e,myThreshold,triangleMsg)
# meanSpectrumMethodMessage = "Did not change meanSpectrumMethod from %s since atmos. variation %.2f<%.2f & %.3f<%.1fK (%s)." % (meanSpectrumMethod,b,skyTransmissionThreshold,e,myThreshold,triangleMsg)
casalogPost(meanSpectrumMethodMessage)
if meanSpectrumMethod == 'mom0mom8jointMask':
# Cycle 6 Heuristic
centralArcsecField = None
centralArcsecLimitedField = None
if normalizeByMAD == 'auto' and img != '': # second phrase added on March 22, 2019
a,b,c,d,e = atmosphereVariation(img, imageInfo, chanInfo, airmass=airmass, pwv=pwv)
if (b > skyTransmissionThreshold or e > skyTempThreshold):
normalizeByMAD = True
meanSpectrumMethodMessage = "will potentially normalize by MAD since atmospheric variation %.2f>%.2f or %.3f>%.1fK." % (b,skyTransmissionThreshold,e,skyTempThreshold)
badAtmosphere = True
elif (e > tdmSkyTempThreshold and abs(channelWidth*nchan) > 1e9): # tdmSpectrum(channelWidth,nchan)):
normalizeByMAD = True
meanSpectrumMethodMessage = "will potentially normalize by MAD since atmospheric variation %.2f>%.2f or %.3f>%.2fK." % (b,skyTransmissionThreshold,e,tdmSkyTempThreshold)
badAtmosphere = True
else:
badAtmosphere = False
normalizeByMAD = False
meanSpectrumMethodMessage = "normalizeByMAD was 'auto' but atmospheric variation is considered too small to use it"
casalogPost(meanSpectrumMethodMessage)
else:
# Cycle 4+5 Heuristic
maxpixels = bytes/67 # float(1024)*1024*960
centralArcsecField = centralArcsec
centralArcsecLimitedField = -1
if (centralArcsec == 'auto' and img != ''):
pixelsNotAProblem = useIAGetProfile and meanSpectrumMethod in ['meanAboveThreshold','peakOverMad'] and mask != ''
if (npixels > maxpixels and (not pixelsNotAProblem)):
casalogPost("Excessive number of pixels (%.0f > %.0f) %dx%dx%d" % (npixels,maxpixels,naxis1,naxis2,nchan))
totalWidthArcsec = abs(cdelt2*naxis2)
if (mask == ''):
centralArcsecField = totalWidthArcsec*maxpixels/npixels
else:
casalogPost("Finding size of the central square that fully contains the mask.")
centralArcsecField = widthOfMaskArcsec(mask, maskInfo)
casalogPost("Width = %.3f arcsec" % (centralArcsecField))
newWidth = int(naxis2*centralArcsecField/totalWidthArcsec)
centralArcsecLimitedField = float(centralArcsecField) # Remember in case we need to reinvoke later
maxpixpos = getMaxpixpos(img)
if (maxpixpos[0] > naxis2/2 - newWidth/2 and
maxpixpos[0] < naxis2/2 + newWidth/2 and
maxpixpos[1] > naxis2/2 - newWidth/2 and
maxpixpos[1] < naxis2/2 + newWidth/2):
npixels = float(nchan)*newWidth*newWidth
casalogPost('Data max is within the smaller field')
casalogPost("Reducing image width examined from %.2f to %.2f arcsec to avoid memory problems." % (totalWidthArcsec,centralArcsecField))
else:
centralArcsecField = centralArcsec
if (meanSpectrumMethod == 'peakOverMad'):
casalogPost('Data max is NOT within the smaller field. Keeping meanSpectrumMethod of peakOverMad over full field.')
else:
meanSpectrumMethod = 'peakOverMad'
meanSpectrumMethodMessage = "Data max NOT within the smaller field. Switch to meanSpectrumMethod='peakOverMad'"
casalogPost(meanSpectrumMethodMessage)
else:
casalogPost("Using whole field since npixels=%d < maxpixels=%d" % (npixels, maxpixels))
centralArcsecField = -1 # use the whole field
elif (centralArcsec == 'fwhm' and img != ''):
centralArcsecField = primaryBeamArcsec(frequency=np.mean([firstFreq,lastFreq]),
showEquation=False)
npixels = float(nchan)*(centralArcsecField**2/abs(cdelt2)/abs(cdelt1))
else:
centralArcsecField = centralArcsec
if img != '':
npixels = float(nchan)*(centralArcsec**2/abs(cdelt2)/abs(cdelt1))
if amendMaskIterations > 0:
# 0=normal run, 1=after amending mask (if necessary), 2=extraMask or onlyExtraMask (if necessary)
# 3=lower sigmaFindContinuum (if necessary)
selection = ''
aggregateBandwidth = 0
amendMask = True
else:
amendMask = False
intersectionOfSelections = False
better = ''
amendMaskDecision = False # result from amendMaskYesOrNo (need to initialize it here in case it never gets run)
extraMaskDecision = False # result from extraMaskYesOrNo (need to initialize it here in case it never gets run)
extraMaskDecision2 = False # result from second extraMaskYesOrNo (need to initialize it here in case it never gets run)
autoLowerDecision = False # result from third extraMaskYesOrNo (need to initialize it here in case it never gets run)
finalMom8fc = ''
finalMom0fc = ''
if checkIfMaskWouldHaveBeenAmended:
buildMom8fc = True # this option needs to be True for this to work
##############################################################################################
# Each iteration is associated with a name, which corresponds to the heuristic that shall be
# followed. The name for the next iteration is set before the prior iteration concludes.
# This name is appended to all output files produced during this iteration to avoid confusion.
##############################################################################################
if amendMaskIterations == 0:
amendMaskIterationNames = ['']
else:
# Start with just the original loop, additional names will be added later
amendMaskIterationNames = ['.original'] + ['']*amendMaskIterations
# other possible names are: '.amendedMask', '.extraMask', '.onlyExtraMask', '.autoLower'
mom8fc = {} # dictionary of image names, keyed by amendMaskIterationName
mom0fc = {} # dictionary of image names, keyed by amendMaskIterationName
momDiff = {} # dictionary of image names, keyed by amendMaskIterationName
# Define the default name (used if amendMaskIterations==0), so that we can also use it to create
# the longer names generated when amendMaskIterations > 0
if outdir == '':
# img is guaranteed to not end in a '/' since img.rstrip('/') is called above
mom8fc[''] = img + '.mom8fc'
mom0fc[''] = img + '.mom0fc'
# Make sure that we write the pbmom to the local directory that contains mom8fc and mom0fc
pbmom = mom0fc[''].replace('.residual.mom0fc','.pbmom')
else:
mom8fc[''] = os.path.join(outdir,os.path.basename(img)) + '.mom8fc'
mom0fc[''] = os.path.join(outdir,os.path.basename(img)) + '.mom0fc'
pbmom = os.path.join(outdir,os.path.basename(pbcube)) + 'mom'
if (amendMask or checkIfMaskWouldHaveBeenAmended):
momDiffLevel = 8.0 # 8.0 is what we want in general (for 16293 maser: 8 gives 6 pix, while 7.5 gives 13 pix)
# momDiffLevel gets modified further down by -0.5 if mom8fc is more diagnostic (as it is for that maser)
momDiffLevelBadAtm = 11.5 # for amendMask and extraMask
momDiffLevelBadAtmOnlyExtraMask = 11.0
mom8level = 8.5
mom8levelBadAtm = 12
cubeLevel = 7.5
cubeLevel2 = 7.25 # needs to be somewhat smaller than cubeLevel
momDiffCode = None
mom8code = None # mom8code
momDiffPeak0 = None # necessary in case neither AmendMask or ExtraMask is invoked but LowBW+LowSpread triggers a new range
storeExtraMask = False # if True, the write the extraMask as a mask image, otherwise just write it to userJointMask
for amendMaskIteration in range(amendMaskIterations+1):
# On subsequent iterations, the amendMaskIterationName (and hence heuristics to follow)
# will change based on decisions made from each new .mom8fc image results.
amendMaskIterationName = amendMaskIterationNames[amendMaskIteration]
if amendMask:
casalogPost("====================================================================")
casalogPost("---------- amendMaskIteration %d (%s) ---------------------" % (amendMaskIteration, amendMaskIterationName))
casalogPost("====================================================================")
previousPng = png
previousSelection = selection
previousAggregateBandwidth = aggregateBandwidth
iteration = 0
fullLegend = False
if meanSpectrumMethod != 'mom0mom8jointMask':
# There can be multiple iterations, so highlight the first one in the log.
casalogPost('---------- runFindContinuum iteration 0')
if vis != '':
singleContinuum = isSingleContinuum(vis, spw)
print("amendMaskIteration%d (%s) runFindContinuum:" % (amendMaskIteration, amendMaskIterationName))
if False:
# dump all parameter values to casa log (for debugging purposes)
for v,variable in enumerate([img, pbcube, psfcube, minbeamfrac, spw, transition,
baselineModeA,baselineModeB, sigmaCube, nBaselineChannels, sigmaFindContinuum,
sigmaFindContinuumMode, verbose, png, pngBasename, nanBufferChannels,
source, useAbsoluteValue, trimChannels, percentile, continuumThreshold, narrow,
separator, overwrite, titleText, maxTrim, maxTrimFraction,
meanSpectrumFile, centralArcsecField,
channelWidth, alternateDirectory, imageInfo, chanInfo,
plotAtmosphere, airmass, pwv, channelFractionForSlopeRemoval, mask,
invert, meanSpectrumMethod, peakFilterFWHM,
fullLegend, iteration, meanSpectrumMethodMessage,
minGroupsForSFCAdjustment, regressionTest, quadraticFit,
npixels*1e-6, triangularPatternSeen, maxMemory, negativeThresholdFactor,
bytes, singleContinuum, applyMaskToMask, plotBaselinePoints,
dropBaselineChannels, madRatioUpperLimit, madRatioLowerLimit, projectCode,
useIAGetProfile,useThresholdWithMask, dpi, normalizeByMAD, overwriteMoments,
minPeakOverMadForSFCAdjustment, minPixelsInJointMask, userJointMask,
signalRatioTier1, snrThreshold, mom0minsnr, mom8minsnr, overwriteMasks,
rmStatContQuadratic, quadraticNsigma,
bidirectionalMaskPhase2, makeMovie, outdir, allowBluePruning,
avoidance, enableRejectNarrowInnerWindows,
avoidExtremaInNoiseCalcForJointMask, amendMask, momentdir]):
casalogPost('%d) %s.'%(v,str(variable)))
result = runFindContinuum(img, pbcube, psfcube, minbeamfrac, spw, transition,
baselineModeA,baselineModeB,
sigmaCube, nBaselineChannels, sigmaFindContinuum, sigmaFindContinuumMode,
verbose, png, pngBasename, nanBufferChannels,
source, useAbsoluteValue, trimChannels,
percentile, continuumThreshold, narrow,
separator, overwrite, titleText,
maxTrim, maxTrimFraction,
meanSpectrumFile, centralArcsecField,
channelWidth,
alternateDirectory, imageInfo, chanInfo,
plotAtmosphere, airmass, pwv,
channelFractionForSlopeRemoval, mask,
invert, meanSpectrumMethod, peakFilterFWHM,
fullLegend, iteration, meanSpectrumMethodMessage,
minGroupsForSFCAdjustment=minGroupsForSFCAdjustment,
regressionTest=regressionTest, quadraticFit=quadraticFit,
megapixels=npixels*1e-6, triangularPatternSeen=triangularPatternSeen,
maxMemory=maxMemory, negativeThresholdFactor=negativeThresholdFactor,
byteLimit=bytes, singleContinuum=singleContinuum,
applyMaskToMask=applyMaskToMask, plotBaselinePoints=plotBaselinePoints,
dropBaselineChannels=dropBaselineChannels,
madRatioUpperLimit=madRatioUpperLimit,
madRatioLowerLimit=madRatioLowerLimit, projectCode=projectCode,
useIAGetProfile=useIAGetProfile,useThresholdWithMask=useThresholdWithMask,
dpi=dpi, normalizeByMAD=normalizeByMAD,
overwriteMoments=overwriteMoments,
minPeakOverMadForSFCAdjustment=minPeakOverMadForSFCAdjustment,
minPixelsInJointMask=minPixelsInJointMask, userJointMask=userJointMask,
signalRatioTier1=signalRatioTier1, snrThreshold=snrThreshold,
mom0minsnr=mom0minsnr, mom8minsnr=mom8minsnr, overwriteMasks=overwriteMasks,
rmStatContQuadratic=rmStatContQuadratic, quadraticNsigma=quadraticNsigma,
bidirectionalMaskPhase2=bidirectionalMaskPhase2,
makeMovie=makeMovie,
outdir=outdir, allowBluePruning=allowBluePruning,
avoidance=avoidance,
enableRejectNarrowInnerWindows=enableRejectNarrowInnerWindows,
avoidExtremaInNoiseCalcForJointMask=avoidExtremaInNoiseCalcForJointMask,
amendMask=amendMask, momentdir=momentdir, skipchan=skipchan,
amendMaskIterationName=amendMaskIterationName, fontsize=fontsize)
if result is None:
return
if amendMaskIterationName in ['.extraMask','.onlyExtraMask','.autoLower']:
# We only want to keep the new selection, not the other info
selection = result[0]
else:
selection, mypng, slope, channelWidth, nchan, useLowBaseline, mom0snrs, mom8snrs, useMiddleChannels, selectionPreBluePruning, finalSigmaFindContinuum, jointMask, avgspectrumAboveThreshold, medianTrue, labelDescs, ax1, ax2, positiveThreshold, areaString, rangesDropped, effectiveSigma, baselineMAD, upperXlabel, allBaselineChannelsXY = result
if png == '':
png = mypng
mytest = False
if meanSpectrumMethod != 'mom0mom8jointMask':
# Cycle 4+5 Heuristic
if (centralArcsec == 'auto' and img != '' and len(selection.split(separator)) < 2):
# Only one range was found, so look closer into the center for a line
casalogPost("Only one range of channels was found")
myselection = selection.split(separator)[0]
if (myselection.find('~') > 0):
a,b = [int(i) for i in myselection.split('~')]
print("Comparing range of %d~%d to %d/2" % (a,b,nchan))
mytest = b-a+1 > nchan/2
else:
mytest = True
if (mytest):
reductionFactor = 10.0
if (naxis1 > 1*6*reductionFactor): # reduced field must be at least 1 beam across (assuming 6 pix per beam)
# reduce the field size to one tenth of the previous
bmaj, bmin, bpa, cdelt1, cdelt2, naxis1, naxis2, freq, imgShape, crval1, crval2, maxBaselineIgnore = imageInfo
imageWidthArcsec = 0.5*(np.abs(naxis2*cdelt2) + np.abs(naxis1*cdelt1))
npixels /= reductionFactor**2
centralArcsecField = imageWidthArcsec/reductionFactor
casalogPost("Reducing the field size to 1/10 of previous (%f arcsec to %f arcsec) (%d to %d pixels)" % (imageWidthArcsec,centralArcsecField,naxis1,naxis1/10))
# could change the 128 to tdmSpectrum(channelWidth,nchan), but this heuristic may also help
# for excerpts of larger cubes with narrower channel widths.
if (nBaselineChannels < 1 and nchan <= 128):
nBaselineChannels = float(np.min([0.5, nBaselineChannels*1.5]))
overwrite = True
casalogPost("Re-running findContinuum over central %.1f arcsec with nBaselineChannels=%g" % (centralArcsecField,nBaselineChannels))
iteration += 1
result = runFindContinuum(img, pbcube, psfcube, minbeamfrac, spw, transition, baselineModeA, baselineModeB,
sigmaCube, nBaselineChannels, sigmaFindContinuum, sigmaFindContinuumMode,
verbose, png, pngBasename, nanBufferChannels,
source, useAbsoluteValue, trimChannels,
percentile, continuumThreshold, narrow,
separator, overwrite, titleText,
maxTrim, maxTrimFraction,
meanSpectrumFile, centralArcsecField, channelWidth,
alternateDirectory, imageInfo, chanInfo,
plotAtmosphere, airmass, pwv,
channelFractionForSlopeRemoval, mask,
invert, meanSpectrumMethod, peakFilterFWHM,
fullLegend,iteration,meanSpectrumMethodMessage,
minGroupsForSFCAdjustment=minGroupsForSFCAdjustment,
regressionTest=regressionTest, quadraticFit=quadraticFit,
megapixels=npixels*1e-6, triangularPatternSeen=triangularPatternSeen,
maxMemory=maxMemory, negativeThresholdFactor=negativeThresholdFactor,
byteLimit=bytes, singleContinuum=singleContinuum,
applyMaskToMask=applyMaskToMask, plotBaselinePoints=plotBaselinePoints,
dropBaselineChannels=dropBaselineChannels,
madRatioUpperLimit=madRatioUpperLimit,
madRatioLowerLimit=madRatioLowerLimit, projectCode=projectCode,
useIAGetProfile=useIAGetProfile,useThresholdWithMask=useThresholdWithMask,
dpi=dpi, overwriteMoments=overwriteMoments,
minPeakOverMadForSFCAdjustment=minPeakOverMadForSFCAdjustment,
minPixelsInJointMask=minPixelsInJointMask, userJointMask=userJointMask,
signalRatioTier1=signalRatioTier1, snrThreshold=snrThreshold,
mom0minsnr=mom0minsnr, mom8minsnr=mom8minsnr,
overwriteMasks=overwriteMasks, rmStatContQuadratic=rmStatContQuadratic,
quadraticNsigma=quadraticNsigma, bidirectionalMaskPhase2=bidirectionalMaskPhase2,
makeMovie=makeMovie, outdir=outdir, allowBluePruning=allowBluePruning, avoidance=avoidance,
enableRejectNarrowInnerWindows=enableRejectNarrowInnerWindows,
avoidExtremaInNoiseCalcForJointMask=avoidExtremaInNoiseCalcForJointMask,
amendMask=amendMask, momentdir=momentdir, skipchan=skipchan,
amendMaskIterationName=amendMaskIterationName, fontsize=fontsize)
if result is None:
return
selection, mypng, slope, channelWidth, nchan, useLowBaseline, mom0snrs, mom8snrs, useMiddleChannels, selectionPreBluePruning, finalSigmaFindContinuum, jointMask, avgspectrumAboveThreshold, medianTrue, labelDescs, ax1, ax2, positiveThreshold, areaString, rangesDropped, effectiveSigma, baselineMAD, upperXlabel, allBaselineChannelsXY = result
if png == '':
png = mypng
print("************ b) png passed into initial call was blank")
else:
casalogPost("*** Not reducing field size since it would be less than 1 beam across")
else:
casalogPost("*** Not reducing field size since more than 1 range found")
aggregateBandwidth = computeBandwidth(selection, channelWidth, 0)
if (meanSpectrumMethodRequested == 'auto'):
# Cycle 4+5 Heuristic
# Here we check to see if we need to switch the method of computing the mean spectrum
# based on any undesirable characteristics of the results, and if so, re-run it.
groups = len(selection.split(separator))
if (aggregateBandwidth <= 0.00001 or
# the following line was an attempt to solve CAS-9269 case reported by Ilsang Yoon for SCOPS-2571
# (mytest and meanSpectrumMethod!='peakOverMad' and groups < 2) or
(not useLowBaseline and meanSpectrumMethod=='peakOverMad' and not tdmSpectrum(channelWidth,nchan)) or
(meanSpectrumMethod=='peakOverMad' and meanSpectrumMethodMessage != ''
and groups > maxGroupsForSkyThreshold
# the following line maintains peakOverMad for dataset in CAS-8908 (and also OMC1_NW spw39 in CAS-8720)
and aggregateBandwidth < minBandwidthFractionForSkyThreshold*nchan*channelWidth*1e-9
and not tdmSpectrum(channelWidth,nchan))):
# If less than 10 kHz is found (or two other situations), then try the other approach.
if (meanSpectrumMethod == 'peakOverMad'):
meanSpectrumMethod = 'meanAboveThreshold'
if (aggregateBandwidth <= 0.00001):
meanSpectrumMethodMessage = "Reverted to meanSpectrumMethod='%s' because no continuum found." % (meanSpectrumMethod)
casalogPost("Re-running findContinuum with the other meanSpectrumMethod: %s (because aggregateBW=%eGHz is less than 10kHz)" % (meanSpectrumMethod,aggregateBandwidth))
elif (not useLowBaseline and meanSpectrumMethod=='peakOverMad' and not tdmSpectrum(channelWidth,nchan)):
meanSpectrumMethodMessage = "Reverted to meanSpectrumMethod='%s' because useLowBaseline=F." % (meanSpectrumMethod)
casalogPost("Re-running findContinuum with the other meanSpectrumMethod: %s (because useLowBaseline=False)" % (meanSpectrumMethod))
elif (aggregateBandwidth < minBandwidthFractionForSkyThreshold*nchan*channelWidth*1e-9 and groups>maxGroupsForSkyThreshold):
meanSpectrumMethodMessage = "Reverted to meanSpectrumMethod='%s' because groups=%d>%d and not TDM." % (meanSpectrumMethod,groups,maxGroupsForSkyThreshold)
casalogPost("Re-running findContinuum with the other meanSpectrumMethod: %s because it is an FDM spectrum with many groups (%d>%d) and aggregate bandwidth (%f GHz) < %.2f of total bandwidth." % (meanSpectrumMethod,groups,maxGroupsForSkyThreshold,aggregateBandwidth,minBandwidthFractionForSkyThreshold))
else:
if groups < 2:
if sigmaFindContinuum == 'auto':
# Fix for CAS-9639: strong line near band edge and odd noise characteristic
sigmaFindContinuum = 6.5
casalogPost('Setting sigmaFindContinuum = %.1f since groups < 2' % sigmaFindContinuum)
casalogPost("Re-running findContinuum with meanSpectrumMethod: %s because groups=%d<2." % (meanSpectrumMethod,groups))
else: # a value was specified for sigmaFindContinuum
if sigmaFindContinuumMode == 'auto':
sigmaFindContinuum += 3.0
meanSpectrumMethodMessage = "Increasing sigmaFindContinuum by 3 to %.1f because groups=%d<2 and not TDM." % (sigmaFindContinuum,groups)
casalogPost("Re-running findContinuum with meanSpectrumMethod: %s because groups=%d<2." % (meanSpectrumMethod,groups))
else:
meanSpectrumMethodMessage = ''
elif groups > maxGroupsForSkyThreshold:
# hot core FDM case
meanSpectrumMethodMessage = "Reverted to meanSpectrumMethod='%s' because groups=%d>%d and not TDM." % (meanSpectrumMethod,groups,maxGroupsForSkyThreshold)
casalogPost("Re-running findContinuum with meanSpectrumMethod: %s." % (meanSpectrumMethod))
else:
meanSpectrumMethodMessage = "Reverted to meanSpectrumMethod='%s' because groups=%d and not TDM." % (meanSpectrumMethod,groups)
casalogPost("Re-running findContinuum with meanSpectrumMethod: %s." % (meanSpectrumMethod))
if (centralArcsecLimitedField > 0):
# Re-establish the previous limit
centralArcsecField = centralArcsecLimitedField
casalogPost("Re-establishing the limit on field width determined earlier: %f arcsec" %(centralArcsecField))
else:
meanSpectrumMethod = 'peakOverMad'
if (mytest and groups < 2):
centralArcsecField = -1
casalogPost("Re-running findContinuum with meanSpectrumMethod: %s (because still only 1 group found after zoom)" % (meanSpectrumMethod))
else:
casalogPost("Re-running findContinuum with the other meanSpectrumMethod: %s (because aggregateBW=%eGHz is less than 10kHz)" % (meanSpectrumMethod,aggregateBandwidth))
iteration += 1
if os.path.exists(png):
os.remove(png)
result = runFindContinuum(img, pbcube, psfcube, minbeamfrac, spw, transition, baselineModeA, baselineModeB,
sigmaCube, nBaselineChannels, sigmaFindContinuum, sigmaFindContinuumMode,
verbose, png, pngBasename, nanBufferChannels,
source, useAbsoluteValue, trimChannels,
percentile, continuumThreshold, narrow,
separator, overwrite, titleText,
maxTrim, maxTrimFraction,
meanSpectrumFile, centralArcsecField, channelWidth,
alternateDirectory, imageInfo, chanInfo,
plotAtmosphere, airmass, pwv,
channelFractionForSlopeRemoval, mask,
invert, meanSpectrumMethod, peakFilterFWHM,
fullLegend, iteration,
meanSpectrumMethodMessage,
minGroupsForSFCAdjustment=minGroupsForSFCAdjustment,
regressionTest=regressionTest, quadraticFit=quadraticFit,
megapixels=npixels*1e-6, triangularPatternSeen=triangularPatternSeen,
maxMemory=maxMemory, negativeThresholdFactor=negativeThresholdFactor,
byteLimit=bytes, singleContinuum=singleContinuum,
applyMaskToMask=applyMaskToMask, plotBaselinePoints=plotBaselinePoints,
dropBaselineChannels=dropBaselineChannels,
madRatioUpperLimit=madRatioUpperLimit,
madRatioLowerLimit=madRatioLowerLimit, projectCode=projectCode,
useIAGetProfile=useIAGetProfile,useThresholdWithMask=useThresholdWithMask,
dpi=dpi, overwriteMoments=overwriteMoments,
minPeakOverMadForSFCAdjustment=minPeakOverMadForSFCAdjustment,
minPixelsInJointMask=minPixelsInJointMask, userJointMask=userJointMask,
signalRatioTier1=signalRatioTier1, snrThreshold=snrThreshold,
mom0minsnr=mom0minsnr, mom8minsnr=mom8minsnr,
overwriteMasks=overwriteMasks, rmStatContQuadratic=rmStatContQuadratic,
quadraticNsigma=quadraticNsigma, bidirectionalMaskPhase2=bidirectionalMaskPhase2,
makeMovie=makeMovie, outdir=outdir, allowBluePruning=allowBluePruning,
avoidance=avoidance, enableRejectNarrowInnerWindows=enableRejectNarrowInnerWindows,
avoidExtremaInNoiseCalcForJointMask=avoidExtremaInNoiseCalcForJointMask,
amendMask=amendMask, momentdir=momentdir, skipchan=skipchan,
amendMaskIterationName=amendMaskIterationName, fontsize=fontsize)
selection, mypng, slope, channelWidth, nchan, useLowBaseline, mom0snrs, mom8snrs, useMiddleChannels, selectionPreBlueTruncation, finalSigmaFindContinuum, jointMask, avgspectrumAboveThreshold, medianTrue, labelDescs, ax1, ax2, positiveThreshold, areaString, rangesDropped, effectiveSigma, baselineMAD, upperXlabel, allBaselineChannelsXY = result
if png == '' or amendMask:
if png == '':
print("************ c) png passed into initial call was blank")
png = mypng # update to the latest png name
else:
casalogPost("Not re-running findContinuum with the other method because the results are deemed acceptable.")
if amendMaskIteration == 0:
originalWarnings = gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction) # only used to prevent reversion
originalSelection = selection # only used in the final reversion based on 4-letter code
originalPng = png # only used in the final reversion based on 4-letter code
# At this point, we are done the (potentially multiple) run(s) of runFindContinuum for this amendMaskIteration
if (amendMask or buildMom8fc) and img != '':
if amendMaskIteration == 0:
if pbcube is not None:
# create a mean pb image on the first iteration
removeIfNecessary(pbmom) # in case there was a prior run of findContinuum
casalogPost("Running immoments('%s', moments=[-1], outfile='%s')" % (pbcube,pbmom))
immoments(pbcube, moments=[-1], outfile=pbmom)
# This mask test should remain as the original joint mask throughout the
# process, not change to the amended mask, which is what happens to jointMask
# in the return call from runFindContinuum(userJointMask=amendedMask).
jointMaskTest = '"' + jointMask + '"==0'
if useAnnulus and pbcube is not None:
lowerAnnulusLevel, higherAnnulusLevel = findOuterAnnulusForPBCube(pbcube, False, False)
jointMaskTestAnnulus = jointMaskTest + ' && "%s">%f && "%s"<%f' % (pbmom,lowerAnnulusLevel,pbmom,higherAnnulusLevel)
else:
jointMaskTestAnnulus = ''
# save the first png again, with the tag "reverted" added, in case we need it later
labelDescs[-1].remove()
revertedAreaString = areaString + ', reverted'
casalogPost("Drawing new areaString: %s" % (revertedAreaString))
labelDesc = ax1.text(0.5,0.99-3*0.03, revertedAreaString, transform=ax1.transAxes, ha='center', size=fontsize-1)
labelDescs.append(labelDesc)
revertedPng = png.replace('.png','.reverted.png')
plt.savefig(revertedPng, dpi=dpi)
# put back the original label
labelDescs[-1].remove()
labelDesc = ax1.text(0.5,0.99-3*0.03, areaString, transform=ax1.transAxes, ha='center', size=fontsize-1)
labelDescs.append(labelDesc)
plt.draw()
# Write out the original _findContinuum.dat file, for testing future improvements of heuristics
if (meanSpectrumFile == ''):
meanSpectrumFile = buildMeanSpectrumFilename(img, meanSpectrumMethod, peakFilterFWHM, amendMaskIterationName)
if outdir != '':
meanSpectrumFile = os.path.join(outdir, os.path.basename(meanSpectrumFile))
if spw == '':
# Try to find the name of the spw from the image name
if os.path.basename(img).find('spw') > 0:
spw = os.path.basename(img).split('spw')[1].split('.')[0]
# only the base name of the meanSpectrumFile is used by writeContDat, not the contents
writeContDat(meanSpectrumFile, selection, png, aggregateBandwidth,
firstFreq, lastFreq, channelWidth, img, imageInfo, vis, spw=spw)
#######################################
# build a new mom8fc on every iteration
#######################################
mom8fc[amendMaskIterationName] = mom8fc[''] + amendMaskIterationName # here is where .original is appended on the first round
mymom8fc = mom8fc[amendMaskIterationName] # just a short-hand notation for easier use
removeIfNecessary(mymom8fc) # in case there was a prior run of findContinuum
if not os.path.exists(mymom8fc): # save time during development phase
print("Running immoments('%s', moments=[8], chans='%s', outfile='%s')" % (img,selection,mymom8fc))
immoments(img, moments=[8], chans=selection, outfile=mymom8fc)
#######################################
# build a new mom0fc on every iteration
#######################################
mom0fc[amendMaskIterationName] = mom0fc[''] + amendMaskIterationName # here is where .original is appended on the first round
mymom0fc = mom0fc[amendMaskIterationName] # just a short-hand notation for easier use
removeIfNecessary(mymom0fc) # in case there was a prior run of findContinuum
# if not os.path.exists(mymom0fc): # save time during development phase
print("Running immoments('%s', moments=[0], chans='%s', outfile='%s')" % (img,selection,mymom0fc))
immoments(img, moments=[0], chans=selection, outfile=mymom0fc)
#######################
# create scaled version
#######################
mom0fcScaled = mymom0fc+'.scaled'
removeIfNecessary(mom0fcScaled) # in case there was a prior run of findContinuum
chanWidthKms = 299792.458*channelWidth/meanFreqHz
fcChannels = len(convertSelectionIntoChannelList(selection))
factor = 1.0/chanWidthKms/fcChannels
immath([mymom0fc], mode='evalexpr', expr='IM0*%f'%(factor), outfile=mom0fcScaled)
##########################################
# Create a new momDiff on every iteration
##########################################
momDiff[amendMaskIterationName] = mymom8fc + '.minus.mom0fcScaled'
mymomDiff = momDiff[amendMaskIterationName]
removeIfNecessary(mymomDiff) # in case there was a prior run of findContinuum
if pbcube is None:
mymask = ''
else:
mymask = '"%s">0.23' % (pbmom)
print("immath(['%s','%s'], mode='evalexpr', expr='IM0-IM1', mask='%s', outfile='%s')" % (mymom8fc,mom0fcScaled,mymask,mymomDiff))
immath([mymom8fc, mom0fcScaled], mode='evalexpr', expr='IM0-IM1', mask=mymask,
outfile=mymomDiff)
finalMom8fc = mymom8fc
finalMom0fc = mymom0fc
finalMomDiff = mymomDiff
print("Wrote %s" % (mymom8fc))
useTrimmedMADIfNeeded = False # experimental method that was abandonded
##############################################
# Recalculate stats for mom8fc every iteration
##############################################
casalogPost("%d) Running imageSNR('%s',mask='%s',applyMaskToAll=False)" % (amendMaskIteration, mymom8fc, jointMaskTest))
# This is the SNR with peak measured over the whole image; but median and MAD from outside mask (and in annulus if useAnnulus=True)
mom8fcSNR, mom8fcPeak, mom8fcMedian, mom8fcMAD = imageSNR(mymom8fc, mask=jointMaskTest, maskWithAnnulus=jointMaskTestAnnulus,
useAnnulus=useAnnulus, returnAllStats=True, applyMaskToAll=False)
casalogPost("%d) Running imageSNR('%s',mask='%s',applyMaskToAll=True)" % (amendMaskIteration, mymom8fc,jointMaskTestAnnulus))
# This is the SNR outside the mask
if useAnnulus:
mom8fcSNRMom8, mom8fcPeakOutside = imageSNRAnnulus(mymom8fc, jointMaskTest, jointMaskTestAnnulus)
else:
mom8fcSNRMom8, mom8fcPeakOutside, ignore0, ignore1 = imageSNR(mymom8fc, mask=jointMaskTest, returnAllStats=True,
applyMaskToAll=True)
mom8fcSum = imstat(mymom8fc, listit=imstatListit)['sum'][0]
casalogPost("%d) mom8fcMedian = %f, mom8fcScaledMAD = %f, mom8fcSNRinside = %f, mom8fcSNROutside=%f, mom8fcSum = %f, mom8fcPeak=%f, mom8fcPeakOutside=%f" % (amendMaskIteration, mom8fcMedian, mom8fcMAD, mom8fcSNR, mom8fcSNRMom8, mom8fcSum, mom8fcPeak, mom8fcPeakOutside))
###############################################
# Recalculate stats for momDiff every iteration
###############################################
momDiffSNR, momDiffPeak, momDiffMedian, momDiffMAD = imageSNR(mymomDiff, mask=jointMaskTest, maskWithAnnulus=jointMaskTestAnnulus,
useAnnulus=useAnnulus, returnAllStats=True, applyMaskToAll=False)
if amendMaskIterationName in ['.extraMask','.onlyExtraMask','.autoLower']:
prefix = amendMaskIterationName + ': '
else:
prefix = ''
casalogPost('%smomDiff: %s' % (prefix,mymomDiff))
casalogPost('%smomDiffSNR: %f peakAnywhere: %f median: %f scaledMAD: %f' % (prefix,momDiffSNR,momDiffPeak, momDiffMedian, momDiffMAD))
# This is the SNR outside the mask
if useAnnulus:
momDiffSNROutside, momDiffPeakOutside = imageSNRAnnulus(mymomDiff, jointMaskTest, jointMaskTestAnnulus)
else:
momDiffSNROutside, momDiffPeakOutside, ignore0, ignore1 = imageSNR(mymomDiff, mask=jointMaskTest, returnAllStats=True,
applyMaskToAll=True)
momDiffSum = imstat(mymomDiff, listit=imstatListit)['sum'][0]
if meanSpectrumMethod == 'mom0mom8jointMask' and (amendMask or checkIfMaskWouldHaveBeenAmended):
if amendMaskIterationName == '.original':
if badAtmosphere is None:
# in case this was not established above (i.e. someone ran it with normalizeByMAD=False)
a,b,c,d,e = fc.atmosphereVariation(cube, imageInfo, chanInfo, airmass=airmass, pwv=pwv)
badAtmosphere = False
if (b > skyTransmissionThreshold or e > skyTempThreshold):
badAtmosphere = True
elif (e > tdmSkyTempThreshold and abs(channelWidth*nchan) > 1e9):
badAtmosphere = True
# Moment8 image: Set levels and count number of pixels above each level (and outside the joint mask)
NpixMom8Median = computeNpixMom8Median(mom8fcMedian, mom8level, mom8fcMAD, mymom8fc, jointMaskTest)
NpixMom8MedianBadAtm = computeNpixMom8MedianBadAtm(mom8fcMedian, mom8levelBadAtm, mom8fcMAD, mymom8fc, jointMaskTest)
if badAtmosphere:
casalogPost("NpixMom8MedianBadAtm = %d" % (NpixMom8MedianBadAtm))
else:
casalogPost("NpixMom8Median = %d" % (NpixMom8Median))
casalogPost("Running fc.cubeNoiseLevel('%s', pbcube='%s', mask='%s', chans='%s')" % (img, pbmom, jointMaskTest, selection))
MADCubeOutside, cubeMedian = cubeNoiseLevel(img, pbcube=pbmom, mask=jointMaskTest, chans=selection) # no need for jointMaskTestAnnulus since pbmom is passed already
mom8fcSNRCube = (mom8fcPeakOutside-mom8fcMedian)/MADCubeOutside
npix = imstat(img, listit=imstatListit)['npts']
TenEventSigma = oneEvent(npix, 10)
casalogPost("%d pixels yields 10-event sigma = %f, MAD of cube outside joint mask = %f" % (npix, TenEventSigma, MADCubeOutside))
NpixCubeMedian = computeNpixCubeMedian(mom8fcMedian, cubeLevel, MADCubeOutside, mymom8fc, jointMaskTest)
NpixCubeMedian2 = computeNpixCubeMedian(mom8fcMedian, cubeLevel2, MADCubeOutside, mymom8fc, jointMaskTest)
# MomentDiff image: Set levels and count number of pixels above each level (and outside the joint mask)
# This is the SNR with peak measured over the whole image.
momDiffSNR, momDiffPeak, momDiffMedian, momDiffMAD = imageSNR(mymomDiff, mask=jointMaskTest, maskWithAnnulus=jointMaskTestAnnulus,
useAnnulus=useAnnulus, returnAllStats=True, applyMaskToAll=False)
casalogPost('momDiff: %s' % (momDiff))
casalogPost('momDiffSNR: %f peakAnywhere: %f median: %f scaledMAD: %f' % (momDiffSNR,momDiffPeak, momDiffMedian, momDiffMAD))
# This is the SNR outside the mask
if useAnnulus:
# Need 2 calls to get peak anywhere outside the joint mask, but median and MAD from the annulus
momDiffSNROutside, momDiffPeakOutside = imageSNRAnnulus(mymomDiff, jointMaskTest, jointMaskTestAnnulus)
else:
momDiffSNROutside, momDiffPeakOutside, ignore0, ignore1 = imageSNR(mymomDiff, mask=jointMaskTest, returnAllStats=True,
applyMaskToAll=True)
momDiffSNRCube = (momDiffPeakOutside-momDiffMedian)/MADCubeOutside
casalogPost('momDiffSNRCube: %f peakOutside: %f median: %f cubeScaledMAD: %f' % (momDiffSNRCube, momDiffPeakOutside, momDiffMedian, MADCubeOutside))
momDiffSum = imstat(mymomDiff, listit=imstatListit)['sum'][0]
if mom8fcSNRMom8 > 0:
momDiffRatio = momDiffSNR/mom8fcSNRMom8
casalogPost('Test for reducing momDiffLevel: momDiffSNR/mom8fcSNRMom8 = %f < 0.3?' % (momDiffRatio))
if momDiffRatio < 0.3:
momDiffLevel -= 0.5
casalogPost('Reducing momDiffLevel by 0.5 to %f' % (momDiffLevel))
NpixMomDiff = computeNpixMom8Median(momDiffMedian, momDiffLevel, momDiffMAD, mymomDiff, jointMaskTest)
NpixMomDiff2 = computeNpixMom8Median(momDiffMedian, momDiffLevel+0.5, momDiffMAD, mymomDiff, jointMaskTest)
NpixMomDiffBadAtm = computeNpixMom8MedianBadAtm(momDiffMedian, momDiffLevelBadAtm, momDiffMAD, mymomDiff, jointMaskTest)
NpixMomDiff2BadAtm = computeNpixMom8MedianBadAtm(momDiffMedian, momDiffLevelBadAtm+0.5, momDiffMAD, mymomDiff, jointMaskTest)
if badAtmosphere:
casalogPost("NpixMomDiffBadAtm = %d" % (NpixMomDiffBadAtm))
casalogPost("NpixMomDiff2BadAtm = %d" % (NpixMomDiff2BadAtm))
else:
casalogPost("NpixMomDiff = %d" % (NpixMomDiff))
casalogPost("NpixMomDiff2 = %d" % (NpixMomDiff2))
NpixDiffCubeMedian = computeNpixCubeMedian(momDiffMedian, cubeLevel, MADCubeOutside, mymomDiff, jointMaskTest) # used by amendMaskYesOrNo
NpixDiffCubeMedian2 = computeNpixCubeMedian(momDiffMedian, cubeLevel2, MADCubeOutside, mymomDiff, jointMaskTest) # used by amendMaskYesOrNo
if skipAmendMask:
casalogPost('Skipping amendMask due to skipAmendMask request')
amendMaskDecision = 'No'
sigmaUsedToAmendMask = 0
else:
npixels = imstat(mymom8fc, listit=imstatListit)['npts'][0]
mymask = '%s && "%s"<0' % (jointMaskTest,mymom8fc)
print("Calling imstat('%s', mask='%s')" % (mymom8fc,mymask))
mystats = imstat(mymom8fc, mask=mymask, listit=imstatListit)['npts']
if len(mystats) == 0:
negativePixels = 0
else:
negativePixels = mystats[0]
fractionNegativePixels = float(negativePixels)/npixels
casalogPost('Fraction of negative pixels: %f' % (fractionNegativePixels))
if useMomentDiff:
# MomDiff Decision
casalogPost('------------Assessing moment difference image first--------------')
amendMaskDecision, AmendMaskLevel, sigmaUsedToAmendMask = amendMaskYesOrNo(badAtmosphere, momDiffMedian, momDiffSNROutside, momDiffSNRCube,
NpixMomDiff, NpixMomDiffBadAtm, NpixDiffCubeMedian,
TenEventSigma, momDiffMAD, MADCubeOutside, momDiffLevel,
momDiffLevelBadAtm, cubeLevel, NpixDiffCubeMedian2,
fractionNegativePixels, NpixMomDiff2, NpixMomDiff2BadAtm)
else:
# Moment8 decision
casalogPost('----------------Assessing moment8 image first------------------')
amendMaskDecision, AmendMaskLevel, sigmaUsedToAmendMask = amendMaskYesOrNo(badAtmosphere, mom8fcMedian, mom8fcSNRMom8, mom8fcSNRCube,
NpixMom8Median, NpixMom8MedianBadAtm, NpixCubeMedian,
TenEventSigma, mom8fcMAD, MADCubeOutside, mom8level,
mom8levelBadAtm, cubeLevel, NpixCubeMedian2,
fractionNegativePixels)
if amendMaskDecision == 'No' or not amendMask:
# Normally, the decision is all we need, but we also check the directive amendMask, because it
# could be False if the user set the parameter checkIfMaskWouldHaveBeenAmended=True.
casalogPost("Not amending mask")
if amendMask and enableOnlyExtraMask and sigmaUsedToAmendMask >= -1: # sigmaUsedToAmendMask=-1 means PixelRatio, -2 means PixelsAboveThreshold are too high, and we want to exit on that
# Assess whether onlyExtraMask should be run
result = imstat(img, chans=selection, listit=imstatListit)
cubePeak = result['max'][0] # in whole cube (not merely outside the mask), so we need a fresh call
casalogPost('channel selection: %s' % (selection))
casalogPost("cube peak anywhere = %f, medianOutside = %f, scaledMADoutside = %f, (peak-medianOutside)/scaledMADoutside=%f" % (cubePeak,cubeMedian,MADCubeOutside,(cubePeak-cubeMedian)/MADCubeOutside))
NpixMomDiff = computeNpixMom8Median(momDiffMedian, momDiffLevel, momDiffMAD, mymomDiff, '')
NpixMomDiffBadAtm = computeNpixMom8MedianBadAtm(momDiffMedian, momDiffLevelBadAtmOnlyExtraMask, momDiffMAD, mymomDiff, '')
NpixCubeDiffMedian = computeNpixCubeMedian(momDiffMedian, cubeLevel, MADCubeOutside, mymomDiff, '') # unused now
# This is the one of the remaining places where the choice of useMomentDiff is assumed to be True.
extraMaskDecision, ExtraMaskLevel, extraMaskSigmaUsed = onlyExtraMaskYesOrNo(badAtmosphere, momDiffMedian, momDiffSNR,
momDiffSNRCube, NpixMomDiff, NpixMomDiffBadAtm,
NpixCubeDiffMedian, TenEventSigma, momDiffMAD,
MADCubeOutside, cubePeak, cubeMedian,
momDiffLevel, momDiffLevelBadAtmOnlyExtraMask, cubeLevel,
cubeSigmaThreshold, npixThreshold)
warning = tooLittleBandwidth(selection, chanInfo, smallBandwidthFraction)
if warning is not None:
casalogPost('Current fractional bandwidth is too small to allow Extra Mask.')
extraMaskDecision = False
else:
casalogPost('Current fractional bandwidth is large enough to allow Extra Mask.')
if extraMaskDecision in [False,'No']:
casalogPost("Only Extra Mask will not be tried")
if amendMask:
#########################################################
# Result0: No Amend Mask, no Extra Mask ---------------------
#########################################################
# write an 'S' after the number of pixels in the legend (remove the zeros since they are not
# yet calculated and will not be used anyway)
warnings = gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction)
channelRanges = len(selection.split(separator))
if not useMomentDiff:
labelDescs, areaString, mom8code = compute4LetterCodeAndUpdateLegend(mom8fcPeak,
mom8fcPeak, mom8fcPeakOutside,
mom8fcPeakOutside, mom8fcSum, mom8fcSum, # there is no difference between mom8fcSum,mom8fcSum0
mom8fcMAD, mom8fcMAD, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
else:
labelDescs, areaString, momDiffCode = compute4LetterCodeAndUpdateLegend(momDiffPeak,
momDiffPeak, momDiffPeakOutside,
momDiffPeakOutside, momDiffSum, momDiffSum,
momDiffMAD, momDiffMAD, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision,
extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
plt.savefig(png, dpi=dpi)
break # do not run a second round
else:
# The following step is needed, e.g. for Nessie_F1 spw 18 and 24
casalogPost("Extra Mask will be tried")
# remember the previous values in case the extra mask is indeed used
mom8fcPeak0 = mom8fcPeak
mom8fcPeakOutside0 = mom8fcPeakOutside
mom8fcMAD0 = mom8fcMAD
mom8fcSum0 = mom8fcSum
momDiffPeak0 = momDiffPeak
momDiffPeakOutside0 = momDiffPeakOutside
momDiffMAD0 = momDiffMAD
momDiffSum0 = momDiffSum
print("extraMaskSigmaUsed = %f, ExtraMaskLevel = %f" % (extraMaskSigmaUsed,ExtraMaskLevel))
print("--------------------------------------------------------------------------")
print("momDiff = ", mymomDiff)
print("pbmom = ", pbmom)
print("--------------------------------------------------------------------------")
channels, frequency, intensity, normalized = computeStatisticalSpectrumFromMask(img, '"%s">%f' % (mymomDiff,ExtraMaskLevel), '"%s">0.23'%(pbmom), imageInfo, 'mean', normalizeByMAD, outdir=outdir, jointMaskForNormalize=userJointMask)
lineFullSelection = invertChannelRanges(selection, nchan) # selection = line-free ranges
myChannelList = convertSelectionIntoChannelList(selection) # myChannelList = list of line-free channels
casalogPost('Continuum channels so far: %s' % (selection))
casalogPost('Line-full channels so far: %s' % (lineFullSelection))
scaledMADFCSubset, medianFCSubset = robustMADofContinuumRanges(myChannelList, intensity)
edgesUsed = 0
meanSpectrumFile = mymom8fc + '.meanSpectrumFile_fromOnlyExtraMask_beforeNoiseReplacement'
writeMeanSpectrum(meanSpectrumFile, frequency, intensity, intensity, ExtraMaskLevel,
nchan, edgesUsed, centralArcsec='mom0mom8jointMask')
intensity = replaceLineFullRangesWithNoise(intensity, lineFullSelection, medianFCSubset, scaledMADFCSubset)
meanSpectrumFile = mymom8fc + '.meanSpectrumFile_fromOnlyExtraMask'
writeMeanSpectrum(meanSpectrumFile, frequency, intensity,
intensity, ExtraMaskLevel, nchan,
edgesUsed, centralArcsec='mom0mom8jointMask')
# update selection0 to hold the amendMask selection
selection0 = selection
if keepIntermediatePngs:
png = '' # reset name so that new name will be chosen and prior plot not overwritten
amendMaskIterationNames[amendMaskIteration+1] = '.onlyExtraMask'
signalRatioTier1 = 1.0
if False:
sigmaFindContinuumMode = 'autolower'
sigmaFindContinuum = finalSigmaFindContinuum
casalogPost("Using sigmaFindContinuumMode='autolower' with sigmaFindContinuum=%f" % (sigmaFindContinuum))
overwrite = False # do not overwrite the spectrum we just wrote when runFindContinuum runs again!
continue # from .original iteration to .onlyExtraMask iteration
# endif amendMaskDecision=='No'
# Remember the previous values if we are proceeding with amending the mask.
# These *0's will be used to compute the mom8fc 4-letter code generation
selection0 = selection
mom8fcPeak0 = mom8fcPeak
mom8fcPeakOutside0 = mom8fcPeakOutside
mom8fcMAD0 = mom8fcMAD
mom8fcSum0 = mom8fcSum
# These *0's will be used to compute the MomDiff 4-letter code generation
momDiffPeak0 = momDiffPeak
momDiffPeakOutside0 = momDiffPeakOutside
momDiffMAD0 = momDiffMAD
momDiffSum0 = momDiffSum
if outdir == '':
userJointMask = img + '.amendedJointMask' + amendMaskIterationName
if storeExtraMask:
addedMask = img + '.addedMask' + amendMaskIterationName
else:
userJointMask = os.path.join(outdir, os.path.basename(img) + '.amendedJointMask' + amendMaskIterationName)
if storeExtraMask:
addedMask = os.path.join(outdir, os.path.basename(img) + '.addedMask' + amendMaskIterationName)
removeIfNecessary(userJointMask)
casalogPost("Amending the mask with new fluxThreshold = %f" % (AmendMaskLevel))
expression = 'iif((IM0==1)||(IM1>%f), 1, 0)' % (AmendMaskLevel)
print("Running immath(imagename=['%s','%s'], outfile='%s', mode='evalexpr', expr='%s'" % (jointMask,mymom8fc,userJointMask,expression))
immath(imagename=[jointMask,mymom8fc], outfile=userJointMask, mode='evalexpr',
expr=expression)
if storeExtraMask:
# Make spectrum of the newly-added portion of the amendedJointMask
expression = 'iif((IM0==0)&&(IM1==1), 1, 0)'
removeIfNecessary(addedMask)
print("Running immath(imagename=['%s','%s'], mode='evalexpr', outfile='%s', expr='%s')" % (jointMask,userJointMask,addedMask,expression))
immath(imagename=[jointMask,userJointMask], mode='evalexpr', outfile=addedMask, expr=expression)
plotStatisticalSpectrumFromMask(img, addedMask, pbcube, statistic='mean', normalizeByMAD=normalizeByMAD)
maskedPixelsBefore = countPixelsAboveZero(jointMask)
maskedPixelsAfter = countPixelsAboveZero(userJointMask)
newPixels = maskedPixelsAfter - maskedPixelsBefore
print("---------------------------------------------------------------------")
casalogPost("amending mask %s into %s, adding %d-%d=%d pixels" % (jointMask,userJointMask,maskedPixelsAfter,maskedPixelsBefore,newPixels))
if maskedPixelsBefore > 0:
# protect against divide by zero
if (float(maskedPixelsAfter)/maskedPixelsBefore) < 2:
sigmaFindContinuumMode = 'autolower'
sigmaFindContinuum = finalSigmaFindContinuum
casalogPost("Setting sigmaFindContinuumMode = %s" % (sigmaFindContinuumMode))
casalogPost("Setting sigmaFindContinuum = finalSigmaFindContinuum = %f" % (finalSigmaFindContinuum))
else:
casalogPost("Not changing sigmaFindContinuumMode for the next run.")
if keepIntermediatePngs:
png = '' # reset name so that new name will be chosen and prior plot not overwritten
# Set the name/heuristic of the next iteration
amendMaskIterationNames[amendMaskIteration+1] = '.amendedMask'
continue # from .original iteration to .amendedMask iteration
# endif amendMaskIterationName == '.original'
if amendMaskIterationName == '.amendedMask':
print("---------------------------------------------------------------------")
# mom8fcSNRMom8 (outside) is already calculated fresh (above) on the latest .mom8fc on every iteration
mom8fcSNRCube = (mom8fcPeakOutside-mom8fcMedian)/MADCubeOutside
NpixMom8Median = computeNpixMom8Median(mom8fcMedian, np.max([5.5,TenEventSigma]), mom8fcMAD, mymom8fc, jointMaskTest)
NpixMom8MedianBadAtm = computeNpixMom8MedianBadAtm(mom8fcMedian, 8, mom8fcMAD, mymom8fc, jointMaskTest)
# use mom8fc image
NpixCubeMedian = computeNpixMom8Median(mom8fcMedian, np.max([5.0,TenEventSigma]), mom8fcMAD, mymom8fc, jointMaskTest) # unused now
if useMomentDiff:
# These values are no longer used in extraMaskYesOrNo, so could probably not bother calculating them now.
NpixMomDiff = computeNpixMom8Median(momDiffMedian, momDiffLevel, momDiffMAD, mymomDiff, '')
NpixMomDiffBadAtm = computeNpixMom8MedianBadAtm(momDiffMedian, momDiffLevelBadAtm, momDiffMAD, mymomDiff, '')
NpixCubeDiffMedian = computeNpixCubeMedian(momDiffMedian, cubeLevel, MADCubeOutside, mymomDiff, '') # unused now
extraMaskDecision, ExtraMaskLevel, extraMaskSigmaUsed = extraMaskYesOrNo(badAtmosphere, momDiffMedian, momDiffSNR,
momDiffSNRCube, NpixMomDiff, NpixMomDiffBadAtm,
NpixCubeDiffMedian, TenEventSigma, momDiffMAD,
MADCubeOutside, momDiffLevel, momDiffLevelBadAtm,
cubeLevel)
else:
extraMaskDecision, ExtraMaskLevel, extraMaskSigmaUsed = extraMaskYesOrNo(badAtmosphere, mom8fcMedian, mom8fcSNRMom8,
mom8fcSNRCube, NpixMom8Median, NpixMom8MedianBadAtm,
NpixCubeMedian, TenEventSigma, mom8fcMAD,
MADCubeOutside, mom8level, mom8levelBadAtm,
cubeLevel)
casalogPost('momDiff: %s' % (mymomDiff))
casalogPost('momDiffSNR: %f peakAnywhere: %f median: %f scaledMAD: %f' % (momDiffSNR,momDiffPeak, momDiffMedian, momDiffMAD))
casalogPost('momDiffSNRCube: %f scaledMAD: %f' % (momDiffSNRCube, MADCubeOutside))
warning = tooLittleBandwidth(selection, chanInfo, smallBandwidthFraction)
if warning is not None:
casalogPost('Current fractional bandwidth is too small to allow Extra Mask.')
extraMaskDecision = False
else:
casalogPost('Current fractional bandwidth is enough to allow Extra Mask.')
if extraMaskDecision == 'No':
# If No, we are done
########################################
# Result1: Amended Mask but no Extra Mask
########################################
if warning is None:
casalogPost("Extra Mask not deemed necessary, so we are done.")
# derive 4-letter codes and update the final line of plot legend
warnings = gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction)
channelRanges = len(selection.split(separator))
if not useMomentDiff:
labelDescs, areaString, mom8code = compute4LetterCodeAndUpdateLegend(mom8fcPeak, mom8fcPeak0, mom8fcPeakOutside,
mom8fcPeakOutside0, mom8fcSum, mom8fcSum0,
mom8fcMAD, mom8fcMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
else:
# Needs momDiff 4-letter code
labelDescs, areaString, momDiffCode = compute4LetterCodeAndUpdateLegend(momDiffPeak,
momDiffPeak0, momDiffPeakOutside,
momDiffPeakOutside0, momDiffSum, momDiffSum0,
momDiffMAD, momDiffMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision,
extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
plt.savefig(png, dpi=dpi)
break
else:
casalogPost("Extra Mask is needed")
#################################################################
# There is still excess emission.
# Can intersect help? build mom8fc.amendedMask - mom8fc.original
#################################################################
if useMomentDiff:
momDiff['.diff'] = momDiff['.amendedMask'] + '.diff'
removeIfNecessary(momDiff['.diff'])
immath([momDiff['.original'], momDiff['.amendedMask']], expr='IM1-IM0', outfile=momDiff['.diff'])
if useAnnulus:
casalogPost("Running fc.imageSNRAnnulus('%s', '%s', '%s', applyMaskToAll=False)" % (momDiff['.diff'], jointMaskTest, jointMaskTestAnnulus))
# diffStats = imageSNRAnnulus(momDiff['.diff'], jointMaskTest, jointMaskTestAnnulus) # this is what we used on July 30
diffStats = imageSNRAnnulus(momDiff['.diff'], jointMaskTest, jointMaskTestAnnulus, applyMaskToAll=False)
else:
diffStats = imageSNR(momDiff['.diff'], returnAllStats=True, mask=jointMaskTest)
else:
mom8fc['.diff'] = mom8fc['.amendedMask'] + '.diff'
removeIfNecessary(mom8fc['.diff'])
immath([mom8fc['.original'], mom8fc['.amendedMask']], expr='IM1-IM0', outfile=mom8fc['.diff'])
if useAnnulus:
casalogPost("Running fc.imageSNRAnnulus('%s', '%s', '%s', applyMaskToAll=False)" % (mom8fc['.diff'], jointMaskTest, jointMaskTestAnnulus))
diffStats = imageSNRAnnulus(mom8fc['.diff'], jointMaskTest, jointMaskTestAnnulus, applyMaskToAll=False)
else:
diffStats = imageSNR(mom8fc['.diff'], returnAllStats=True, mask=jointMaskTest)
if badAtmosphere:
testDiffLev = mom8levelBadAtm * np.sqrt(2)
else:
testDiffLev = mom8level * np.sqrt(2)
if diffStats[0] > testDiffLev:
casalogPost('**************************************************************')
casalogPost('There is significant emission in difference image (%f > %f): Needs intersection of ranges.' % (diffStats[0], testDiffLev))
casalogPost('**************************************************************')
channelList0 = convertSelectionIntoChannelList(selection0)
channelList1 = convertSelectionIntoChannelList(selection)
channelList = np.intersect1d(channelList0,channelList1)
casalogPost("Taking intersection of %d channels with %d channels, to get %d channels" % (len(channelList0), len(channelList1), len(channelList)))
casalogPost("first channel list = %s" % (selection0))
casalogPost("second channel list = %s" % (selection))
if len(channelList) == 0:
casalogPost("No intersecting ranges found: Reverting to first channel selection and stopping.")
channelList = channelList0 # reverting
selection = convertChannelListIntoSelection(channelList)
##########################################################################################
# Result2: Amended Mask and Extra Mask was tried but reverted due to no remaining channels
##########################################################################################
# derive 4-letter codes and update the final line of the plot legend
warnings = gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction)
channelRanges = len(selection.split(separator))
if not useMomentDiff:
labelDescs, areaString, mom8code = compute4LetterCodeAndUpdateLegend(mom8fcPeak, mom8fcPeak0, mom8fcPeakOutside,
mom8fcPeakOutside0, mom8fcSum, mom8fcSum0,
mom8fcMAD, mom8fcMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
else:
# Needs momDiff 4-letter code
labelDescs, areaString, momDiffCode = compute4LetterCodeAndUpdateLegend(momDiffPeak,
momDiffPeak0, momDiffPeakOutside,
momDiffPeakOutside0, momDiffSum, momDiffSum0,
momDiffMAD, momDiffMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision,
extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
plt.savefig(png, dpi=dpi)
break
intersectionOfSelections = True
###############################################################
# create mom8fcIntersect from original and amendedMask regions
###############################################################
selection = convertChannelListIntoSelection(channelList)
casalogPost("Building new mom8fc with chans='%s'" % (selection))
mom8fc['.intersect'] = mom8fc[''] + '.intersectChannelRanges'
mom8fcIntersect = mom8fc['.intersect']
removeIfNecessary(mom8fcIntersect)
immoments(img, moments=[8], chans=selection, outfile=mom8fcIntersect)
finalMom8fc = mom8fcIntersect
# Compute statistics of mom8fcIntersect image
# Assess mom8fc.intersect using original mask
mom8fcSNR, mom8fcPeak, mom8fcMedian, mom8fcMAD = imageSNR(mom8fcIntersect, mask=jointMaskTest, maskWithAnnulus=jointMaskTestAnnulus,
useAnnulus=useAnnulus, returnAllStats=True, applyMaskToAll=False)
if useAnnulus:
# Need 2 calls to get peak anywhere outside the joint mask, but median and MAD from the annulus
mom8fcSNROutside, mom8fcPeakOutside = imageSNRAnnulus(mom8fcIntersect, jointMaskTest, jointMaskTestAnnulus)
else:
mom8fcSNROutside, mom8fcPeakOutside, ignore0, ignore1 = imageSNR(mom8fcIntersect, mask=jointMaskTest, returnAllStats=True,
applyMaskToAll=True)
mom8fcSNRCube = (mom8fcPeakOutside-mom8fcMedian)/MADCubeOutside
mom8fcSum = imstat(mom8fcIntersect, listit=imstatListit)['sum'][0]
casalogPost("new mom8fcMedian = %f, mom8fcScaledMAD = %f, mom8fcSNR = %f, mom8fcSNROutside = %f, mom8fcSum = %f" % (mom8fcMedian, mom8fcMAD, mom8fcSNR, mom8fcSNROutside, mom8fcSum))
NpixMom8Median = computeNpixMom8Median(mom8fcMedian, np.max([5.5,TenEventSigma]), mom8fcMAD, mom8fcIntersect, jointMaskTest)
NpixMom8MedianBadAtm = computeNpixMom8MedianBadAtm(mom8fcMedian, 8, mom8fcMAD, mom8fcIntersect, jointMaskTest)
NpixCubeMedian = computeNpixMom8Median(mom8fcMedian, np.max([5.0,TenEventSigma]), mom8fcMAD, mom8fcIntersect, jointMaskTest)
########################################################################
# Need to build a new mom0fc image to match the new mom8fcIntersect
########################################################################
mom0fc['.intersect'] = mom0fc[''] + '.intersectChannelRanges'
mom0fcIntersect = mom0fc['.intersect']
removeIfNecessary(mom0fcIntersect) # in case there was a prior run of findContinuum
print("Running immoments('%s', moments=[0], chans='%s', outfile='%s')" % (img,selection,mom0fcIntersect))
immoments(img, moments=[0], chans=selection, outfile=mom0fcIntersect)
finalMom0fc = mom0fcIntersect
# create scaled version
mom0fcIntersectScaled = mom0fcIntersect + '.scaled'
removeIfNecessary(mom0fcIntersectScaled) # in case there was a prior run of findContinuum
chanWidthKms = 299792.458*channelWidth/meanFreqHz
fcChannels = len(convertSelectionIntoChannelList(selection))
factor = 1.0/chanWidthKms/fcChannels
immath([mom0fcIntersect], mode='evalexpr', expr='IM0*%f'%(factor), outfile=mom0fcIntersectScaled)
# Need to build a new momDiff image
momDiff['.intersect'] = mom8fcIntersect + '.minus.mom0fcScaled'
momDiffIntersect = momDiff['.intersect']
removeIfNecessary(momDiffIntersect) # in case there was a prior run of findContinuum
mymask = '"%s">0.3' % (pbmom)
print("immath(['%s','%s'], mode='evalexpr', expr='IM0-IM1', mask='%s', outfile='%s')" % (mom8fcIntersect,mom0fcIntersectScaled,mymask,momDiffIntersect))
immath([mom8fcIntersect, mom0fcIntersectScaled], mode='evalexpr', expr='IM0-IM1', mask='"%s">0.23'%(pbmom), outfile=momDiffIntersect)
mymomDiff = momDiffIntersect
finalMomDiff = momDiffIntersect
###########################################################################################
# Recalculate momDiff stats for purposes of extraMaskYesOrNo, and the second 4-letter code
###########################################################################################
# momDiffSNR, momDiffPeak = peak anywhere in image
momDiffSNR, momDiffPeak, momDiffMedian, momDiffMAD = imageSNR(mymomDiff, mask=jointMaskTest, maskWithAnnulus=jointMaskTestAnnulus,
useAnnulus=useAnnulus, returnAllStats=True, applyMaskToAll=False)
casalogPost('momDiff: %s' % (mymomDiff))
casalogPost('momDiffSNR: %f peakAnywhere: %f median: %f scaledMAD: %f' % (momDiffSNR, momDiffPeak, momDiffMedian, momDiffMAD))
# This is the SNR outside the mask
if useAnnulus:
# Need 2 calls to get peak anywhere outside the joint mask, but median and MAD from the annulus
momDiffSNROutside, momDiffPeakOutside = imageSNRAnnulus(mymomDiff, jointMaskTest, jointMaskTestAnnulus)
else:
momDiffSNROutside, momDiffPeakOutside, ignore0, ignore1 = imageSNR(mymomDiff, mask=jointMaskTest, returnAllStats=True,
applyMaskToAll=True)
momDiffSNRCube = (momDiffPeakOutside-momDiffMedian)/MADCubeOutside
casalogPost('momDiffSNRCube: %f scaledMAD: %f' % (momDiffSNRCube, MADCubeOutside))
momDiffSum = imstat(mymomDiff, listit=imstatListit)['sum'][0]
if useMomentDiff:
# These values are no longer used in extraMaskYesOrNo, so could probably not bother calculating them now.
NpixMomDiff = computeNpixMom8Median(momDiffMedian, momDiffLevel, momDiffMAD, mymomDiff, '')
NpixMomDiffBadAtm = computeNpixMom8MedianBadAtm(momDiffMedian, momDiffLevelBadAtm, momDiffMAD, mymomDiff, '')
NpixCubeDiffMedian = computeNpixCubeMedian(momDiffMedian, cubeLevel, MADCubeOutside, mymomDiff, '') # unused now
extraMaskDecision2, ExtraMaskLevel, extraMaskSigmaUsed = extraMaskYesOrNo(badAtmosphere, momDiffMedian, momDiffSNR,
momDiffSNRCube, NpixMomDiff, NpixMomDiffBadAtm,
NpixCubeDiffMedian, TenEventSigma, momDiffMAD,
MADCubeOutside, momDiffLevel, momDiffLevelBadAtm,
cubeLevel)
else:
# NpixMom8Median used here was defined above as max([5.5,TenEventSigma])
extraMaskDecision2, ExtraMaskLevel, extraMaskSigmaUsed = extraMaskYesOrNo(badAtmosphere, mom8fcMedian, mom8fcSNRMom8,
mom8fcSNRCube, NpixMom8Median, NpixMom8MedianBadAtm,
NpixCubeMedian, TenEventSigma, mom8fcMAD,
MADCubeOutside, mom8level, mom8levelBadAtm, cubeLevel)
if extraMaskDecision2 == 'No':
casalogPost('Extra Mask is no longer deemed necessary. Stopping.')
# If No, stop: because Taking intersection was enough, so do not proceed with extraMask.
###################################################
# Result3: Amended Mask and Intersection was enough
###################################################
# Derive 4-letter codes and update the plot (channel ranges and legend)
aggregateBandwidth = updateChannelRangesOnPlot(labelDescs, selection, ax1, separator,
avgspectrumAboveThreshold, skipchan,
medianTrue, positiveThreshold,
upperXlabel, ax2, channelWidth, fontsize)
warnings = gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction)
channelRanges = len(selection.split(separator))
if not useMomentDiff:
labelDescs, areaString, mom8code = compute4LetterCodeAndUpdateLegend(mom8fcPeak,
mom8fcPeak0, mom8fcPeakOutside,
mom8fcPeakOutside0, mom8fcSum, mom8fcSum0,
mom8fcMAD, mom8fcMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision,
extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
else:
# Needs momDiff 4-letter code
labelDescs, areaString, momDiffCode = compute4LetterCodeAndUpdateLegend(momDiffPeak,
momDiffPeak0, momDiffPeakOutside,
momDiffPeakOutside0, momDiffSum, momDiffSum0,
momDiffMAD, momDiffMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision,
extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
# give the modified figure a new name so that it does not overwrite the pre-extraMask png
png = png.replace('.amendedMask.meanSpectrum','.amendedMaskIntersect.meanSpectrum')
plt.savefig(png, dpi=dpi)
break
if len(amendMaskIterationNames) <= amendMaskIteration+1: # len will be 2 if amendMaskIterations was 1
extraMaskDecision = 'No'
#############################################################
# Result4: amendMaskIterations was set to only 1 (not 2 or 3)
#############################################################
aggregateBandwidth = updateChannelRangesOnPlot(labelDescs, selection, ax1, separator,
avgspectrumAboveThreshold, skipchan,
medianTrue, positiveThreshold,
upperXlabel, ax2, channelWidth, fontsize)
warnings = gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction)
channelRanges = len(selection.split(separator))
if not useMomentDiff:
labelDescs, areaString, mom8code = compute4LetterCodeAndUpdateLegend(mom8fcPeak, mom8fcPeak0, mom8fcPeakOutside,
mom8fcPeakOutside0, mom8fcSum, mom8fcSum0,
mom8fcMAD, mom8fcMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
else:
labelDescs, areaString, momDiffCode = compute4LetterCodeAndUpdateLegend(momDiffPeak,
momDiffPeak0, momDiffPeakOutside,
momDiffPeakOutside0, momDiffSum, momDiffSum0,
momDiffMAD, momDiffMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision,
extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
casalogPost('Stopping because amendMaskIterations was set less than 2')
plt.savefig(png, dpi=dpi)
break
# If Yes, do the extraMask procedure:
print("extraMaskSigmaUsed = %f, ExtraMaskLevel = %f" % (extraMaskSigmaUsed,ExtraMaskLevel))
if useMomentDiff:
channels, frequency, intensity, normalized = computeStatisticalSpectrumFromMask(img, '"%s">%f' % (momDiffIntersect,ExtraMaskLevel), pbcube, imageInfo, 'mean', normalizeByMAD, outdir=outdir, jointMaskForNormalize=userJointMask)
else:
# use mom8fc.intersect to set the mask level from fc.imageSNR calcs (using original mask)
channels, frequency, intensity, normalized = computeStatisticalSpectrumFromMask(img, '"%s">%f' % (mom8fcIntersect,ExtraMaskLevel), pbcube, imageInfo, 'mean', normalizeByMAD, outdir=outdir, jointMaskForNormalize=userJointMask)
lineFullSelection = invertChannelRanges(selection, nchan) # converts a string to a string
myChannelList = convertSelectionIntoChannelList(selection)
casalogPost('Continuum channels so far: %s' % (selection))
casalogPost('Line-full channels so far: %s' % (lineFullSelection))
scaledMADFCSubset, medianFCSubset = robustMADofContinuumRanges(myChannelList, intensity)
edgesUsed = 0
meanSpectrumFile = mymom8fc + '.meanSpectrumFile_fromExtraMask_beforeNoiseReplacement'
writeMeanSpectrum(meanSpectrumFile, frequency, intensity, intensity, ExtraMaskLevel,
nchan, edgesUsed, centralArcsec='mom0mom8jointMask')
intensity = replaceLineFullRangesWithNoise(intensity, lineFullSelection, medianFCSubset, scaledMADFCSubset)
meanSpectrumFile = mymom8fc + '.meanSpectrumFile_fromExtraMask'
writeMeanSpectrum(meanSpectrumFile, frequency, intensity, intensity, ExtraMaskLevel,
nchan, edgesUsed, centralArcsec='mom0mom8jointMask')
# update selection0 to hold the amendMask/intersect selection
selection0 = selection
if keepIntermediatePngs:
png = '' # reset name so that new name will be chosen and prior plot not overwritten
if False:
sigmaFindContinuumMode = 'autolower'
sigmaFindContinuum = finalSigmaFindContinuum
casalogPost("Using sigmaFindContinuumMode='autolower' with sigmaFindContinuum=%f" % (sigmaFindContinuum))
amendMaskIterationNames[amendMaskIteration+1] = '.extraMask'
signalRatioTier1 = 1.0
overwrite = False # do not overwrite the spectrum we just wrote when runFindContinuum runs again!
continue # from .amendedMask iteration to .extraMask iteration
else:
casalogPost('**************************************************************')
casalogPost('There is no significant emission in difference image (%f < %f).' % (diffStats[0], testDiffLev))
casalogPost('**************************************************************')
if len(amendMaskIterationNames) <= amendMaskIteration+1:
####################################################
# Result5: amendMaskIterations was to only 1 (not 2)
####################################################
extraMaskDecision = 'No'
aggregateBandwidth = updateChannelRangesOnPlot(labelDescs, selection, ax1, separator,
avgspectrumAboveThreshold, skipchan,
medianTrue, positiveThreshold,
upperXlabel, ax2, channelWidth, fontsize)
warnings = gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction)
channelRanges = len(selection.split(separator))
if not useMomentDiff:
labelDescs, areaString, mom8code = compute4LetterCodeAndUpdateLegend(mom8fcPeak, mom8fcPeak0, mom8fcPeakOutside,
mom8fcPeakOutside0, mom8fcSum, mom8fcSum0,
mom8fcMAD, mom8fcMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
else:
labelDescs, areaString, momDiffCode = compute4LetterCodeAndUpdateLegend(momDiffPeak,
momDiffPeak0, momDiffPeakOutside,
momDiffPeakOutside0, momDiffSum, momDiffSum0,
momDiffMAD, momDiffMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision,
extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
casalogPost('Stopping because amendMaskIterations was set less than 2')
plt.savefig(png, dpi=dpi)
break
# do the extraMaskProcedure:
# jointMaskTest will be the original mask, unless I take action here
casalogPost("sigmaUsed to set level = %f, computing spectrum from pixels > %f" % (extraMaskSigmaUsed,ExtraMaskLevel))
if useMomentDiff:
channels, frequency, intensity, normalized = computeStatisticalSpectrumFromMask(img, '"%s">%f' % (mymomDiff,ExtraMaskLevel), pbcube, imageInfo, 'mean', normalizeByMAD, outdir=outdir, jointMaskForNormalize=userJointMask)
else:
channels, frequency, intensity, normalized = computeStatisticalSpectrumFromMask(img, '"%s">%f' % (mymom8fc,ExtraMaskLevel), pbcube, imageInfo, 'mean', normalizeByMAD, outdir=outdir, jointMaskForNormalize=userJointMask)
lineFullSelection = invertChannelRanges(selection, nchan) # converts a string to a string
myChannelList = convertSelectionIntoChannelList(selection)
casalogPost('Continuum channels so far: %s' % (selection))
casalogPost('Line-full channels so far: %s' % (lineFullSelection))
scaledMADFCSubset, medianFCSubset = robustMADofContinuumRanges(myChannelList, intensity)
edgesUsed = 0
meanSpectrumFile = mymom8fc + '.meanSpectrumFile_fromExtraMask_beforeNoiseReplacement'
writeMeanSpectrum(meanSpectrumFile, frequency, intensity, intensity, ExtraMaskLevel,
nchan, edgesUsed, centralArcsec='mom0mom8jointMask')
intensity = replaceLineFullRangesWithNoise(intensity, lineFullSelection, medianFCSubset, scaledMADFCSubset)
meanSpectrumFile = mymom8fc + '.meanSpectrumFile_fromExtraMask'
writeMeanSpectrum(meanSpectrumFile, frequency, intensity,
intensity, ExtraMaskLevel, nchan,
edgesUsed, centralArcsec='mom0mom8jointMask')
# update selection0 to hold the amendMask/intersect selection
selection0 = selection
if keepIntermediatePngs:
png = '' # reset name so that new name will be chosen and prior plot not overwritten
amendMaskIterationNames[amendMaskIteration+1] = '.extraMask'
signalRatioTier1 = 1.0
if False:
sigmaFindContinuumMode = 'autolower'
sigmaFindContinuum = finalSigmaFindContinuum
casalogPost("Using sigmaFindContinuumMode='autolower' with sigmaFindContinuum=%f" % (sigmaFindContinuum))
overwrite = False # do not overwrite the spectrum we just wrote when runFindContinuum runs again!
continue # from .amendedMask iteration to .extraMask iteration
# end if diffStats[0] > testDiffLev else:
# end if amendMaskIterationName == '.amendedMask'
if amendMaskIterationName in ['.extraMask','.onlyExtraMask','.autoLower']:
# Intersect extraMask spectral (cyan) ranges with amendMask spectral (cyan) ranges
# selection0 will hold the amendMask selection if .extraMask was run
channelList0 = convertSelectionIntoChannelList(selection0)
channelList1 = convertSelectionIntoChannelList(selection)
channelList = np.intersect1d(channelList0,channelList1)
casalogPost("Taking intersection of %d channels with %d channels, to get %d channels" % (len(channelList0), len(channelList1), len(channelList)))
casalogPost("first channel list = %s" % (selection0))
casalogPost("second channel list = %s" % (selection))
if len(channelList) == 0:
###########################################################################################
# Result6: ExtraMask or onlyExtraMask was used but no channels were left after intersection
###########################################################################################
casalogPost("No intersection: Reverting to first channel selection")
channelList = channelList0 # reverting
# derive 4-letter codes and update the final line of the plot legend
warnings = gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction)
channelRanges = len(selection.split(separator))
if not useMomentDiff:
labelDescs, areaString, mom8code = compute4LetterCodeAndUpdateLegend(mom8fcPeak,
mom8fcPeak0, mom8fcPeakOutside,
mom8fcPeakOutside0, mom8fcSum, mom8fcSum0,
mom8fcMAD, mom8fcMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision,
extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
else:
# Needs momDiff 4-letter code, but only if it was .extraMask
labelDescs, areaString, momDiffCode = compute4LetterCodeAndUpdateLegend(momDiffPeak,
momDiffPeak0, momDiffPeakOutside,
momDiffPeakOutside0, momDiffSum, momDiffSum0,
momDiffMAD, momDiffMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision,
extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
plt.savefig(png, dpi=dpi)
break
if amendMaskIterationName in ['.onlyExtraMask']: # ,'.autoLower']:
# save the .original.meanSpectrum.*.png before modifying the channel ranges and legend below
# save the .amendedMask.meanSpectrum.*.png before modifying the channel ranges and legend below
casalogPost("Saving %s" % (png))
plt.savefig(png, dpi=dpi)
selection = convertChannelListIntoSelection(channelList)
if list(channelList) == list(channelList0):
if amendMaskIterationName == '.extraMask':
extraMaskDecision = 'NoImprovement'
elif amendMaskIterationName == '.autoLower':
autoLowerDecision = 'NoImprovement'
casalogPost('No change in channel list')
elif len(channelList) == 1 and nchan > 500:
# disallow first and final channel if either is the adjacent channel
casalogPost('intersected channel list: %s' % (selection))
onlyChannel = channelList[0]
channelList = range(np.max([1,onlyChannel-1]),np.min([onlyChannel+1,nchan-2])+1)
casalogPost('Forcing single channel to include the adjacent channels because nchan>500')
selection = convertChannelListIntoSelection(channelList)
# else: # process new channel selection (even if it is the same, because we want to produce the autoLowerIntersect momDiff)
casalogPost('intersected channel list: %s' % (selection))
##########################################
# Build a (potentially) final mom8fc image
##########################################
myIntersectName = amendMaskIterationName+'Intersect'
mom8fc[myIntersectName] = mom8fc[amendMaskIterationName] + 'Intersect'
removeIfNecessary(mom8fc[myIntersectName])
casalogPost("Running immoments('%s', moments=[8], chans='%s', outfile='%s')" % (img,selection, mom8fc[myIntersectName]))
immoments(img, moments=[8], chans=selection, outfile=mom8fc[myIntersectName])
finalMom8fc = mom8fc[myIntersectName]
# compute final 4-letter code and update the final line of the plot legend
mom8fcSNR, mom8fcPeak, mom8fcMedian, mom8fcMAD = imageSNR(mom8fc[myIntersectName], mask=jointMaskTest, maskWithAnnulus=jointMaskTestAnnulus,
useAnnulus=useAnnulus, returnAllStats=True, applyMaskToAll=False)
if useAnnulus:
# Need 2 calls to get peak anywhere outside the joint mask, but median and MAD from the annulus
mom8fcSNROutside, mom8fcPeakOutside = imageSNRAnnulus(mom8fc[myIntersectName], jointMaskTest, jointMaskTestAnnulus)
else:
mom8fcSNROutside, mom8fcPeakOutside, ignore0, ignore1 = imageSNR(mom8fc[myIntersectName], mask=jointMaskTest,
returnAllStats=True, applyMaskToAll=True)
# need to recalculate mom8fcSNRCube because the numerator has changed
mom8fcSNRCube = (mom8fcPeakOutside-mom8fcMedian)/MADCubeOutside
mom8fcSum = imstat(mom8fc[myIntersectName], listit=imstatListit)['sum'][0]
##########################################
# Build a (potentially) final mom0fc image
##########################################
mom0fc[myIntersectName] = mom0fc[amendMaskIterationName] + 'Intersect'
removeIfNecessary(mom0fc[myIntersectName])
casalogPost("Running immoments('%s', moments=[0], chans='%s', outfile='%s')" % (img, selection, mom0fc[myIntersectName]))
immoments(img, moments=[0], chans=selection, outfile=mom0fc[myIntersectName])
finalMom0fc = mom0fc[myIntersectName]
# create scaled version
mom0fcIntersectScaled = mom0fc[myIntersectName] + '.scaled'
removeIfNecessary(mom0fcIntersectScaled) # in case there was a prior run of findContinuum
chanWidthKms = 299792.458*channelWidth/meanFreqHz
fcChannels = len(convertSelectionIntoChannelList(selection))
factor = 1.0/chanWidthKms/fcChannels
immath(mom0fc[myIntersectName], mode='evalexpr', expr='IM0*%f'%(factor), outfile=mom0fcIntersectScaled)
# Need to build a new momDiff image since we have a new mom8fc[myIntersectName]
momDiff[myIntersectName] = mom8fc[myIntersectName] + '.minus.mom0fcScaled'
removeIfNecessary(momDiff[myIntersectName]) # in case there was a prior run of findContinuum
mymask = '"%s">0.3' % (pbmom)
print("immath(['%s','%s'], mode='evalexpr', expr='IM0-IM1', mask='%s', outfile='%s')" % (mom8fc[myIntersectName],mom0fcIntersectScaled,mymask,momDiff[myIntersectName]))
immath([mom8fc[myIntersectName], mom0fcIntersectScaled], mode='evalexpr', expr='IM0-IM1', mask='"%s">0.23'%(pbmom), outfile=momDiff[myIntersectName])
finalMomDiff = momDiff[myIntersectName]
# Recalculate for purposes of second 4-letter code
momDiffSNR, momDiffPeak, momDiffMedian, momDiffMAD = imageSNR(momDiff[myIntersectName], mask=jointMaskTest,
maskWithAnnulus=jointMaskTestAnnulus, useAnnulus=useAnnulus,
returnAllStats=True, applyMaskToAll=False)
casalogPost('momDiff: %s' % (momDiff[myIntersectName]))
casalogPost('momDiffSNR: %f peakAnywhere: %f median: %f scaledMAD: %f' % (momDiffSNR, momDiffPeak, momDiffMedian, momDiffMAD))
# This is the SNR outside the mask
if useAnnulus:
# Need 2 calls to get peak anywhere outside the joint mask, but median and MAD from the annulus
momDiffSNROutside, momDiffPeakOutside = imageSNRAnnulus(momDiff[myIntersectName], jointMaskTest, jointMaskTestAnnulus)
else:
momDiffSNROutside, momDiffPeakOutside, ignore0, ignore1 = imageSNR(momDiff[myIntersectName], mask=jointMaskTest,
returnAllStats=True, applyMaskToAll=True)
momDiffSum = imstat(momDiff[myIntersectName], listit=imstatListit)['sum'][0]
momDiffSNRCube = (momDiffPeakOutside-momDiffMedian)/MADCubeOutside
casalogPost('momDiffSNRCube: %f scaledMAD: %f' % (momDiffSNRCube, MADCubeOutside))
if (amendMaskIterationName == '.extraMask' and amendMaskIterations > 2) or (amendMaskIterationName == '.onlyExtraMask' and amendMaskIterations > 1):
warnings = gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction)
if len(warnings) == 0:
# These values are no longer used in extraMaskYesOrNo(), so no need to compute them
# NpixMomDiff = computeNpixMom8Median(momDiffMedian, momDiffLevel, momDiffMAD, momDiff[myIntersectName], '')
# NpixMomDiffBadAtm = computeNpixMom8MedianBadAtm(momDiffMedian, momDiffLevelBadAtm, momDiffMAD, momDiff[myIntersectName], '')
# NpixCubeDiffMedian is unused now
autoLowerDecision, autoLowerLevel, autoLowerSigmaUsed = extraMaskYesOrNo(badAtmosphere,
momDiffMedian, momDiffSNR,
momDiffSNRCube, NpixMomDiff, NpixMomDiffBadAtm,
NpixCubeDiffMedian, TenEventSigma, momDiffMAD,
MADCubeOutside, momDiffLevel, momDiffLevelBadAtm, cubeLevel)
casalogPost('autoLowerDecision = %s, autoLowerLevel = %f' % (autoLowerDecision, autoLowerLevel))
if autoLowerDecision in ['YesMom','YesCube']:
if keepIntermediatePngs:
png = '' # reset name so that new name will be chosen and prior plot not overwritten
amendMaskIterationNames[amendMaskIteration+1] = '.autoLower'
signalRatioTier1 = 1.0
sigmaFindContinuumMode = 'autolower'
sigmaFindContinuum = finalSigmaFindContinuum
selection0 = selection # be sure we intersect against this latest channel selection
casalogPost("Using sigmaFindContinuumMode='autolower' with sigmaFindContinuum=%f" % (sigmaFindContinuum))
continue # from .extraMask/.onlyExtraMask to .autoLower
else:
casalogPost("Not trying .autoLower because we have %d bandwidth warning(s)" % (len(warnings)))
else:
if amendMaskIterationName != '.autoLower':
casalogPost("Not trying autoLower because amendMaskIterations=%d" % (amendMaskIterations))
# was: endif list(channelList) == list(channelList0) else:
warnings = gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction)
channelRanges = len(selection.split(separator))
# update the plot channel ranges
casalogPost('A) calling updateChannelRangesOnPlot')
aggregateBandwidth = updateChannelRangesOnPlot(labelDescs, selection, ax1, separator,
avgspectrumAboveThreshold, skipchan,
medianTrue, positiveThreshold,
upperXlabel, ax2, channelWidth, fontsize)
########################################################################
# Result7: ExtraMask or onlyExtraMask, and possibly autoLower were used
########################################################################
if not useMomentDiff:
labelDescs, areaString, mom8code = compute4LetterCodeAndUpdateLegend(mom8fcPeak,
mom8fcPeak0, mom8fcPeakOutside,
mom8fcPeakOutside0, mom8fcSum, mom8fcSum0,
mom8fcMAD, mom8fcMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision,
extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
else:
# Needs momDiff 4-letter code
labelDescs, areaString, momDiffCode = compute4LetterCodeAndUpdateLegend(momDiffPeak,
momDiffPeak0, momDiffPeakOutside,
momDiffPeakOutside0, momDiffSum, momDiffSum0,
momDiffMAD, momDiffMAD0, thresholdForSame,
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision,
extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask)
# give the modified figure a new name so that it does not overwrite the pre-extraMask png
if amendMaskIterationName == '.onlyExtraMask':
print("onlyExtraMask) Changing name of png: ", png)
png = png.replace('.original.meanSpectrum','.onlyExtraMaskIntersect.meanSpectrum')
casalogPost("Saving %s" % png)
plt.savefig(png, dpi=dpi)
# do not allow it to try a 3rd iteration, since we are done
break
elif amendMaskIterationName == '.extraMask': # new
print("extraMask) Changing name of png: ", png)
png = png.replace('.extraMask.meanSpectrum','.extraMaskIntersect.meanSpectrum')
png = png.replace('.amendedMask.meanSpectrum','.extraMaskIntersect.meanSpectrum')
casalogPost("Saving %s" % png)
plt.savefig(png, dpi=dpi)
break # not needed if amendMaskIterations=2, but needed if it is 3
elif amendMaskIterationName == '.autoLower': # new
print("autoLower) Changing name of png: ", png)
png = png.replace('.amendedMask.meanSpectrum','.autoLowerIntersect.meanSpectrum')
png = png.replace('.original.meanSpectrum','.autoLowerIntersect.meanSpectrum')
casalogPost("%s) Saving %s" % (amendMaskIterationName,png))
plt.savefig(png, dpi=dpi)
break # needed if amendMaskIterations == 3 and we did not amendMask
# endif amendMaskIterationName in ['.extraMask','.onlyExtraMask','.autoLower']
# endif meanSpectrumMethod=='mom0mom8jointMask' and (amendMask or checkIfMaskWouldHaveBeenAmended):
# end 'for' loop over amendMaskIterations
if selection == '' and amendMask:
# I think this should never happen due to protection above, but I leave this code here, just in case.
casalogPost("Blank selection found: Reverting to prior iteration with selection: %s" % (selection))
selection = previousSelection
png = previousPng
aggregateBandwidth = previousAggregateBandwidth
##########################################################################################
# At this point, all of the images and pngs have been saved with the correct names.
# Now compare the 4-letter 'code' to see if we need to revert the channel range results,
# the png returned, and the final image returned to the .original versions of these files.
##########################################################################################
if amendMask:
reversionCodes = ['HHHH','HHHS']
revert = False
if momDiffCode is None:
code = mom8code
if mom8code in reversionCodes:
revert = True
casalogPost('This mom8fc code triggers reversion to original result.')
else:
code = momDiffCode
if momDiffCode in reversionCodes:
revert = True
casalogPost('This momDiff code triggers reversion to original result.')
if not revert:
warnings = gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction)
if len(warnings) == 2:
# do our addition
continuumChannels = convertSelectionIntoChannelList(selection)
if continuumChannels[0] > (nchan-continuumChannels[-1]):
endchan = continuumChannels[-1] - nchan*smallSpreadFraction - 1
channelRange = [0,endchan]
else:
startchan = continuumChannels[0] + nchan*smallSpreadFraction + 1
channelRange = [startchan,nchan]
mylist = findWidestContiguousListInChannelRange(allBaselineChannelsXY[0], channelRange, continuumChannels)
if mylist is None:
casalogPost(' %s ****** No blue points found in the wider side of the spectrum' % (projectCode))
else:
# we added a new selection, so need to update everything
secondSelectionAdded = '%d~%d' % (mylist[0],mylist[-1])
if selection == '':
selection = secondSelectionAdded
else:
if mylist[0] > continuumChannels[-1]:
# add to end of string (to maintain increasing order)
selection += separator + secondSelectionAdded
else:
# add to beginning of string (to maintain increasing order)
selection = secondSelectionAdded + separator + selection
casalogPost(' %s ****** Added a second selection in wider side of the spectrum: %s' % (projectCode,secondSelectionAdded))
if momDiffPeak0 is None:
# No AmendMask nor ExtraMask was done, so we need to store the original values.
# These *0's will be used to compute the MomDiff 4-letter code generation
momDiffPeak0 = momDiffPeak
momDiffPeakOutside0 = momDiffPeakOutside
momDiffMAD0 = momDiffMAD
momDiffSum0 = momDiffSum
mom8fcPeak0 = mom8fcPeak
mom8fcPeakOutside0 = mom8fcPeakOutside
mom8fcMAD0 = mom8fcMAD
mom8fcSum0 = mom8fcSum
removeRanges = True
else:
removeRanges = False
# update the plot again
casalogPost('B) calling updateChannelRangesOnPlot')
aggregateBandwidth = updateChannelRangesOnPlot(labelDescs, selection, ax1, separator,
avgspectrumAboveThreshold, skipchan,
medianTrue, positiveThreshold,
upperXlabel, ax2, channelWidth, fontsize,
remove=removeRanges)
warnings = gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction)
# recompute a new mom8fc, mom0fc, momDiff, mom8fc.minus.mom0fcScaled
##################################
# Build a final mom8fc image
##################################
myIntersectName = amendMaskIterationName+'Intersect'
mom8fc[myIntersectName] = mom8fc[amendMaskIterationName] + 'Intersect'
removeIfNecessary(mom8fc[myIntersectName])
casalogPost("Running immoments('%s', moments=[8], chans='%s', outfile='%s')" % (img,selection, mom8fc[myIntersectName]))
immoments(img, moments=[8], chans=selection, outfile=mom8fc[myIntersectName])
finalMom8fc = mom8fc[myIntersectName]
mom8fcSNR, mom8fcPeak, mom8fcMedian, mom8fcMAD = imageSNR(mom8fc[myIntersectName], mask=jointMaskTest,
maskWithAnnulus=jointMaskTestAnnulus, useAnnulus=useAnnulus,
returnAllStats=True, applyMaskToAll=False)
if useAnnulus:
# Need 2 calls to get peak anywhere outside the joint mask, but median and MAD from the annulus
mom8fcSNROutside, mom8fcPeakOutside = imageSNRAnnulus(mom8fc[myIntersectName], jointMaskTest, jointMaskTestAnnulus)
else:
mom8fcSNROutside, mom8fcPeakOutside, ignore0, ignore1 = imageSNR(mom8fc[myIntersectName], mask=jointMaskTest,
returnAllStats=True, applyMaskToAll=True)
# need to recalculate mom8fcSNRCube because the numerator has changed
mom8fcSNRCube = (mom8fcPeakOutside-mom8fcMedian)/MADCubeOutside
mom8fcSum = imstat(mom8fc[myIntersectName], listit=imstatListit)['sum'][0]
##################################
# Build a final mom0fc image
##################################
mom0fc[myIntersectName] = mom0fc[amendMaskIterationName] + 'Intersect'
removeIfNecessary(mom0fc[myIntersectName])
casalogPost("Running immoments('%s', moments=[0], chans='%s', outfile='%s')" % (img, selection, mom0fc[myIntersectName]))
immoments(img, moments=[0], chans=selection, outfile=mom0fc[myIntersectName])
finalMom0fc = mom0fc[myIntersectName]
# create scaled version
mom0fcIntersectScaled = mom0fc[myIntersectName] + '.scaled'
removeIfNecessary(mom0fcIntersectScaled) # in case there was a prior run of findContinuum
chanWidthKms = 299792.458*channelWidth/meanFreqHz
fcChannels = len(convertSelectionIntoChannelList(selection))
factor = 1.0/chanWidthKms/fcChannels
immath(mom0fc[myIntersectName], mode='evalexpr', expr='IM0*%f'%(factor), outfile=mom0fcIntersectScaled)
# Need to build a new momDiff image since we have a new mom8fc[myIntersectName]
momDiff[myIntersectName] = mom8fc[myIntersectName] + '.minus.mom0fcScaled'
removeIfNecessary(momDiff[myIntersectName]) # in case there was a prior run of findContinuum
mymask = '"%s">0.3' % (pbmom)
print("immath(['%s','%s'], mode='evalexpr', expr='IM0-IM1', mask='%s', outfile='%s')" % (mom8fc[myIntersectName],mom0fcIntersectScaled,mymask,momDiff[myIntersectName]))
immath([mom8fc[myIntersectName], mom0fcIntersectScaled], mode='evalexpr', expr='IM0-IM1', mask='"%s">0.23'%(pbmom), outfile=momDiff[myIntersectName])
finalMomDiff = momDiff[myIntersectName]
# Recalculate for purposes of second 4-letter code
momDiffSNR, momDiffPeak, momDiffMedian, momDiffMAD = imageSNR(momDiff[myIntersectName], mask=jointMaskTest,
maskWithAnnulus=jointMaskTestAnnulus, useAnnulus=useAnnulus,
returnAllStats=True, applyMaskToAll=False)
casalogPost('momDiff: %s' % (momDiff[myIntersectName]))
casalogPost('momDiffSNR: %f peakAnywhere: %f median: %f scaledMAD: %f' % (momDiffSNR, momDiffPeak, momDiffMedian, momDiffMAD))
# This is the SNR outside the mask
if useAnnulus:
# Need 2 calls to get peak anywhere outside the joint mask, but median and MAD from the annulus
momDiffSNROutside, momDiffPeakOutside = imageSNRAnnulus(momDiff[myIntersectName], jointMaskTest, jointMaskTestAnnulus)
else:
momDiffSNROutside, momDiffPeakOutside, ignore0, ignore1 = imageSNR(momDiff[myIntersectName], mask=jointMaskTest,
returnAllStats=True, applyMaskToAll=True)
momDiffSum = imstat(momDiff[myIntersectName], listit=imstatListit)['sum'][0]
momDiffSNRCube = (momDiffPeakOutside-momDiffMedian)/MADCubeOutside
casalogPost('momDiffSNRCube: %f scaledMAD: %f' % (momDiffSNRCube, MADCubeOutside))
if not useMomentDiff:
labelDescs, areaString, mom8code = compute4LetterCodeAndUpdateLegend(mom8fcPeak,
mom8fcPeak0, mom8fcPeakOutside,
mom8fcPeakOutside0, mom8fcSum, mom8fcSum0,
mom8fcMAD, mom8fcMAD0, thresholdForSame*5, # use looser definition of Same
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision,
extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask,
replace=True, addedSelection=True)
else:
# Needs momDiff 4-letter code
labelDescs, areaString, momDiffCode = compute4LetterCodeAndUpdateLegend(momDiffPeak,
momDiffPeak0, momDiffPeakOutside,
momDiffPeakOutside0, momDiffSum, momDiffSum0,
momDiffMAD, momDiffMAD0, thresholdForSame*5, # use looser definition of Same
areaString, labelDescs, fontsize, ax1,
amendMaskDecision, extraMaskDecision,
extraMaskDecision2, autoLowerDecision,
intersectionOfSelections, useMomentDiff,
warnings, channelRanges, sigmaUsedToAmendMask, replace=True, addedSelection=True)
if png == originalPng:
png = originalPng.replace('.png','_addedRange.png')
plt.savefig(png, dpi=dpi)
# but we *might* have made it worse, so check again
if momDiffCode is None:
code = mom8code
if mom8code in reversionCodes:
revert = True
casalogPost('This mom8fc code triggers reversion to original result.')
else:
code = momDiffCode
if momDiffCode in reversionCodes:
revert = True
casalogPost('This momDiff code triggers reversion to original result.')
if len(warnings) == 2:
revert = True
casalogPost('This combination of bandwidth warnings triggers reversion to original result, which had only %d warning(s).' % (len(originalWarnings)))
if revert:
finalMomDiff = momDiff['.original']
selection = originalSelection
casalogPost('code=%s: Reverting to original selection: %s' % (code,selection))
aggregateBandwidth = computeBandwidth(selection, channelWidth, 0)
if originalPng != png:
png = revertedPng
# update the modification time to current time (like Unix 'touch') so that it appears to be most recent
os.utime(png,None)
else:
os.remove(revertedPng)
##########################################################################################
# Determine the name of the finalImageToReturn, in case it was requested
##########################################################################################
if useMomentDiff:
finalImageToReturn = finalMomDiff
else:
finalImageToReturn = finalMom8fc
casalogPost('finalImageToReturn = %s' % (finalImageToReturn))
pathCode = ''
if amendMask:
momDiffSNR = imageSNR(finalImageToReturn, returnAllStats=False, applyMaskToAll=False,
mask=jointMaskTest, maskWithAnnulus=jointMaskTestAnnulus, useAnnulus=useAnnulus)
casalogPost('********************************************')
casalogPost('***** SNR in final momDiff image: %.1f *****' % (momDiffSNR))
casalogPost('********************************************')
if areaString.find(',') > 0:
pathCode = areaString.split(',')[1].strip()
casalogPost('***** pathCode: %s momDiffCode: %s *****' % (pathCode,momDiffCode))
else:
momDiffCode = ''
##########################################################################
# Finish up by generating warnings (if necessary), checking for AllCont,
# and writing summary of results to the _findContinuum.dat file
##########################################################################
if (meanSpectrumFile == ''):
meanSpectrumFile = buildMeanSpectrumFilename(img, meanSpectrumMethod, peakFilterFWHM, amendMaskIterationName)
if outdir != '':
meanSpectrumFile = os.path.join(outdir, os.path.basename(meanSpectrumFile))
if spw == '':
# Try to find the name of the spw from the image name
if os.path.basename(img).find('spw') > 0:
spw = os.path.basename(img).split('spw')[1].split('.')[0]
# only the base name of the meanSpectrumFile is used by writeContDat, not the contents
writeContDat(meanSpectrumFile, selection, png, aggregateBandwidth,
firstFreq, lastFreq, channelWidth, img, imageInfo, vis, spw=spw)
executionTimeSeconds = timeUtilities.time() - executionTimeSeconds
casalogPost("Final selection: %s" % (selection))
#####################################################################################
# Post Final warnings to the log (as they might have changed due to reversion above)
#####################################################################################
warningStrings = gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction)
warningStrings = [w.split()[-1] for w in warningStrings] # picks out LowBW and/or LowSpread at end of string
casalogPost("Final warnings: %s" % (str(warningStrings)))
if selection.find(';') > 0 or aggregateBandwidth*1e9 < 0.5*np.abs(channelWidth*nchan):
# There is more than 1 group after blue pruning, page 258, so there might be a line in that gap
# or less than half was selected for continuum, so this is dubious enough that we should clean it
allContinuum = False
else:
allContinuum = allContinuumSelected(selectionPreBluePruning, nchan)
casalogPost("final png: %s" % (png))
casalogPost("Finished findContinuum.py. Execution time: %.1f seconds" % (executionTimeSeconds))
########################################################################
# There are 16 possible return lists, based on control parameters.
# The pipeline use case (which has changed with release) is noted below.
########################################################################
if returnSnrs:
if returnAllContinuumBoolean:
if returnSigmaFindContinuum:
if returnWarnings:
return(selection, png, aggregateBandwidth, allContinuum, warningStrings, mom0snrs, mom8snrs, maxBaseline, finalSigmaFindContinuum, intersectionOfSelections, rangesDropped, amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision, finalImageToReturn, pathCode, momDiffCode)
else:
return(selection, png, aggregateBandwidth, allContinuum, mom0snrs, mom8snrs, maxBaseline, finalSigmaFindContinuum, intersectionOfSelections, rangesDropped, amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision, finalImageToReturn, pathCode, momDiffCode)
else:
if returnWarnings:
return(selection, png, aggregateBandwidth, allContinuum, warningStrings, mom0snrs, mom8snrs, maxBaseline)
else:
return(selection, png, aggregateBandwidth, allContinuum, mom0snrs, mom8snrs, maxBaseline)
else:
if returnSigmaFindContinuum:
if returnWarnings:
return(selection, png, aggregateBandwidth, warningStrings, mom0snrs, mom8snrs, maxBaseline, finalSigmaFindContinuum, intersectionOfSelections, rangesDropped, amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision, finalImageToReturn, pathCode, momDiffCode)
else:
return(selection, png, aggregateBandwidth, mom0snrs, mom8snrs, maxBaseline, finalSigmaFindContinuum, intersectionOfSelections, rangesDropped, amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision, finalImageToReturn, pathCode, momDiffCode)
else:
if returnWarnings:
return(selection, png, aggregateBandwidth, warningStrings, mom0snrs, mom8snrs, maxBaseline)
else:
return(selection, png, aggregateBandwidth, mom0snrs, mom8snrs, maxBaseline)
else:
if returnAllContinuumBoolean:
if returnSigmaFindContinuum:
if returnWarnings:
return(selection, png, aggregateBandwidth, allContinuum, warningStrings, finalSigmaFindContinuum, intersectionOfSelections, rangesDropped, amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision, finalImageToReturn, pathCode, momDiffCode)
else:
return(selection, png, aggregateBandwidth, allContinuum, finalSigmaFindContinuum, intersectionOfSelections, rangesDropped, amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision, finalImageToReturn, pathCode, momDiffCode)
else:
if returnWarnings:
# Pipeline 2020 use case
return(selection, png, aggregateBandwidth, allContinuum, warningStrings)
else:
# Pipeline Cycle 7 use case
return(selection, png, aggregateBandwidth, allContinuum)
else:
if returnSigmaFindContinuum:
if returnWarnings:
return(selection, png, aggregateBandwidth, warningStrings, finalSigmaFindContinuum, intersectionOfSelections, rangesDropped, amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision, finalImageToReturn, pathCode, momDiffCode)
else:
return(selection, png, aggregateBandwidth, finalSigmaFindContinuum, intersectionOfSelections, rangesDropped, amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision, finalImageToReturn, pathCode, momDiffCode)
else:
if returnWarnings:
return(selection, png, aggregateBandwidth, returnWarningStrings)
else:
# Pipeline Cycle 6 (and prior) use case
return(selection, png, aggregateBandwidth)
# end of findContinuum
[docs]def robustMADofContinuumRanges(myChannelList, intensity):
"""
Used in Pipeline2020 going forward.
myChannelList: subset of channels that are currently the continuum selection
intensity: list of intensities in *all* channels of the spectrum
"""
totalChannels = len(intensity)
intensityInFCSubset = intensity[myChannelList]
medianFCSubset = np.median(intensityInFCSubset)
casalogPost('Median of the %d continuum channels: %f' % (len(myChannelList), medianFCSubset))
if len(myChannelList) >= 10:
# Original simpler method:
# scaledMADFCSubset = MAD(intensityInFCSubset)
# casalogPost('using MAD of the %d continuum channels: %f' % (len(myChannelList),scaledMADFCSubset))
# This method removes the effect of a slope in the baseline on the MAD
myChannelLists = splitListIntoContiguousLists(myChannelList)
MADs = []
for mylist in myChannelLists:
MADs.append(MAD(intensity[mylist]))
scaledMADFCSubset = np.median(MADs) / np.sqrt(2)
casalogPost('using (median of the MADs)/sqrt(2) of the %d ranges: %f' % (len(myChannelLists),scaledMADFCSubset))
snr = (np.max(intensityInFCSubset)-medianFCSubset) / scaledMADFCSubset
casalogPost('SNR = (peak-median)/medianMAD = %f' % (snr))
casalogPost(' (MAD of all channels together = %f)' % (MAD(intensityInFCSubset)))
idx = np.argsort(intensityInFCSubset)
npts = len(myChannelList)
nBaselineChannels = int(np.ceil(0.19 * npts))
# If there are strong lines in the "continuum" regions, then the MAD can be not representative
# So we examine the 19% lowest channels instead
if nBaselineChannels > 35 and snr > 4: # put the cutoff between TDM and FDM240 (35/.19=184chan)
mad = MAD(intensityInFCSubset[idx][:nBaselineChannels])
# percentile = 100.0*nBaselineChannels / totalChannels
correctionFactor = 1.0 # sigmaCorrectionFactor('min', npts, percentile)
newMAD = mad * correctionFactor
casalogPost('Because there is significant signal in the continuum-free ranges, computing the MAD inferred from the lowest %d channels: %f (using correctionFactor=%f)' % (nBaselineChannels, newMAD, correctionFactor))
if newMAD < scaledMADFCSubset:
newMedianFCSubset = np.median(intensityInFCSubset[idx][:nBaselineChannels])
casalogPost('Because the MAD is lower, computing the MAD and median (%f) of the baseline channels.' % (newMedianFCSubset))
scaledMADFCSubset = np.mean([scaledMADFCSubset, newMAD])
medianFCSubset = np.mean([medianFCSubset, newMedianFCSubset])
casalogPost('Using the mean of the two methods: median=%f MAD=%f' % (medianFCSubset, scaledMADFCSubset))
# medianFCSubset = medianCorrected('min', percentile, np.median(intensityInFCSubset[idx][:nBaselineChannels]),
# scaledMADFCSubset, 1, True)
# casalogPost('Inferred median = %f' % (medianFCSubset))
else: # there are too few channels to compute a reliable MAD
casalogPost('using MAD of all channels because there are too few channels in line-free regions (%d)' % (len(myChannelList)))
scaledMADFCSubset = MAD(intensity)
return scaledMADFCSubset, medianFCSubset
[docs]def gatherWarnings(selection, chanInfo, smallBandwidthFraction, smallSpreadFraction):
"""
Used in Pipeline2020 going forward.
Return list of warning strings (often empty).
"""
warningStrings = []
warning = tooLittleBandwidth(selection, chanInfo, smallBandwidthFraction)
if warning is not None:
warningStrings.append(warning)
warning = tooLittleSpread(selection, chanInfo, smallSpreadFraction)
if warning is not None:
warningStrings.append(warning)
return warningStrings
[docs]def compute4LetterCodeAndUpdateLegend(mom8fcPeak, mom8fcPeak0, mom8fcPeakOutside,
mom8fcPeakOutside0, mom8fcSum, mom8fcSum0,
mom8fcMAD, mom8fcMAD0, thresholdForSame, areaString,
labelDescs, fontsize, ax1, amendMaskDecision,
extraMaskDecision, extraMaskDecision2, autoLowerDecision,
intersectionOfSelections,
useMomentDiff, warnings, channelRanges, sigmaUsedToAmendMask,
skipDecisionCode=False, replace=False, addedSelection=False):
"""
Used in Pipeline2020 going forward.
Add a variable length code and a 4-letter code to the end of the last line of the upper legend.
useMomentDiff: specified in the call to findContinuum()
Returns: labelDescs, areaString, mycode
"""
if replace:
areaString = areaString.split(',')[0]
if not skipDecisionCode:
decisionCode = ''
print("amend=%s, extra=%s, extra2=%s, autoLower=%s" % (amendMaskDecision, extraMaskDecision, extraMaskDecision2, autoLowerDecision))
if amendMaskDecision in ['No',False]:
if extraMaskDecision in ['No',False]:
if sigmaUsedToAmendMask == AMENDMASK_PIXEL_RATIO_EXCEEDED:
casalogPost("Adding ', PR' to the plot legend")
decisionCode += ', PR'
elif sigmaUsedToAmendMask == AMENDMASK_PIXELS_ABOVE_THRESHOLD_EXCEEDED:
casalogPost("Adding ', P#' to the plot legend")
decisionCode += ', P#'
else:
casalogPost("Adding ', S' to the plot legend")
decisionCode += ', S'
elif extraMaskDecision == 'NoImprovement':
casalogPost("Adding ', Se' to the plot legend")
decisionCode += ", Se"
elif extraMaskDecision == 'YesMom':
casalogPost("Adding ', ED' to the plot legend")
decisionCode += ', ED'
elif extraMaskDecision == 'YesCube':
casalogPost("Adding ', EC' to the plot legend")
decisionCode += ', EC'
else:
decisionCode += ', A'
if amendMaskDecision == 'YesMom':
if useMomentDiff:
decisionCode += 'D'
else:
decisionCode += '8'
elif amendMaskDecision == 'YesCube':
decisionCode += 'C'
if extraMaskDecision in ['YesMom','YesCube'] or extraMaskDecision2 in ['YesMom','YesCube']:
# if extraMaskDecision in ['YesMom','YesCube']:
# decisionCode += 'E'
if intersectionOfSelections:
decisionCode += 'I'
if extraMaskDecision2 in ['YesMom','YesCube']:
decisionCode += 'E'
else:
decisionCode += 'E'
elif intersectionOfSelections:
# this will only happen if findContinuum was run with amendMaskIterations=1 instead of 2
decisionCode += 'I'
if autoLowerDecision in ['YesMom','YesCube']:
decisionCode += 'X'
elif autoLowerDecision == 'NoImprovement':
decisionCode += 'x'
elif autoLowerDecision not in ['No',False]:
decisionCode += '?'
if addedSelection:
decisionCode += '+' # added a selection of channels at the end due to LowBW/LowSpread
areaString += decisionCode
casalogPost('Decision code: %s' % (decisionCode.split(',')[1]))
mycode = compute4LetterCode(mom8fcPeak, mom8fcPeak0, mom8fcPeakOutside,
mom8fcPeakOutside0, mom8fcSum, mom8fcSum0,
mom8fcMAD, mom8fcMAD0, thresholdForSame)
areaString += ', ' + mycode
if not skipDecisionCode:
for warning in warnings:
if warning.find('amount') > 0:
areaString += ', LowBW'
if warning.find('spread') > 0:
areaString += ', LowSpread'
labelDescs[-1].remove()
token = areaString.split(';')
areaString = ';'.join([token[0], ' %d ranges'%(channelRanges), token[2]])
casalogPost("Drawing new areaString: %s" % (areaString))
labelDesc = ax1.text(0.5,0.99-3*0.03, areaString, transform=ax1.transAxes, ha='center', size=fontsize-1)
labelDescs.append(labelDesc)
plt.draw()
return labelDescs, areaString, mycode
[docs]def updateChannelRangesOnPlot(labelDescs, selection, ax1, separator, avgspectrumAboveThreshold,
skipchan, medianTrue, positiveThreshold, upperXlabel, ax2,
channelWidth, fontsize=10, remove=True):
"""
Used in Pipeline2020 going forward.
channelWidth: in Hz
Returns: new aggregate bandwidth in GHz
"""
if remove:
# exclude final item, which is the 3rd line of the legend
# print("labelDescs: ", labelDescs)
for ld,labelDesc in enumerate(labelDescs[:-1]):
# print("type(labelDesc) = ", type(labelDesc))
if type(labelDesc) == list: # plot artists (i.e., the cyan lines)
labelDesc.pop(0).remove()
else: # text artists (i.e., the chan0~chan1 labels above the cyan lines)
try:
labelDesc.remove()
except:
# prevent a crash; not sure why this happens sometimes on the first call
pass
print("Calling plotChannelSelections: ", selection)
labelDescs = plotChannelSelections(ax1, selection, separator,
avgspectrumAboveThreshold, skipchan, medianTrue,
positiveThreshold)
loc = upperXlabel.find('contBW')
aggregateBandwidth = computeBandwidth(selection, channelWidth, 0)
if loc > 0:
upperXlabel = upperXlabel[:loc] + 'contBW: %.4f MHz' % (aggregateBandwidth * 1000)
casalogPost("Setting new upper x-axis label: %s" % (upperXlabel))
ax2.set_xlabel(upperXlabel, size=fontsize) # this artist cannot be removed in CASA 5.6, so just overwrite it
return aggregateBandwidth # , labelDescs
[docs]def replaceLineFullRangesWithNoise(intensity, selection, median, scaledMAD):
"""
Used in Pipeline2020 going forward.
intensity: an array of intensity values of *all* channels in the spectrum
selection: a channel selection string in CASA format, whose intensities should be replaced
median: of the channels *not* in the selection string (for the noise generation)
scaledMAD: of the channels *not* in the selection string (for the noise generation)
Returns: a new intensity array, of the same dimension as the input array
"""
ranges = selection.split(';')
random.seed(2001) # A Space Odyssey (guarantees that .onlyExtraMask will get same result everytime)
for myrange in ranges:
c0,c1 = [int(i) for i in myrange.split('~')]
peakSNR = (np.max(intensity[c0:c1+1]) - median) / scaledMAD
if peakSNR > 15:
# Note because the continuum is still in this spectrum, it can have a slope which
# will make the MAD higher than normal, so we may need to account for this he scaledMAD
# somewhat.
intensity[c0:c1+1] = median + pickRandomErrors(c1-c0+1) * scaledMAD
casalogPost("Replacing channel range %d:%d (peakSNR=%f) with median=%f +- randomNumber*scaledMAD=%f" % (c0,c1,peakSNR,median,scaledMAD))
else:
casalogPost("Not replacing channel range %d:%d (peakSNR=%f)" % (c0,c1,peakSNR))
return intensity
[docs]def compute4LetterCode(mom8fcPeak, mom8fcPeak0, mom8fcPeakOutside, mom8fcPeakOutside0, mom8fcSum,
mom8fcSum0, mom8fcMAD, mom8fcMAD0, threshold=0.01, returnRatios=False):
"""
Used in Pipeline2020 going forward.
threshold: to use for Peak, PeakOutside, and Sum; double this value is used for MAD (i.e. 2%)
"""
better = ''
changeInPeakInside = (mom8fcPeak-mom8fcPeak0)/mom8fcPeak0
ratios = [] # these will be negative values when the quantity improved
ratios.append(changeInPeakInside)
if changeInPeakInside > threshold:
casalogPost('peak inside = H because (%f-%f)/%f=%f is > %f' % (mom8fcPeak,mom8fcPeak0,mom8fcPeak0,changeInPeakInside,threshold))
better += 'H'
elif changeInPeakInside < -threshold:
casalogPost('peak inside = L because (%f-%f)/%f=%f is < -%f' % (mom8fcPeak,mom8fcPeak0,mom8fcPeak0,changeInPeakInside,threshold))
better += 'L'
else:
casalogPost('peak inside = S because (%f-%f)/%f=%f is between -%f and +%f' % (mom8fcPeak,mom8fcPeak0,mom8fcPeak0,changeInPeakInside,threshold,threshold))
better += 'S'
changeInPeakOutside = (mom8fcPeakOutside-mom8fcPeakOutside0)/mom8fcPeakOutside0
ratios.append(changeInPeakOutside)
if changeInPeakOutside > threshold:
casalogPost('peak outside = H because (%f-%f)/%f=%f is > %f' % (mom8fcPeakOutside,mom8fcPeakOutside0,mom8fcPeakOutside0,changeInPeakOutside,threshold))
better += 'H'
elif changeInPeakOutside < -threshold:
casalogPost('peak outside = L because (%f-%f)/%f=%f is < -%f' % (mom8fcPeakOutside,mom8fcPeakOutside0,mom8fcPeakOutside0,changeInPeakOutside,threshold))
better += 'L'
else:
casalogPost('peak outside = S because (%f-%f)/%f=%f is between -%f and +%f' % (mom8fcPeakOutside,mom8fcPeakOutside0,mom8fcPeakOutside0,changeInPeakOutside,threshold,threshold))
better += 'S'
changeInSum = (mom8fcSum-mom8fcSum0)/mom8fcSum0
ratios.append(changeInSum)
if changeInSum > threshold:
casalogPost('sum = H because (%f-%f)/%f=%f is > %f ' % (mom8fcSum,mom8fcSum0,mom8fcSum0,changeInSum,threshold))
better += 'H'
elif changeInSum < -threshold:
casalogPost('sum = L because (%f-%f)/%f=%f is < -%f ' % (mom8fcSum,mom8fcSum0,mom8fcSum0,changeInSum,threshold))
better += 'L'
else:
casalogPost('sum = S because (%f-%f)/%f=%f is between -%f and +%f ' % (mom8fcSum,mom8fcSum0,mom8fcSum0,changeInSum,threshold,threshold))
better += 'S'
changeInMad = (mom8fcMAD-mom8fcMAD0)/mom8fcMAD0
ratios.append(changeInMad)
madThreshold = 2*threshold
if changeInMad > madThreshold:
casalogPost('MAD = H because (%f-%f)/%f=%f is > %f ' % (mom8fcMAD,mom8fcMAD0,mom8fcMAD0,changeInMad,madThreshold))
better += 'H'
elif changeInMad < -madThreshold:
casalogPost('MAD = L because (%f-%f)/%f=%f is < -%f ' % (mom8fcMAD,mom8fcMAD0,mom8fcMAD0,changeInMad,madThreshold))
better += 'L'
else:
casalogPost('MAD = S because (%f-%f)/%f=%f is between -%f and +%f' % (mom8fcMAD,mom8fcMAD0,mom8fcMAD0,changeInMad,madThreshold,madThreshold))
better += 'S'
casalogPost("4-letter code: %s" % (better))
if returnRatios:
return better, ratios
else:
return better
[docs]def cubeNoiseLevel(cube, pbcube='', mask='', percentile=50, chans=''):
"""
Used in Pipeline2020 going forward.
Computes median of the per-channel scaled MAD within the standard noise annulus
and outside the specified mask
chans: channel selection string (passed to imstat)
mask: either mask image to use (e.g. the string passed to imstat will be '"mask" == 0')
or a proper mask string containing >, < or ==
percentile: compute this percentile value of the per-channel scaled MAD
(examples: 50 = median, 90 = close to the highest value)
if it is a list, then a list of values is returned
Returns: median of MAD, median
or [list of MADs], median
"""
if not os.path.exists(cube):
print("Could not find the cube")
return
if pbcube == '':
pbcube = cube.replace('.image','.pb').replace('.residual','.pb')
if not os.path.exists(pbcube):
casalogPost("Could not find the corresponding pbcube: %s" % (pbcube))
casalogPost("Will use the whole image (or the specified mask) to compute the MAD.")
if mask == '':
mask = cube.replace('.image','.mask').replace('.residual','.mask')
if not os.path.exists(mask):
mask = cube + '.joint.mask2'
if not os.path.exists(mask):
mask = cube + '.joint.mask'
if not os.path.exists(mask):
print("Could not find any corresponding clean mask or findContinuum jointmask")
mask = ''
if os.path.exists(pbcube):
lowerAnnulusLevel, higherAnnulusLevel = findOuterAnnulusForPBCube(pbcube, False, False)
if mask != '':
if mask.find("==") < 0 and mask.find(">") < 0 and mask.find("<") < 0:
mask = '"'+mask+'"==0'
print(" Using mask: ", mask)
mymask = '"%s">%f && "%s"<%f && %s' % (pbcube, lowerAnnulusLevel, pbcube, higherAnnulusLevel, mask)
else:
mymask = '"%s">%f && "%s"<%f' % (pbcube, lowerAnnulusLevel, pbcube, higherAnnulusLevel)
else:
mymask = ''
print(" mymask expression = ", mymask)
results = imstat(cube, axes=[0,1,2], chans=chans, mask=mymask, stretch=True, listit=imstatListit)
cubeMedian = results['median'][0]
result = results['medabsdevmed']
if type(percentile) != list and type(percentile) != np.ndarray:
percentile = [percentile]
cubeScaledMAD = []
for p in percentile:
cubeScaledMAD.append(scoreatpercentile(result,p) / 0.6745)
if len(cubeScaledMAD) > 1:
return cubeScaledMAD, cubeMedian
else:
return cubeScaledMAD[0], cubeMedian
[docs]def computeSpread(selection, channelWidth):
"""
Used in Pipeline2020 going forward.
"""
ranges = selection.split(';')
for i,r in enumerate(ranges):
result = r.split('~')
if (len(result) == 2):
a,b = result
if i == 0:
startchan = int(a)
# keep extending the endchan where there is more than 1 group
endchan = int(b)
return (endchan-startchan+1)*channelWidth
[docs]def amendMaskYesOrNo(badAtmosphere, median, momSNR, momSNRCube, Npix, NpixBadAtm, NpixCube,
TenEventSigma, MADMomOutside, MADCubeOutside, momLevel,
momLevelBadAtm, cubeLevel, NpixCube2, fractionNegativePixels,
Npix2=None, Npix2BadAtm=None, verbose=True):
"""
Used in Pipeline2020 going forward.
median: meidan of mom8fc img if fc(useMomentDiff=False), or momDiff (mom8-mom0scaled) if useMomentDiff=True
momSNR: SNR of mom8fc img if fc(useMomentDiff=False), or momDiff (mom8-mom0scaled) if useMomentDiff=True
etc.
badAtmosphere: either a boolean or a string ('goodAtm' or 'badAtm')
"""
AMENDMASK_PIXEL_RATIO_EXCEEDED = -1
AMENDMASK_PIXELS_ABOVE_THRESHOLD_EXCEEDED = -2
decision = 'No'
AmendMaskLevel = 0.0
sigmaUsed = AMENDMASK_PIXEL_RATIO_EXCEEDED
if Npix2 is not None:
if Npix2 > 0:
pixelRatio = float(Npix)/Npix2
if pixelRatio > 2.2: # IRAS16293 maser gives 2.1667, NGC6334I spw16 gives 2.333
casalogPost('Pixel ratio is too high to trigger AmendMask: %f > 2.2' % (pixelRatio))
return decision, AmendMaskLevel, sigmaUsed
if badAtmosphere in ['badAtm',True]:
badAtmosphere = True
if fractionNegativePixels > 0.1 and NpixBadAtm > 850:
sigmaUsed = AMENDMASK_PIXELS_ABOVE_THRESHOLD_EXCEEDED
casalogPost('Fraction of negative pixels (%f) > 0.1 and too many pixels (%d) above threshold to trigger AmendMask' % (fractionNegativePixels,NpixBadAtm))
return decision, AmendMaskLevel, sigmaUsed
elif badAtmosphere in ['goodAtm',False]:
badAtmosphere = False
if fractionNegativePixels > 0.1 and Npix > 850:
sigmaUsed = AMENDMASK_PIXELS_ABOVE_THRESHOLD_EXCEEDED
casalogPost('Fraction of negative pixels (%f) > 0.1 and too many pixels (%d) above threshold to trigger AmendMask' % (fractionNegativePixels,Npix))
return decision, AmendMaskLevel, sigmaUsed
else:
casalogPost('fractionNegativePixels = %f, Npix = %d, allows it to trigger AmendMask' % (fractionNegativePixels,Npix))
else:
casalogPost('badAtmosphere has unrecognized value = %s' % (badAtmosphere))
sigmaUsed = 0 # this is the value that will be returned if threshold are not met
if badAtmosphere:
casalogPost("Using badAtmosphere thresholds: momLevelBadAtm=%f" % (momLevelBadAtm))
if momSNR >= momLevelBadAtm and NpixBadAtm >= 9:
decision = 'YesMom'
sigmaUsed = 8
AmendMaskLevel = median + sigmaUsed*MADMomOutside
casalogPost('Found YesMom (BadAtm) with sigma=%g yielding AmendMaskLevel = %f' % (sigmaUsed,AmendMaskLevel))
else:
if verbose:
print("-------------------------------------")
print("amendMaskYesOrNo(): median=%f, MADMomOutside=%f, momSNR=%f, Npix=%.f" % (median, MADMomOutside, momSNR, Npix))
print("amendMaskYesOrNo(): momSNRCube=%f, MADCubeOutside=%f, NpixCube=%.1f" % (momSNRCube, MADCubeOutside, NpixCube))
print("-------------------------------------")
if momSNR >= momLevel and Npix >= 9:
decision = 'YesMom'
AmendMaskLevel = median + np.max([5.5,TenEventSigma]) * MADMomOutside
sigmaUsed = np.max([5.5,TenEventSigma])
casalogPost('Found YesMom with sigma=%g (because momSNR=%f >= momLevel=%f) yielding AmendMaskLevel = %f' % (sigmaUsed,momSNR,momLevel,AmendMaskLevel))
elif momSNRCube >= cubeLevel and NpixCube >= 9:
# Need to guard against divide by zero below, so we only assess the
# ratio if we know that NpixCube > 0, rather than making it
# a 3-term 'elif' clause above.
# The float() below is to insure real arithmetic in CASA < 6.
if float(NpixCube2)/NpixCube < 2.0:
decision = 'YesCube'
AmendMaskLevel = median + np.max([5.0,TenEventSigma]) * MADCubeOutside
sigmaUsed = np.max([5.0,TenEventSigma])
casalogPost('YesCube: sigmaUsed = %f' % (sigmaUsed))
return decision, AmendMaskLevel, sigmaUsed
[docs]def allContinuumSelected(selection, nchan, fraction=ALL_CONTINUUM_CRITERION):
"""
Returns True if only one range is selected and that range covers sufficient
bandwidth (default fraction = 92.5%)
nchan: number of channels in the spectrum
fraction: default = 0.925 which is 92.5%
"""
ranges = selection.split(';')
if len(ranges) == 1:
if ranges[0].find('~') > 0:
c0,c1 = [int(i) for i in ranges[0].split('~')]
elif ranges[0] == '':
c0 = 1
c1 = 0
else: # in case a single channel is selected
c0,c1 = [int(ranges[0]),int(ranges[0])]
# Require this single range to be centered:
# if c0 <= fraction*nchan and c1 >= (1-fraction)*(nchan-1):
# Range is not required to be centered:
myfraction = (c1-c0+1)*1.0 / nchan
if (myfraction >= fraction):
casalogPost("continuum fraction: %d-%d: %f >= %f" % (c0,c1,myfraction, fraction))
return True
else:
casalogPost("continuum fraction: %d-%d: %f < %f" % (c0,c1,myfraction, fraction))
return False
[docs]def byteDecode(buffer):
if casaVersion >= '5.9.9':
return buffer.decode('utf-8')
else:
return buffer
[docs]def maskArgumentMismatch(mask, meanSpectrumFile, useThresholdWithMask):
"""
This function is called by checkForMismatch.
Determines if the requested mask does not match what was used to generate
the specified meanSpectrumFile.
Returns: True or False
"""
casalogPost('Entered maskArgumentMismatch()')
grepResult = byteDecode(grep(meanSpectrumFile,'mask')[0])
if (mask == '' or mask == False):
if (grepResult != ''):
casalogPost('Mismatch: mask was used previously, but we did not request one this time')
# mask was used previously, but we did not request one this time
return True
else:
# mask was not used previously and we are not requesting one this time
casalogPost('No mismatch: mask was not used previously and we are not requesting one this time')
False
elif (grepResult == ''):
# requested mask was not used previously
casalogPost('Mismatch: requested mask was not used previously')
return True
else:
# requested mask was used previously. Except if the new mask is a subset
# of the old name, so check for that possibility.
tokens = grep(meanSpectrumFile,mask)[0].split()
newtoken = []
for token in tokens:
newtoken.append(byteDecode(token))
tokens = newtoken
if mask not in tokens:
casalogPost('Mismatch: mask name not in meanSpectrumFile')
return True
else:
# check if threshold was 0.00 before and we are using one now
f = open(meanSpectrumFile,'r')
lines = f.readlines()
f.close()
token = lines[1].split()
threshold = float(token[0])
if (useThresholdWithMask and threshold == 0.0) or (not useThresholdWithMask and threshold != 0.0):
casalogPost('Mismatch in threshold: useThresholdWithMask=%s, threshold=%f' % (useThresholdWithMask, threshold))
return True
casalogPost('No mismatch in threshold')
return False
[docs]def centralArcsecArgumentMismatch(centralArcsec, meanSpectrumFile, iteration):
"""
This function is called by checkForMismatch.
Determines if the requested centralArcsec value does not match what was used to
generate the specified meanSpectrumFile.
Returns: True or False
"""
if (centralArcsec == 'auto' or centralArcsec == -1):
if (grep(meanSpectrumFile,'centralArcsec=auto')[0] == '' and
grep(meanSpectrumFile,'centralArcsec=-1')[0] == ''):
casalogPost("request for auto but 'centralArcsec=auto' not found and 'centralArcsec=-1' not found")
return True
else:
result = grep(meanSpectrumFile,'centralArcsec=auto')[0]
if (iteration == 0 and result != ''):
# This will force re-run of any result that went to a smaller size.
token = result.split('=auto')
if (len(token) > 1):
if (token[1].strip().replace('.','').isdigit()):
return True
else:
return False
else:
return False
else:
return False
elif (grep(meanSpectrumFile,'centralArcsec=auto %s'%(str(centralArcsec)))[0] == '' and
grep(meanSpectrumFile,'centralArcsec=mom0mom8jointMask True')[0] == '' and
grep(meanSpectrumFile,'centralArcsec=mom0mom8jointMask False')[0] == '' and
grep(meanSpectrumFile,'centralArcsec=%s'%(str(centralArcsec)))[0] == ''):
token = grep(meanSpectrumFile,'centralArcsec=')[0].split('centralArcsec=')
# the Boolean after centralArcsec=mom0mom8jointMask is: initialQuadraticRemoved
if (len(token) < 2):
# This should never happen, but if it does, prevent a crash by returning now.
casalogPost("Did not find string 'centralArcsec=' with a value in the meanSpectrum file.")
return True
value = token[1].replace('auto ','').split()[0]
print(" value = ", value)
if value.find('mom0mom8jointMask') == 0:
return False
try:
previousRequest = float(value)
centralArcsecThresholdPercent = 2
if (100*abs(previousRequest-centralArcsec)/centralArcsec < centralArcsecThresholdPercent):
casalogPost("request for specific value (%s) and previous value (%s) is within %d percent" % (str(centralArcsec),str(previousRequest),centralArcsecThresholdPercent))
return False
else:
casalogPost("request for specific value but 'centralArcsec=auto %s' not within %d percent" % (str(centralArcsec),centralArcsecThresholdPercent))
return True
except:
# If there is any trouble reading the previous value, then return now.
casalogPost("Failed to parse floating point value: %s" % str(value), debug=True)
return True
else:
return False
[docs]def plotMeanSpectrum(meanSpectrumFile, plotfile='', selection=''):
"""
Reads a *.findcont.residual.meanSpectrum.<method> file and plots it as
intensity vs. channel.
selection: channel selection string to draw in cyan horizontal lines
"""
if not os.path.exists(meanSpectrumFile):
print("Could not find file: ", meanSpectrumFile)
return
f = open(meanSpectrumFile,'r')
lines = f.readlines()
f.close()
channel = []
intensity = []
for line in lines[3:]:
a,b,c,d = line.split()
channel.append(int(a))
intensity.append(float(c))
plt.clf()
plt.plot(channel, intensity, 'k-')
plt.xlabel('Channel')
plt.ylabel('Intensity')
if selection != '':
myChannelLists = splitListIntoContiguousLists(convertSelectionIntoChannelList(selection))
for myChannelList in myChannelLists:
plt.plot(myChannelList, [0] * len(myChannelList), 'c-', lw=2)
plt.draw()
if plotfile != '':
if plotfile == True:
plotfile = meanSpectrumFile + '.png'
plt.savefig(plotfile)
print("Wrote ", plotfile)
[docs]def readContDat(filename):
"""
Reads the channel selection for a _findContinuum.dat file
"""
f = open(filename,'r')
line = f.readlines()
selection = line[0].split()[0]
f.close()
return selection
[docs]def readContDatAggregateContinuum(filename):
"""
Reads the aggregate continuum from a _findContinuum.dat file.
Returns: value in GHz
"""
f = open(filename,'r')
line = f.readlines()
selection = float(line[0].split()[-1])
f.close()
return selection
[docs]def writeContDat(meanSpectrumFile, selection, png, aggregateBandwidth,
firstFreq, lastFreq, channelWidth, img, imageInfo, vis='',
numchan=None, chanInfo=None, lsrkwidth=None, spw=''):
"""
This function is called by findContinuum.
Writes out an ASCII file called <meanSpectrumFile>_findContinuum.dat
that contains the selected channels, the png name and the aggregate
bandwidth in GHz.
Returns: None
meanSpectrumFile: only used to generate the name of the .dat file, unless
firstFreq <= 0, in which case the readPreviousMeanSpectrum is called on it
(It splits off name before '.meanSpectrum' and appends _findContinuum.dat)
vis: if specified, then also write a line with the topocentric velocity ranges
spw: integer or string ID number; if specified (along with vis), then also write
a final line with the topocentric channel ranges for this spw
"""
if (meanSpectrumFile.find('.meanSpectrumFile') > 0):
# contDat = meanSpectrumFile.split('.meanSpectrum')[0] + '_findContinuum.dat'
# remove meanSpectrum from the name to avoid confusion that this is not a mean spectrum file
contDat = meanSpectrumFile.replace('.meanSpectrumFile','') + '_findContinuum.dat'
else:
contDat = meanSpectrumFile + '_findContinuum.dat'
contDatDir = os.path.dirname(contDat)
if firstFreq <= 0:
result = readPreviousMeanSpectrum(meanSpectrumFile)
nchan = result[4]
firstFreq = result[6]
lastFreq = result[7]
if (firstFreq > lastFreq and channelWidth > 0):
# restore negative channel widths if appropriate
channelWidth *= -1
# print("firstFreq=%f, channelWidth=%f" % (firstFreq,channelWidth))
lsrkfreqs = 1e-9*np.arange(firstFreq, lastFreq+channelWidth*0.5, channelWidth)
if (len(contDatDir) < 1):
contDatDir = '.'
if (not os.access(contDatDir, os.W_OK) and contDatDir != '.'):
# Tf there is no write permission, then use the directory of the png
# since this has been established as writeable in runFindContinuum.
contDat = os.path.join(os.path.dirname(png), os.path.basename(contDat))
# try:
if True:
f = open(contDat, 'w')
casalogPost('aggregateBandwidth = %s' % (aggregateBandwidth))
f.write('%s %s %g\n' % (selection, png, aggregateBandwidth))
# Now write the LSRK frequency ranges on a new line
freqRange = ''
for i,s in enumerate(selection.split(';')):
c0,c1 = [int(j) for j in s.split('~')]
minFreq = np.min([lsrkfreqs[c0],lsrkfreqs[c1]])-0.5*abs(channelWidth*1e-9)
maxFreq = np.max([lsrkfreqs[c0],lsrkfreqs[c1]])+0.5*abs(channelWidth*1e-9)
freqRange += '%.9fGHz~%.9fGHz LSRK ' % (minFreq,maxFreq)
f.write(freqRange+'\n')
if (len(vis) > 0):
# vis is a non-blank list or non-blank string
if (type(vis) == str):
vis = vis.split(',')
# vis is now assured to be a non-blank list
topoFreqRanges = [] # this will be a list of lists
for v in vis:
casalogPost("Converting from LSRK to TOPO for vis = %s" % (v))
vselection = ''
for i,s in enumerate(selection.split(';')):
c0,c1 = [int(j) for j in s.split('~')]
minFreq = np.min([lsrkfreqs[c0],lsrkfreqs[c1]])-0.5*abs(channelWidth*1e-9)
maxFreq = np.max([lsrkfreqs[c0],lsrkfreqs[c1]])+0.5*abs(channelWidth*1e-9)
freqRange = '%.5fGHz~%.5fGHz' % (minFreq,maxFreq)
casalogPost("LSRK freqRange = %s" % str(freqRange))
if img == '':
result = cubeLSRKToTopo(img, imageInfo, freqRange, vis=v, nchan=nchan, f0=firstFreq, f1=lastFreq, chanwidth=channelWidth)*1e-9
else:
result = cubeLSRKToTopo(img, imageInfo, freqRange, vis=v)*1e-9
# pipeline calls uvcontfit with GHz label only on upper freq
freqRange = '%.5f~%.5fGHz' % (np.min(result),np.max(result))
casalogPost("Topo freqRange = %s" % str(freqRange))
if (i > 0): vselection += ';'
vselection += freqRange
f.write('%s %s\n' % (v,vselection))
topoFreqRanges.append(vselection)
if spw != '':
for i,v in enumerate(vis):
topoChanRanges = topoFreqRangeListToChannel(freqlist=topoFreqRanges[i], spw=spw, vis=v)
f.write('%s %s\n' % (v,topoChanRanges))
f.close()
casalogPost("Wrote %s" % (contDat))
# except:
# casalogPost("Failed to write %s" % (contDat))
[docs]def getFieldnameFromPipelineImageName(img, verbose=False):
"""
Extracts the field name from the pipeline image file name.
-Todd Hunter
"""
basename = os.path.basename(img)
fieldname = ''
styles = ['sci','chk','bp','ph','flux','pol'] # I am anticpating that polcal will be called pol.
for style in styles:
if basename.find('_'+style) > 0:
fieldname = basename.split('_'+style)[0]
if fieldname == '':
print("No image found of these styles: ", styels)
return fieldname
# this will now be: a_Xb.s35_0.NGC300
fieldname = fieldname.split('_0.')[1]
fieldname = fieldname.strip('_')
return fieldname
[docs]def getSpwFromPipelineImageName(img, verbose=False):
"""
Extracts the spw ID from the pipeline image file name.
-Todd Hunter
"""
sourceName = os.path.basename(img.rstrip('/'))
if (sourceName.find('.mfs.') < 0):
if (sourceName.find('.cube.') < 0):
return 'sourcename'
else:
sourceName = sourceName.split('.cube.')[0]
else:
sourceName = sourceName.split('.mfs.')[0]
sourceName = sourceName.split('.spw')[1]
if sourceName.isdigit():
return int(sourceName)
else:
return None
[docs]def combineContDat(contdatlist, outputfile='cont.dat', fieldname='', spw=''):
"""
Takes a list of *.dat files created by findContinuum for a single field/spw
and pulls the line that contains LSRK frequency ranges from each one, and
writes it to a new cont.dat file suitable for the pipeline that contains an
entry for each spw of that field.
contdatlist: a list or comma-delimited string of the .dat files, or of
the cube names (to which '_findContinuum.dat' will be appended)
fieldname: use this value if specified, otherwise parse from filename
spw: integer or string integer, use this value if specified, otherwise parse
from the filename
Note: It assumes that none of the files originate from an ALLcont spw, so it will
never write the "ALL" line that the pipeline triggers on. But this feature could be
added by calling the following function if there is only one range of channels read:
allContinuumSelected(selection, nchan, fraction=ALL_CONTINUUM_CRITERION)
We would only need to know nchan.
"""
if type(contdatlist) == str:
contdatlist = contdatlist.split(',')
for c,contdat in enumerate(contdatlist):
if not os.path.exists(contdat) or os.path.isdir(contdat):
if contdat.find('.dat') < 0:
contdatlist[c] = contdat + '_findContinuum.dat'
for c,contdat in enumerate(contdatlist):
if not os.path.exists(contdat):
print("Could not find file: ", contdat)
return
o = open(outputfile,'w')
previousFieldname = ''
for contdat in contdatlist:
cube = contdat.split('_findContinuum.dat')[0]
if fieldname == '':
fieldname = getFieldnameFromPipelineImageName(cube)
if spw == '':
spw = getSpwFromPipelineImageName(cube)
f = open(contdat,'r')
lines = f.readlines()
f.close()
ranges = lines[1].split('LSRK')
if fieldname != previousFieldname:
o.write('Field: %s\n\n' % (fieldname))
o.write('SpectralWindow: %s\n' % (str(spw)))
for myrange in ranges:
myrange = myrange.strip()
if len(myrange) > 2: # could be a line with only a few blanks
if myrange.count('GHz') > 1:
# remove GHz from the first frequency (if it is present on both)
myrange = myrange.replace('GHz', '', 1)
o.write('%s LSRK\n' % (myrange))
o.write('\n')
o.flush()
previousFieldname = fieldname
spw = ''
o.close()
[docs]def drawYlabel(img, typeOfMeanSpectrum, meanSpectrumMethod, meanSpectrumThreshold,
peakFilterFWHM, fontsize, mask, useThresholdWithMask, normalized=False):
"""
This function is called by runFindContinuum.
Draws a descriptive y-axis label based on the origin and type of mean spectrum used.
Returns: None
"""
if (img == ''):
label = 'Mean spectrum passed in as ASCII file'
else:
if (meanSpectrumMethod.find('meanAboveThreshold') >= 0):
if (meanSpectrumMethod.find('OverMad') > 0):
label = '(Average spectrum > threshold=(%g))/MAD' % (roundFigures(meanSpectrumThreshold,3))
elif (meanSpectrumMethod.find('OverRms') > 0):
label = '(Average spectrum > threshold=(%g))/RMS' % (roundFigures(meanSpectrumThreshold,3))
elif (useThresholdWithMask or mask==''):
label = 'Average spectrum > threshold=(%g)' % (roundFigures(meanSpectrumThreshold,3))
else:
label = 'Average spectrum within mask'
elif (meanSpectrumMethod.find('peakOverMad')>=0):
if peakFilterFWHM > 1:
label = 'Per-channel (Peak / MAD) of image smoothed by FWHM=%d pixels' % (peakFilterFWHM)
else:
label = 'Per-channel (Peak / MAD)'
elif (meanSpectrumMethod.find('peakOverRms')>=0):
if peakFilterFWHM > 1:
label = 'Per-channel (Peak / RMS) of image smoothed by FWHM=%d pixels' % (peakFilterFWHM)
else:
label = 'Per-channel (Peak / RMS)'
elif meanSpectrumMethod == 'mom0mom8jointMask':
label = 'Mean profile from mom0+8 joint mask'
if normalized:
label += ' (normalized by MAD)'
else:
label += ' (not normalized by MAD)'
else:
label = 'Unknown method'
plt.ylabel(typeOfMeanSpectrum + ' ' + label, size=fontsize)
[docs]def computeBandwidth(selection, channelWidth, loc=-1):
"""
This function is called by runFindContinuum and findContinuum.
selection: a string of format: '5~6;9~20'
channelWidth: in Hz
Returns: bandwidth in GHz
"""
ranges = selection.split(';')
channels = 0
for r in ranges:
result = r.split('~')
if (len(result) == 2):
a,b = result
channels += int(b)-int(a)+1
aggregateBW = channels * abs(channelWidth) * 1e-9
return(aggregateBW)
[docs]def buildMeanSpectrumFilename(img, meanSpectrumMethod, peakFilterFWHM, amendMaskIterationName=''):
"""
This function is called by findContinuum and runFindContinuum.
Creates the name of the meanSpectrumFile to search for and/or create.
Returns: a string
"""
if (meanSpectrumMethod.find('peak')>=0):
return(img + '.meanSpectrum.'+meanSpectrumMethod+'_%d'%peakFilterFWHM+amendMaskIterationName)
else:
if (meanSpectrumMethod == 'meanAboveThreshold'):
return(img + '.meanSpectrum'+amendMaskIterationName)
else:
return(img + '.meanSpectrum.'+meanSpectrumMethod+amendMaskIterationName)
[docs]def tdmSpectrum(channelWidth, nchan):
"""
This function is called by runFindContinuum and findContinuum.
Use 15e6 instead of 15.625e6 because LSRK channel width can be slightly narrower than TOPO.
Works for single, dual, or full-polarization.
Returns: True or False
"""
if ((channelWidth >= 15e6/2. and nchan>240) or # 1 pol TDM, must avoid 240-chan FDM
(channelWidth >= 15e6)):
# (channelWidth >= 15e6 and nchan<96) or # 2 pol TDM (128 chan)
# (channelWidth >= 30e6 and nchan<96)): # 4 pol TDM (64 chan)
return True
else:
return False
[docs]def atmosphereVariation(img, imageInfo, chanInfo, airmass=1.5, pwv=-1, removeSlope=True):
"""
This function is called by findContinuum.
Computes the absolute and percentage variation in atmospheric transmission
and sky temperature across an image cube.
Returns 5 values: max(Trans)-min(Trans), and as percentage of mean,
Max(Tsky)-min(Tsky), and as percentage of mean,
standard devation of Tsky
"""
freqs, values = CalcAtmTransmissionForImage(img, imageInfo, chanInfo, airmass=airmass, pwv=pwv, value='transmission')
if removeSlope:
slope, intercept = linfit(freqs, values, values*0.001)
casalogPost("Computed atmospheric variation and determined slope: %f per GHz (%.0f,%.2f)" % (slope,freqs[0],values[0]))
values = values - (freqs*slope + intercept) + np.mean(values)
maxMinusMin = np.max(values)-np.min(values)
percentage = maxMinusMin/np.mean(values)
freqs, values = CalcAtmTransmissionForImage(img, imageInfo, chanInfo, airmass=airmass, pwv=pwv, value='tsky')
if removeSlope:
slope, intercept = linfit(freqs, values, values*0.001)
values = values - (freqs*slope + intercept) + np.mean(values)
TmaxMinusMin = np.max(values)-np.min(values)
Tpercentage = TmaxMinusMin*100/np.mean(values)
stdValues = np.std(values)
return(maxMinusMin, percentage, TmaxMinusMin, Tpercentage, stdValues)
[docs]def versionStringToArray(versionString):
"""
This function is called by casaVersionCompare.
Converts '5.3.0-22' to np.array([5,3,0,22], dtype=np.int32)
"""
tokens = versionString.split('-')
t = tokens[0].split('.')
version = [np.int32(i) for i in t]
if len(tokens) > 1:
version += [np.int32(tokens[1])]
return np.array(version)
[docs]def casaVersionCompare(comparitor, versionString):
"""
This function is called by meanSpectrum.
Uses cu.compare_version in CASA >=5, otherwise uses string comparison
No longer works in CASA 6.
"""
if casaMajorVersion < 5:
if comparitor == '>=':
comparison = casadef.casa_version >= versionString
elif comparitor == '>':
comparison = casadef.casa_version > versionString
elif comparitor == '<':
comparison = casadef.casa_version < versionString
elif comparitor == '<=':
comparison = casadef.casa_version <= versionString
else:
print("Unknown comparitor: ", comparitor)
return False
else:
version = versionStringToArray(versionString)
comparison = cu.compare_version(comparitor, version)
return comparison
[docs]def getFreqType(img):
"""
This function is called by runFindContinuum and cubeLSRKToTopo.
"""
myia = iatool()
myia.open(img)
mycs = myia.coordsys()
mytype = mycs.referencecode('spectral')[0]
mycs.done()
myia.close()
return mytype
[docs]def getEquinox(img, myia=None):
"""
This function is called by cubeLSRKToTopo.
"""
if myia is None:
myia = iatool()
myia.open(img)
needToClose = True
else:
needToClose = False
mycs = myia.coordsys()
equinox = mycs.referencecode('direction')[0]
mycs.done()
if needToClose: myia.close()
return equinox
[docs]def getTelescope(img, myia=None):
"""
This function is called by CalcAtmTransmissionForImage and cubeLSRKToTopo.
"""
if myia is None:
myia = iatool()
myia.open(img)
needToClose = True
else:
needToClose = False
mycs = myia.coordsys()
telescope = mycs.telescope()
# telescope = myia.miscinfo()['TELESCOP'] # not in images produced in 4.2.2
mycs.done()
if needToClose: myia.close()
return telescope
[docs]def getDateObs(img, myia=None):
"""
This function is called by cubeLSRKToTopo.
Returns string of format: '2014/05/22/08:47:05' suitable for lsrkToTopo
"""
if myia is None:
myia = iatool()
myia.open(img)
needToClose = True
else:
needToClose = False
mycs = myia.coordsys()
mjd = mycs.epoch()['m0']['value']
mycs.done()
if needToClose: myia.close()
mydate = mjdToUT(mjd).rstrip(' UT').replace('/','-').replace(' ','/')
return mydate
[docs]def removeInitialQuadraticIfNeeded(avgSpectrum, initialQuadraticImprovementThreshold=1.6):
"""
This function is called by runFindContinuum when meanSpectrumMethod='mom0mom8jointMask'.
Fits a quadratic to the specified spectrum, and removes it if the
MAD will improve by more than a factor of threshold
"""
index = list(range(len(avgSpectrum)))
priorMad = MAD(avgSpectrum)
casalogPost("preMad: %f" % (priorMad), debug=True)
fitResult = polyfit(index, avgSpectrum, priorMad)
order2, slope, intercept, xoffset = fitResult
myx = np.arange(len(avgSpectrum)) - xoffset
trialSpectrum = avgSpectrum + nanmean(avgSpectrum)-(myx**2*order2 + myx*slope + intercept)
postMad = MAD(trialSpectrum)
# print("postMad=%e, trialSpectrum has %d non-zero values: " % (postMad,len(np.where(trialSpectrum != 0.0))), trialSpectrum)
if postMad == 0.0:
initialQuadraticRemoved = False
improvementRatio = 1.0
return avgSpectrum, initialQuadraticRemoved, improvementRatio
casalogPost("preMad: %f, postMad: %f, factorReduction: %f" % (priorMad,postMad,priorMad/postMad), debug=True)
improvementRatio = priorMad/postMad
if improvementRatio > initialQuadraticImprovementThreshold:
initialQuadraticRemoved = True
avgSpectrum = trialSpectrum
casalogPost("Initial quadratic removed because improvement ratio: %f > %f" % (improvementRatio,5), debug=True)
else:
casalogPost("Initial quadratic not removed because improvement ratio: %f <= %f" % (improvementRatio,5), debug=True)
initialQuadraticRemoved = False
return avgSpectrum, initialQuadraticRemoved, improvementRatio
[docs]def pick_sFC_TDM(meanSpectrumMethod, singleContinuum):
"""
This function is called by runFindContinuum.
Chooses the value of sigmaFindContinuum for TDM datasets based on
the meanSpectrumMethod and whether the user requested single continuum in
the Observing Tool.
"""
if singleContinuum:
sFC_TDM = 9.0
elif (meanSpectrumMethod.find('meanAboveThreshold') >= 0):
sFC_TDM = 4.5
elif (meanSpectrumMethod.find('peakOverMAD') >= 0):
sFC_TDM = 6.5
elif (meanSpectrumMethod == 'mom0mom8jointMask'):
sFC_TDM = 7.2 # 2018-03-23 was 4.5, 2018-03-26 was 5.5, 2018-03-27 was 6.5, 2018-03-31 was 7.0
else:
sFC_TDM = 4.5
casalogPost("Unknown meanSpectrumMethod: %s" % (meanSpectrumMethod))
return sFC_TDM
[docs]def setYLimitsAvoidingEdgeChannels(avgspectrumAboveThreshold, mad, chan=1):
"""
Called by runFindContinuum to set the y-axis limits..
1) Avoid spikes in edge channels from skewing the plot limits, by
setting plot limits on the basis of channels chan..-chan
2) Enforce a maximum dynamic range (peak-median)/mad so that weak features
can be seen, allowing strongest ones to overflow the top.
"""
y0,y1 = plt.ylim()
y0naturalBuffer = np.min(avgspectrumAboveThreshold) - y0
y1naturalBuffer = y1 - np.max(avgspectrumAboveThreshold)
mymin = np.min(avgspectrumAboveThreshold[chan:-chan]) - y0naturalBuffer
mymax = np.max(avgspectrumAboveThreshold[chan:-chan]) + y1naturalBuffer
mymedian = np.median(avgspectrumAboveThreshold)
highDynamicRange = False
dynamicRange = (mymax-mymedian)/mad
if dynamicRange > DYNAMIC_RANGE_LIMIT_PLOT:
casalogPost('Spectral dynamic range exceeds %d, limiting maximum y-axis value' % (DYNAMIC_RANGE_LIMIT_PLOT))
mymax = DYNAMIC_RANGE_LIMIT_PLOT*mad + mymedian
highDynamicRange = True
plt.ylim([mymin, mymax])
return highDynamicRange
[docs]def ExpandYLimitsForLegend():
"""
Called by runFindContinuum.
Make room for legend text at bottom and top of existing plot.
"""
ylim = plt.ylim()
yrange = ylim[1]-ylim[0]
ylim = [ylim[0]-yrange*0.05, ylim[1]+yrange*0.2]
plt.ylim(ylim)
return plt.ylim()
[docs]def pickRandomErrors(nvalues=1):
"""
Picks a series of random values from a Gaussian distribution with mean 0 and standard
deviation = 1 and returns it as an array.
-Todd Hunter
"""
p = []
for i in range(nvalues):
p.append(pickRandomError())
return(np.array(p))
[docs]def pickRandomError(seed=None):
"""
Picks a random value from a Gaussian distribution with mean 0 and standard deviation = 1.
seed: if specified, then first reseed with random.seed(seed)
-Todd Hunter
"""
w = 1.0
if seed is not None:
random.seed(seed)
while ( w >= 1.0 ):
x1 = 2.0 * random.random() - 1.0
x2 = 2.0 * random.random() - 1.0
w = x1 * x1 + x2 * x2
w = np.sqrt( (-2.0 * np.log( w ) ) / w )
y1 = x1 * w
y2 = x2 * w
return(y1)
[docs]def createNoisyQuadratic(n=1000, nsigma=3):
"""
Function meant simply to test removeStatContQuadratic.
n: number of points in synthetic noise spectrum
nsigma: passed to removeStatContQuadratic
"""
x = np.arange(n)
y = pickRandomErrors(n)
x0 = (x - n/2)
y += 0.001*x0**2
plt.clf()
plt.plot(x, y, 'k-')
y1, idx, yfit = removeStatContQuadratic(y, nsigma, returnExtra=True)
plt.plot(x, yfit, 'r-')
plt.plot(x, y1, 'b-')
plt.draw()
[docs]def removeStatContQuadratic(y, nsigma=2.0, returnExtra=False, minPercentage=50, makeMovie=False, img='', keepContinuum=True):
"""
Iteratively removes outlier points down to nsigma, then fits a quadratic to the remaining
points, removes this quadratic from the full initial spectrum, and returns it, along
with the list of channels used for the fit and the fit evaluated at all channels.
minPercentage: stop removing outliers if number of remaining goes below this threshold
makeMovie: if True, then make a png for each quadratic fit as channels are removed one-by-one
img: only used to name the movie frame pngs
"""
y = np.array(y)
x = np.arange(len(y))
idx = np.array(x) # start with all points
yfit = np.median(y)
if makeMovie:
if img != '':
img = img + '_'
os.system('rm -f ' + img + '*')
ylim = None
diff = y[idx]-yfit
diff = diff - np.median(diff)
while np.max(np.abs(diff)) > nsigma*np.std(diff):
peak = np.argmax(np.abs(diff))
# print("Removing channel %d" % (peak))
idx = np.delete(idx, [peak])
if len(idx) < 3 or len(idx) < minPercentage*len(y)*0.01:
break
# fit only the remaining points
fitResult = polyfit(idx, y[idx], MAD(y[idx]))
order2, slope, intercept, xoffset = fitResult
yfit = (x[idx]-xoffset)**2*order2 + (x[idx]-xoffset)*slope + intercept
diff0 = y[idx] - yfit
diff = diff0 - np.median(diff0)
diffWithoutFit = y[idx] - np.median(y[idx])
if np.max(yfit) - np.min(yfit) > 0:
ratio = (np.max(y[idx]) - np.min(y[idx])) / (np.max(yfit) - np.min(yfit))
else:
ratio = 10
peakOverMAD = (np.max(y[idx]) - np.min(y[idx])) / MAD(y[idx])
ratio = peakOverMAD
if ratio > 5: # 3*(np.max(yfit) - np.min(yfit)) < np.max(y[idx]) - np.min(y[idx]):
# strong lines are still present, so revert to using the median
fitUsed = False
diff = diffWithoutFit
else:
fitUsed = True
if makeMovie:
plt.clf()
plt.subplot(211)
plt.plot(x, y, 'k-', x[idx], yfit, 'r-', x[idx], y[idx], 'b-')
plt.plot([x[0], x[-1]], [np.median(y), np.median(y)], 'k--')
plt.title('%d points (ratio=%.2f)' % (len(idx), ratio))
if ylim is None:
ylim = plt.ylim()
else:
plt.ylim(ylim)
plt.subplot(212)
plt.plot(x[idx], diff, 'k-')
plt.plot([x[0], x[-1]], [np.median(diff), np.median(diff)], 'k--')
if fitUsed:
plt.title('quadratic and median removed')
else:
plt.title('median removed')
png = img + 'fit%04d.png' % (len(y)-len(idx))
plt.savefig(png)
if len(idx) < 3:
casalogPost('Too few points (%d) to fit a quadratic' % (len(idx)))
idx = x
yfit = np.zeroes(len(x))
else:
# remove the fit from all points
yfit = (x-xoffset)**2*order2 + (x-xoffset)*slope + intercept
y = y - yfit
if keepContinuum:
y += np.mean(yfit)
if returnExtra:
return y, idx, yfit
else:
return y
[docs]def runFindContinuum(img='', pbcube=None, psfcube=None, minbeamfrac=0.3,
spw='', transition='',
baselineModeA='min', baselineModeB='min',
sigmaCube=3, nBaselineChannels=0.19,
sigmaFindContinuum='auto', sigmaFindContinuumMode='auto',
verbose=False, png='', pngBasename=False,
nanBufferChannels=2,
source='', useAbsoluteValue=True, trimChannels='auto',
percentile=20, continuumThreshold=None, narrow='auto',
separator=';', overwrite=False, titleText='',
maxTrim=maxTrimDefault, maxTrimFraction=1.0,
meanSpectrumFile='', centralArcsec=-1, channelWidth=0,
alternateDirectory='.', imageInfo=[], chanInfo=[],
plotAtmosphere='transmission', airmass=1.5, pwv=1.0,
channelFractionForSlopeRemoval=0.75, mask='',
invert=False, meanSpectrumMethod='peakOverMad',
peakFilterFWHM=15, fullLegend=False, iteration=0,
meanSpectrumMethodMessage='', minSlopeToRemove=1e-8,
minGroupsForSFCAdjustment=10,
regressionTest=False, quadraticFit=False, megapixels=0,
triangularPatternSeen=False, maxMemory=-1,
negativeThresholdFactor=1.15, byteLimit=-1,
singleContinuum=False, applyMaskToMask=False,
plotBaselinePoints=False, dropBaselineChannels=2.0,
madRatioUpperLimit=1.5, madRatioLowerLimit=1.15,
projectCode='', useIAGetProfile=True,
useThresholdWithMask=False, dpi=dpiDefault,
normalizeByMAD=False, overwriteMoments=False,
initialQuadraticImprovementThreshold=1.6,
minPeakOverMadForSFCAdjustment=19,
maxMadRatioForSFCAdjustment=1.20,
minPixelsInJointMask=3, userJointMask='',
signalRatioTier1=0.965, signalRatioTier2=0.94,
snrThreshold=23, mom0minsnr=MOM0MINSNR_DEFAULT,
mom8minsnr=MOM8MINSNR_DEFAULT,
overwriteMasks=True, rmStatContQuadratic=True,
quadraticNsigma=2, bidirectionalMaskPhase2=False,
plotQuadraticPoints=True, makeMovie=True, outdir='',
allowBluePruning=True, avoidance='',
enableRejectNarrowInnerWindows=True,
avoidExtremaInNoiseCalcForJointMask=False,
amendMask=False, momentdir='', skipchan=1,
amendMaskIterationName='', fontsize=10):
"""
This function is called by findContinuum. It calls functions that:
1) compute the mean spectrum of a dirty cube
2) find the continuum channels
3) plot the results
Inputs: channelWidth: in Hz
Returns: 23 items
*1 A channel selection string suitable for the spw parameter of clean.
*2 The name of the png produced.
*3 The slope of the linear fit (or None if no fit attempted).
*4 The channel width in Hz (only necessary for the fitsTable option).
*5 nchan in the cube (only necessary to return this for the fitsTable option, will be computed otherwise).
*6 Boolean: True if it used low values as the baseline (as opposed to high values)
*7 SNRs in the moment0 image (raw, outside mask, outside phase2 mask)
*8 SNRs in the moment8 image (raw, outside mask, outside phase2 mask)
*9 Boolean: True if the middle-valued channels were used as the baseline (as opposed to low or high)
*10 list: selectionPreBluePruning
*11 float: finalSigmaFindContinuum
*12 string: name of jointMask
*13 float array: avgspectrumAboveThreshold
*14 float: the true median of the spectrum
*15 list: of plot artist descriptors (things to potentially remove and replot)
*16 axis instance: ax1 (lower x-axis)
*17 axis instance: ax2 (upper x-axis)
*18 float: threshold (the positive threshold for line detection)
*19 string: final line of upper text legend
*20 int: number of narrow central groups that were dropped (or would have been dropped
if enableRejectNarrowWindows had been True, but wasn't), summed over all runs
of findContinuumChannels()
*21 float: effectiveSigma (product of finalSigmaFindContinuum*correctionFactor)
*22 float: baselineMAD
*23 string: upper x-axis label
Inputs:
img: the image cube to operate upon
spw: the spw name or number to put in the x-axis label
transition: the name of the spectral transition (for the plot title)
baselineModeA: 'min' or 'edge', method to define the baseline in meanSpectrum()
sigmaCube: multiply this value by the MAD to get the threshold above which a pixel
is included in the mean spectrum
nBaselineChannels: if integer, then the number of channels to use in:
a) computing the mean spectrum with the 'meanAboveThreshold' methods.
b) computing the MAD of the lowest/highest channels in findContinuumChannels
if float, then the fraction of channels to use (i.e. the percentile)
default = 0.19, which is 24 channels (i.e. 12 on each side) of a TDM window
sigmaFindContinuum: passed to findContinuumChannels, 'auto' starts with 3.5
sigmaFindContinuumMode: 'auto', 'autolower', or 'fixed'
verbose: if True, then print additional information during processing
png: the name of the png to produce ('' yields default name)
pngBasename: if True, then remove the directory from img name before generating png name
nanBufferChannels: when removing or replacing NaNs, do this many extra channels
beyond their extent
source: the name of the source, to be shown in the title of the spectrum.
if None, then use the filename, up to the first underscore.
findContinuum: if True, then find the continuum channels before plotting
overwrite: if True, or ASCII file does not exist, then recalculate the mean spectrum
writing it to <img>.meanSpectrum
if False, then read the mean spectrum from the existing ASCII file
trimChannels: after doing best job of finding continuum, remove this many
channels from each edge of each block of channels found (for margin of safety)
If it is a float between 0..1, then trim this fraction of channels in each
group (rounding up). If it is 'auto', use 0.1 but not more than maxTrim channels
and not more than maxTrimFraction
percentile: control parameter used with baselineMode='min'
continuumThreshold: if specified, only use pixels above this intensity level
narrow: the minimum number of channels that a group of channels must have to survive
if 0<narrow<1, then it is interpreted as the fraction of all
channels within identified blocks
if 'auto', then use int(ceil(log10(nchan)))
titleText: default is img name and transition and the control parameter values
meanSpectrumFile: an alternative ASCII text file to use for the mean spectrum.
Must also set img=''.
meanSpectrumMethod: 'peakOverMad', 'peakOverRms', 'meanAboveThreshold',
'meanAboveThresholdOverRms', 'meanAboveThresholdOverMad',
where 'peak' refers to the peak in a spatially-smoothed version of cube
peakFilterFWHM: value used by 'peakOverRms' and 'peakOverMad' to presmooth
each plane with a Gaussian kernel of this width (in pixels)
Set to 1 to not do any smoothing.
centralArcsec: radius of central box within which to compute the mean spectrum
default='auto' means start with whole field, then reduce to 1/10 if only
one window is found (unless mask is specified); or 'fwhm' for the central square
with the same radius as the PB FWHM; or a floating point value in arcseconds
mask: a mask image equal in shape to the parent image that is used to determine the
region to compute the noise (outside the mask) and the mean spectrum (inside the mask)
option 'auto' will look for the <filename>.mask that matches the <filename>.image
and if it finds it, it will use it; otherwise it will use no mask
plotAtmosphere: '', 'tsky', or 'transmission'
airmass: for plot of atmospheric transmission
pwv: in mm (for plot of atmospheric transmission)
channelFractionForSlopeRemoval: if at least this many channels are initially selected, or
if there are only 1-2 ranges found and the widest range has > nchan*0.3 channels, then
fit and remove a linear slope and re-identify continuum channels. Set to 1 to turn off.
quadraticFit: if True, fit quadratic polynomial to the noise regions when appropriate;
otherwise fit only a linear slope
megapixels: simply used to label the plot
triangularPatternSeen: if True, then slightly boost sigmaFindContinuum if it is in 'auto' mode
maxMemory: only used to name the png if it is set
applyMaskToMask: if True, apply the mask inside the user mask image to set its masked pixels to 0
plotBaselinePoints: if True, then plot the baseline-defining points as black dots
dropBaselineChannels: percentage of extreme values to drop in baseline mode 'min'
useIAGetProfile: if True, then for meanAboveThreshold and baselineMode='min', then
use ia.getprofile instead of ia.getregion and subsequent arithmetic (faster)
initialQuadraticImprovementThreshold: if removal of a quadratic from the raw meanSpectrum reduces
the MAD by this factor or more, then proceed with removing this quadratic (new Cycle 6 heuristic)
maxMadRatioForSFCAdjustment: madRatio must be below this value (among other things) in order
for automatic lowering of sigmaFindContinuum value to occur
avoidance: a CASA channel selection string to avoid selecting (for manual use)
Parameters only relevant to mom0mom8jointMask:
mom0minsnr: sets the threshold for the mom0 image (i.e. median + mom0minsnr*scaledMAD)
mom8minsnr: sets the threshold for the mom8 image (i.e. median + mom8minsnr*scaledMAD)
snrThreshold: if SNR is higher than this, and phase2==True, then run a phase 2 mask calculation
minPixelsInJointMask: if fewer than these pixels are found, then use all pixels above pb=0.3
(when meanSpectrumMethod = 'mom0mom8jointMask')
overwriteMoments: if True, then overwrite any existing moment0 or moment8 image
overwriteMasks: if True, then overwrite any existing moment0 or moment8 mask image
rmStatContQuadratic: if True, then fit&remove a quadratic to subset of channels that lack signal
quadraticNsigma: the stopping threshold to use when ignoring outliers prior to fitting quadratic
bidirectionalMaskPhase2: True=extend mask to negative values beyond threshold; False=Cycle6 behavior
makeMovie: if True, then make a png for each quadratic fit as channels are removed one-by-one
outdir: where to write the .mom0 and .mom8 images, .dat, meanSpectrum, and the .png file
Parameters passed to findContinuumChannels (and only used in there):
baselineModeB: 'min' or 'edge', method to define the baseline in findContinuumChannels()
separator: the character to use to separate groups of channels in the string returned
maxTrim: in trimChannels='auto', this is the max channels to trim per group for TDM spws; it is automatically scaled upward for FDM spws.
maxTrimFraction: in trimChannels='auto', the max fraction of channels to trim per group
negativeThresholdFactor: scale the nominal negative threshold by this factor (to adjust
sensitivity to absorption features: smaller values=more sensitive)
signalRatioTier1: threshold for signalRatio, above which we desensitize the level to
consider line emission in order to avoid small differences in noise levels from
affecting the result (e.g. that occur between serial and parallel tclean cubes)
signalRatio=1 means: no lines seen, while closer to zero: more lines seen
signalRatioTier2: second threshold for signalRatio, used for FDM spws (nchan>192) and
for cases of peakOverMad < 5. Should be < signalRatioTier1.
dropBaselineChannels: percentage of extreme values to drop in baseline mode 'min'
madRatioUpperLimit, madRatioLowerLimit: determines when dropBaselineChannels is used
"""
normalized = False # This will be set True only by return value from meanSpectrumFromMom0Mom8JointMask()
startTime = timeUtilities.time()
slope=None
replaceNans = True # This used to be a command-line option, but no longer.
img = img.rstrip('/')
fitsTable = False
typeOfMeanSpectrum = 'Existing'
narrowValueModified = None
RangesDropped = 0
mom0snrs = None
mom8snrs = None
if img == '':
maxBaseline = 0 # added March 22, 2019
if minbeamfrac == 'auto':
minbeamfrac = 0.3
casalogPost("minbeamfrac = 'auto': using %.2f since no image was passed" % (minbeamfrac))
else:
maxBaseline = imageInfo[11]
if minbeamfrac == 'auto':
if maxBaseline < 60:
minbeamfrac = 0.1
casalogPost("minbeamfrac = 'auto': using %.2f since maxBaseline < 60m" % (minbeamfrac))
else:
minbeamfrac = 0.3
casalogPost("minbeamfrac = 'auto': using %.2f since maxBaseline >= 60m" % (minbeamfrac))
if (meanSpectrumFile != '' and os.path.exists(meanSpectrumFile)):
casalogPost("Using existing meanSpectrumFile = %s" % (meanSpectrumFile))
if (is_binary(meanSpectrumFile)):
fitsTable = True
if ((type(nBaselineChannels) == float or type(nBaselineChannels) == np.float64) and not fitsTable):
# chanInfo will be == [] if an ASCII meanSpectrumFile is specified
if len(chanInfo) >= 4:
nchan, firstFreq, lastFreq, channelWidth = chanInfo
channelWidth = abs(channelWidth)
nBaselineChannels = int(round_half_up(nBaselineChannels*nchan))
casalogPost("Found %d channels in the cube" % (nchan))
if (nBaselineChannels < 2 and not fitsTable and len(chanInfo) >= 4):
casalogPost("You must have at least 2 edge channels (nBaselineChannels = %d)" % (nBaselineChannels))
return
if (meanSpectrumFile == ''):
meanSpectrumFile = buildMeanSpectrumFilename(img, meanSpectrumMethod, peakFilterFWHM, amendMaskIterationName)
elif (not os.path.exists(meanSpectrumFile)):
if (len(os.path.dirname(img)) > 0):
meanSpectrumFile = os.path.join(os.path.dirname(img),os.path.basename(meanSpectrumFile))
if not overwrite and amendMaskIterationName not in ['.extraMask','.onlyExtraMask']:
# do not allow overwrite to be changed from False to True if we are in the extraMaskStage
overwrite = checkForMismatch(meanSpectrumFile, img, mask,
useThresholdWithMask, fitsTable,
iteration, centralArcsec)
initialQuadraticRemoved = False
pbBasedMask = False
numberPixelsInMom8Mask = 0 # added March 22, 2019
jointMask = None # will only become populated if meanSpectrumMethod == 'mom0mom8jointMask'
if (((overwrite or not os.path.exists(meanSpectrumFile)) and img != '') and not fitsTable):
typeOfMeanSpectrum = 'Computed'
if meanSpectrumMethod == 'mom0mom8jointMask':
# There should be no Nans that were replaced, but keep this name
# for the spectrum for consistency with the older methods.
if (overwrite):
casalogPost("Regenerating the mean spectrum file with method='%s' (momentdir='%s', numberPixelsInMom8Mask=%s)." % (meanSpectrumMethod,momentdir,numberPixelsInMom8Mask))
else:
casalogPost("Generating the mean spectrum file with method='%s'." % (meanSpectrumMethod))
avgSpectrumNansReplaced, normalized, numberPixelsInJointMask, pbBasedMask, initialQuadraticRemoved, initialQuadraticImprovementRatio, mom0snrs, mom8snrs, regionsPruned, numberPixelsInMom8Mask, mom0peak, mom8peak, jointMask = meanSpectrumFromMom0Mom8JointMask(img, imageInfo, nchan, pbcube, psfcube, minbeamfrac, projectCode, overwriteMoments, overwriteMasks, normalizeByMAD=normalizeByMAD, minPixelsInJointMask=minPixelsInJointMask, userJointMask=userJointMask, snrThreshold=snrThreshold, mom0minsnr=mom0minsnr, mom8minsnr=mom8minsnr, rmStatContQuadratic=rmStatContQuadratic, bidirectionalMaskPhase2=bidirectionalMaskPhase2, outdir=outdir, avoidExtremaInNoiseCalcForJointMask=avoidExtremaInNoiseCalcForJointMask, momentdir=momentdir)
print("returned from meanSpectrumFromMom0Mom8JointMask: numberPixelsInMom8Mask = ", numberPixelsInMom8Mask)
nanmin = None
edgesUsed = None
meanSpectrumThreshold = None
else:
if (overwrite):
casalogPost("Regenerating the mean spectrum file with centralArcsec=%s, mask='%s'." % (str(centralArcsec),mask))
else:
casalogPost("Generating the mean spectrum file with centralArcsec=%s, mask='%s'." % (str(centralArcsec),mask))
result = meanSpectrum(img, nBaselineChannels, sigmaCube, verbose,
nanBufferChannels,useAbsoluteValue,
baselineModeA, percentile,
continuumThreshold, meanSpectrumFile,
centralArcsec, imageInfo, chanInfo, mask,
meanSpectrumMethod, peakFilterFWHM, iteration,
applyMaskToMask, useIAGetProfile, useThresholdWithMask, overwrite)
if result is None:
return
avgspectrum, avgSpectrumNansRemoved, avgSpectrumNansReplaced, meanSpectrumThreshold,\
edgesUsed, nchan, nanmin, percentagePixelsNotMasked = result
if verbose:
print("len(avgspectrum) = %d, len(avgSpectrumNansReplaced)=%d" % (len(avgspectrum),len(avgSpectrumNansReplaced)))
else:
# Here is where nchan is defined for case of FITS table or previous spectrum
if (fitsTable):
result = readMeanSpectrumFITSFile(meanSpectrumFile)
if (result is None):
casalogPost("FITS table is not valid.")
return
avgspectrum, avgSpectrumNansReplaced, meanSpectrumThreshold, edgesUsed, nchan, nanmin, firstFreq, lastFreq = result
percentagePixelsNotMasked = -1
else:
# An ASCII file was specified as the spectrum to process
casalogPost("Running readPreviousMeanSpectrum('%s')" % (meanSpectrumFile))
result = readPreviousMeanSpectrum(meanSpectrumFile, verbose, invert)
if (result is None):
casalogPost("ASCII file is not valid, re-run with overwrite=True")
return
avgspectrum, avgSpectrumNansReplaced, meanSpectrumThreshold, edgesUsed, nchan, nanmin, firstFreq, lastFreq, previousCentralArcsec, previousMask, percentagePixelsNotMasked = result
if meanSpectrumMethod == 'mom0mom8jointMask':
numberPixelsInJointMask = edgesUsed
# Note: previousCentralArcsec and previousMask are not used yet
regionsPruned = 0 # we don't know if pruning was done to generate this spectrum
chanInfo = [result[4], result[6], result[7], abs(result[7]-result[6])/(result[4]-1)]
if verbose:
print("len(avgspectrum) = ", len(avgspectrum))
if (len(avgspectrum) < 2):
casalogPost("ASCII file is too short, re-run with overwrite=True")
return
if (firstFreq == 0 and lastFreq == 0):
# This was an old-format ASCII file, without a frequency column
n, firstFreq, lastFreq, channelWidth = chanInfo
channelWidth = abs(channelWidth)
if (fitsTable or img==''):
if ((type(nBaselineChannels) == float or type(nBaselineChannels) == np.float64) and not fitsTable):
print("nBaselineChannels=%d, nchan=%d" % (nBaselineChannels, nchan))
nBaselineChannels = int(round_half_up(nBaselineChannels*nchan))
n, firstFreq, lastFreq, channelWidth = chanInfo # freqs are in Hz
channelWidth = abs(channelWidth)
print("Setting channelWidth to %g" % (channelWidth))
if (nBaselineChannels < 2):
casalogPost("You must have at least 2 edge channels")
return
# By this point, nchan is guaranteed to be defined, in case it is needed.
if narrow == 'auto':
narrow = pickNarrow(nchan) # May 6, 2020 for ALMAGAL manual usage of narrow
minPeakOverMadForSFCAdjustment = 19
donetime = timeUtilities.time()
casalogPost("%.1f sec elapsed in meanSpectrum()" % (donetime-startTime))
casalogPost("Iteration %d (sigmaFindContinuumMode='%s')" % (iteration,sigmaFindContinuumMode))
sFC_TDM = pick_sFC_TDM(meanSpectrumMethod, singleContinuum)
# This definition of peakOverMad will be high if there is strong continuum
# or if there is a strong line. However, removing the median from the peak
# would eliminate the sensitivity to continuum emission, which we found to
# be a bad idea.
peakOverMad = np.max(avgSpectrumNansReplaced) / MAD(avgSpectrumNansReplaced)
# But for purposes of deciding when to not trigger amend mask (i.e. strong
# continuum) then we do want to discriminate, and removing the median is
# good for that. Here we ignore the edge channels to calculate the peak.
peakMinusMedianOverMad = (np.max(avgSpectrumNansReplaced[1:-1]) - np.median(avgSpectrumNansReplaced)) / MAD(avgSpectrumNansReplaced)
if (sigmaFindContinuum in ['auto','autolower',-1]):
# Choose the starting value for sigmaFindContinuum if automatic mode is selected.
# Here we could consider adding a dependency on whether numberPixelsInMom8Mask > 0
# or not, but let's first try only adjusting this before the second run. 06 Jan 2019.
if (tdmSpectrum(channelWidth, nchan)):
sigmaFindContinuum = sFC_TDM
casalogPost("Setting sigmaFindContinuum = %.1f since it is TDM" % (sFC_TDM))
elif (meanSpectrumMethod.find('meanAboveThreshold') >= 0):
sigmaFindContinuum = 3.5
casalogPost("Setting sigmaFindContinuum = %.1f since we are using meanAboveThreshold" % (sigmaFindContinuum))
elif (meanSpectrumMethod.find('peakOverMAD') >= 0):
sigmaFindContinuum = 6.0
casalogPost("Setting sigmaFindContinuum = %.1f since we are using peakOverMAD" % (sigmaFindContinuum))
elif (meanSpectrumMethod == 'mom0mom8jointMask'):
# Here is some fine tuning based on the type of spw, which we may
# regret in the future, but it gives better practical results for ALMA.
# Will need to revamp if storage of subregions of an spw are ever
# implemented in the correlator.
if nchan < 750:
# i.e. TDM spw or lots of channel averaging
if peakOverMad > 6:
sigmaFindContinuum = 4.2 # was 3.5 on March29, 2018
else:
sigmaFindContinuum = 4.5 # was 3.5 on March29, 2018
else:
# i.e. FDM spw with no (or little) channel averaging
if peakOverMad > 6:
sigmaFindContinuum = 2.5 # was 2.6 in v4.90; was 3.0 on Apr2, was 3.5 on Mar29
else:
sigmaFindContinuum = 3.2 # was 3.0 on Apr2, was 3.5 on Mar29
casalogPost("Setting sigmaFindContinuum = %.1f since we are using mom0mom8jointMask" % (sigmaFindContinuum))
else:
sigmaFindContinuum = 3.0
print("Unknown method")
if triangularPatternSeen and sigmaFindContinuumMode == 'auto':
# this cannot happen with mom0mom8jointMask because we don't check for it
# (because it doesn't happen in that method of constructing a mean spectrum)
sigmaFindContinuum += 0.5
casalogPost("Adding 0.5 to sigmaFindContinuum due to triangularPattern seen")
fCCiteration = 0
if (meanSpectrumMethod == 'mom0mom8jointMask' and rmStatContQuadratic):
# We remove the quadratic from the stored mean spectrum, that way the stored spectrum still
# contains the curved baseline, which may be useful for future reference.
avgSpectrumNansReplaced, channelsFit, yfit = removeStatContQuadratic(avgSpectrumNansReplaced, quadraticNsigma, returnExtra=True, makeMovie=makeMovie, img=img)
casalogPost("Removed quadratic computed from %d/%d channels" % (len(channelsFit),len(avgSpectrumNansReplaced)))
print("Calling findContinuumChannels with nBaselineChannels = %d" % (nBaselineChannels))
result = findContinuumChannels(avgSpectrumNansReplaced, nBaselineChannels,
sigmaFindContinuum, nanmin, baselineModeB,
trimChannels, narrow, verbose, maxTrim,
maxTrimFraction, separator, peakOverMad,
negativeThresholdFactor=negativeThresholdFactor,
dropBaselineChannels=dropBaselineChannels,
madRatioUpperLimit=madRatioUpperLimit,
madRatioLowerLimit=madRatioLowerLimit,
projectCode=projectCode, fCCiteration=fCCiteration,
signalRatioTier1=signalRatioTier1, signalRatioTier2=signalRatioTier2,
sigmaFindContinuumMode=sigmaFindContinuumMode,
enableRejectNarrowInnerWindows=enableRejectNarrowInnerWindows)
continuumChannels,selection,threshold,median,groups,correctionFactor,medianTrue,mad,medianCorrectionFactor,negativeThreshold,lineStrengthFactor,singleChannelPeaksAboveSFC,allGroupsAboveSFC,spectralDiff, trimChannels, useLowBaseline, narrowValueModified, allBaselineChannelsXY, madRatio, useMiddleChannels, signalRatio, rangesDropped = result
RangesDropped += rangesDropped
if verbose: print("aboveBelow run0")
sumAboveMedian, sumBelowMedian, sumRatio, channelsAboveMedian, channelsBelowMedian, channelRatio = \
aboveBelow(avgSpectrumNansReplaced,medianTrue)
spwBandwidth = nchan*channelWidth # in Hz
c_mks = 2.99792458e8
spwBandwidthKms = 0.001 * c_mks * spwBandwidth/np.mean([firstFreq,lastFreq])
sFC_factor = 1.0
newMaxTrim = 0
sFC_adjusted = False
# Due to the following code block, rejectNarrowInnerWindowsChannels() can (by reducing groups from >2 to 2)
# can lead to a different value of maxTrim for FDM datasets.
if (groups <= 2 and not tdmSpectrum(channelWidth, nchan) and
maxTrim==maxTrimDefault and trimChannels=='auto'):
# CAS-8822
newMaxTrim = maxTrimDefault*nchan/128
casalogPost("Changing maxTrim from %s to %s for this FDM spw because trimChannels='%s' and groups=%d." % (str(maxTrim),str(newMaxTrim),trimChannels,groups))
maxTrim = newMaxTrim
# Adjust sigmaFindContinuum if necessary, and re-run findContinuumChannels()
if (singleChannelPeaksAboveSFC==allGroupsAboveSFC and allGroupsAboveSFC>1):
# If all the channels exceeding the threshold are single spikes and there are more
# than one of them, then they may be noise spikes, so raise the threshold a bit
# This 'if' block CAN sometimes be used by meanSpectrumMethod='mom0mom8jointMask'.
if (sigmaFindContinuum < sFC_TDM):
if sigmaFindContinuumMode == 'autolower':
casalogPost("Because we are in autolower, not scaling the threshold upward by a factor of %.2f to avoid apparent noise spikes (%d==%d)." % (sFC_factor, singleChannelPeaksAboveSFC,allGroupsAboveSFC))
elif sigmaFindContinuumMode == 'auto':
# raise the threshold a bit since all the peaks look like all noise spikes
sFC_factor = 1.5
sigmaFindContinuum *= sFC_factor
casalogPost("Scaling the threshold upward by a factor of %.2f to avoid apparent noise spikes (%d==%d)." % (sFC_factor, singleChannelPeaksAboveSFC,allGroupsAboveSFC))
fCCiteration += 1
result = findContinuumChannels(avgSpectrumNansReplaced,
nBaselineChannels, sigmaFindContinuum, nanmin,
baselineModeB, trimChannels, narrow, verbose,
maxTrim, maxTrimFraction, separator, peakOverMad,
negativeThresholdFactor=negativeThresholdFactor,
dropBaselineChannels=dropBaselineChannels,
madRatioUpperLimit=madRatioUpperLimit,
madRatioLowerLimit=madRatioLowerLimit,
projectCode=projectCode, fCCiteration=fCCiteration,
signalRatioTier1=signalRatioTier1, signalRatioTier2=signalRatioTier2,sigmaFindContinuumMode=sigmaFindContinuumMode,
enableRejectNarrowInnerWindows=enableRejectNarrowInnerWindows)
continuumChannels,selection,threshold,median,groups,correctionFactor,medianTrue,mad,medianCorrectionFactor,negativeThreshold,lineStrengthFactor,singleChannelPeaksAboveSFC,allGroupsAboveSFC,spectralDiff,trimChannels,useLowBaseline, narrowValueModified, allBaselineChannelsXY, madRatio, useMiddleChannels, signalRatio, rangesDropped = result
RangesDropped += rangesDropped
if verbose: print("aboveBelow run1")
sumAboveMedian, sumBelowMedian, sumRatio, channelsAboveMedian, channelsBelowMedian, channelRatio = \
aboveBelow(avgSpectrumNansReplaced,medianTrue)
elif ((groups > 3 or (groups > 1 and channelRatio < 1.0) or (channelRatio < 0.5) or (groups == 2 and channelRatio < 1.3)) and sigmaFindContinuumMode in ['auto','autolower'] and meanSpectrumMethod.find('peakOver') < 0 and meanSpectrumMethod.find('mom0mom8jointMask') < 0 and not singleContinuum):
# If there are a lot of groups, or a lot of flux in channels above the median
# compared to channels below the median (i.e. the channelRatio), then lower the
# sigma in order to push the threshold for real lines (or line emission wings) lower.
# However, if there is only 1 group, then there may be no real lines
# present, so lowering the threshold in this case can create needless
# extra groups, so don't allow it.
# **** This 'elif' is NOT used by meanSpectrumMethod='mom0mom8jointMask'. *****
print("A: groups,channelRatio=", groups, channelRatio, channelRatio < 1.0, channelRatio>0.1, tdmSpectrum(channelWidth,nchan), groups>2)
if (channelRatio < 0.9 and channelRatio > 0.1 and (firstFreq>60e9 and not tdmSpectrum(channelWidth,nchan)) and groups>2): # was nchan>256
# Don't allow this much reduction in ALMA TDM mode as it chops up
# line-free quasar spectra too much. The channelRatio>0.1
# requirement prevents failures due to ALMA TFB platforming.
sFC_factor = 0.333
elif (groups <= 2):
if (0.1 < channelRatio < 1.3 and groups == 2 and
not tdmSpectrum(channelWidth,nchan) and channelWidth>=1875e6/600.):
if (channelWidth < 1875e6/360.):
if madRatioLowerLimit < madRatio < madRatioUpperLimit:
sFC_factor = 0.60
else:
sFC_factor = 0.50
# i.e. for galaxy spectra with FDM 480 channel (online-averaging) resolution
# but 0.5 is too low for uid___A001_X879_X47a.s24_0.ELS26_sci.spw25.mfs.I.findcont.residual
# and for uid___A001_X2d8_X2c5.s24_0.2276_444_53712_sci.spw16.mfs.I.findcont.residual
# the latter requires >=0.057
# need to reconcile in future versions
else:
sFC_factor = 0.7 # i.e. for galaxy spectra with FDM 240 channel (online-averaging) resolution
else:
# prevent sigmaFindContinuum going to inf if groups==1
# prevent sigmaFindContinuum going > 1 if groups==2
sFC_factor = 0.9
else:
if tdmSpectrum(channelWidth,nchan):
sFC_factor = 1.0
elif channelWidth>0: # the second factor tempers the reduction as the spw bandwidth decreases
sFC_factor = (np.log(3)/np.log(groups)) ** (spwBandwidth/1.875e9)
else:
sFC_factor = (np.log(3)/np.log(groups))
casalogPost("setting factor to %f because groups=%d, channelRatio=%g, firstFreq=%g, nchan=%d, channelWidth=%e" % (sFC_factor,groups,channelRatio,firstFreq,nchan,channelWidth))
print("---------------------")
casalogPost("Scaling the threshold by a factor of %.2f (groups=%d, channelRatio=%f)" % (sFC_factor, groups,channelRatio))
print("---------------------")
sigmaFindContinuum *= sFC_factor
fCCiteration += 1
result = findContinuumChannels(avgSpectrumNansReplaced,
nBaselineChannels, sigmaFindContinuum, nanmin,
baselineModeB, trimChannels, narrow, verbose, maxTrim,
maxTrimFraction, separator, peakOverMad,
negativeThresholdFactor=negativeThresholdFactor,
dropBaselineChannels=dropBaselineChannels,
madRatioUpperLimit=madRatioUpperLimit,
madRatioLowerLimit=madRatioLowerLimit,
projectCode=projectCode, fCCiteration=fCCiteration,
signalRatioTier1=signalRatioTier1, signalRatioTier2=signalRatioTier2,
sigmaFindContinuumMode=sigmaFindContinuumMode,
enableRejectNarrowInnerWindows=enableRejectNarrowInnerWindows)
continuumChannels,selection,threshold,median,groups,correctionFactor,medianTrue,mad,medianCorrectionFactor,negativeThreshold,lineStrengthFactor,singleChannelPeaksAboveSFC,allGroupsAboveSFC,spectralDiff,trimChannels,useLowBaseline, narrowValueModified, allBaselineChannelsXY, madRatio, useMiddleChannels, signalRatio, rangesDropped = result
RangesDropped += rangesDropped
if verbose: print("aboveBelow run2")
sumAboveMedian, sumBelowMedian, sumRatio, channelsAboveMedian, channelsBelowMedian, channelRatio = \
aboveBelow(avgSpectrumNansReplaced,medianTrue)
else:
if (meanSpectrumMethod.find('peakOver') < 0 and
meanSpectrumMethod.find('mom0mom8jointMask') < 0):
casalogPost("Not adjusting sigmaFindContinuum, because groups=%d, channelRatio=%g, firstFreq=%g, nchan=%d" % (groups,channelRatio,firstFreq,nchan),debug=True)
else:
# We are using either peakOverMad or mom0mom8jointMask
if madRatio is None:
madRatioCheck = True # this was python2 behavior for inequality
else:
madRatioCheck = madRatio < maxMadRatioForSFCAdjustment
casalogPost("numberPixelsInMom8Mask=%d, allContinuumSelected('%s',%d)=%s" % (numberPixelsInMom8Mask,selection,nchan,allContinuumSelected(selection,nchan)))
if ((groups >= minGroupsForSFCAdjustment and not tdmSpectrum(channelWidth,nchan)
or (groups >= 3 and tdmSpectrum(channelWidth,nchan))) and
sigmaFindContinuumMode in ['auto','autolower'] and # added Mar 27, 2019
# (peakOverMad>minPeakOverMadForSFCAdjustment or
((peakOverMad>minPeakOverMadForSFCAdjustment and madRatioCheck)
or meanSpectrumMethod.find('peakOver') >= 0)):
# ***** This 'if' block will never be used when groups==1 *****
#18 set by 312:2016.1.01400.S spw25 not needing it with 11.7 and
# 431:E2E5.1.00036.S spw24 not needing it with 17.44
# 268:2015.1.00190.S spw16 not needing it with 18.7
# 423:E2E5.1.00034.S spw25 needing it with 20.3
# but 262:2015.1.01068.S spw37 does not need it with 38
# The following heuristics for sFC achieve better results on hot cores
# when meanAboveThreshold cannot be used.
maxGroups = 25 # was 40
if groups < maxGroups:
# protect against negative number raised to a power
sFC_factor = np.max([5.0/7.0, (1-groups/float(maxGroups)) ** (spwBandwidth/1.875e9)])
else:
sFC_factor = 5.0/7.0 # was previously 2.5/sigmaFindContinuum
sigmaFindContinuum *= sFC_factor
# madRatio could be 'None' so set to string
casalogPost("%s Adjusting sigmaFindContinuum by %.2f to %f because groups=%d>=%d and not TDM and meanSpectrumMethod = %s and peakOverMad=%f>%g and madRatio=%s<%f" % (projectCode, sFC_factor,sigmaFindContinuum, groups, minGroupsForSFCAdjustment, meanSpectrumMethod, peakOverMad, minPeakOverMadForSFCAdjustment,str(madRatio),maxMadRatioForSFCAdjustment), debug=True)
sFC_adjusted = True # need to set this to force examination of blue points below
elif (numberPixelsInMom8Mask >= 4) and allContinuumSelected(selection,nchan):
normalizedMom0peak = mom0peak/spwBandwidthKms
# before I added tdm/fdm split, it was 1.4 on Jan 26 regression; 1.5 on Feb 01 regression
# page 260 needs <= 1.41
if tdmSpectrum(channelWidth, nchan):
normalizedMom0factor = 1.5
else:
normalizedMom0factor = 1.4 # was 1.4 on Jan 26 regression; 1.5 on Feb 01 regression
if mom8peak > normalizedMom0factor*normalizedMom0peak:
# New for Cycle 7: Here we adjust sFC downward because emission was found
# in the mom8 image but no line regions were found initially. - 06 Jan 2019
sFC_factor = SFC_FACTOR_WHEN_MOM8MASK_EXISTS
sigmaFindContinuum *= sFC_factor
if mom8peak > 3.5*normalizedMom0peak and peakOverMad > 5.5:
# needed for helms61 spw16 in 2018.1.00922.S
sFC_adjusted = True # need to set this to force examination of blue points below
# but do not set it between 1.5-3.5 since it removes too much continuum in line-free cases
casalogPost("%s Adjusting sigmaFindContinuum by %.2f to %f because groups=1 and >=%.3f of the channels were selected and mom8peak=%f > %.2f*(mom0peak=%f/bwkms=%f)=%f" % (projectCode, sFC_factor, sigmaFindContinuum, ALL_CONTINUUM_CRITERION, mom8peak, normalizedMom0factor, mom0peak, spwBandwidthKms, normalizedMom0factor*normalizedMom0peak))
else:
casalogPost("%s Mom8 mask is present and allContinuumSelected, but mom8peak=%f not > %.2f*(mom0peak=%f/bwkms=%f)=%f" % (projectCode, mom8peak, normalizedMom0factor, mom0peak, spwBandwidthKms, normalizedMom0factor*normalizedMom0peak))
elif groups == 3 and tdmSpectrum(channelWidth, nchan):
# This is for the case of double-peaked spectrum where weaker detection in the channels between
# the peaks are below threshold.
sFC_adjusted = True # need to set this to force examination of blue points below
casalogPost('%s Not adjusting sigmaFindContinuum, but setting blue pruning because there are 3 groups and TDM' % (projectCode))
else:
# madRatio could be 'None' so force it to be a string
casalogPost("%s Not adjusting sigmaFindContinuum because groups=%d < %d or peakOverMad=%f<%.0f or madRatio=%s>=%f" % (projectCode, groups,minGroupsForSFCAdjustment,peakOverMad,minPeakOverMadForSFCAdjustment,str(madRatio),maxMadRatioForSFCAdjustment), debug=True)
if (newMaxTrim > 0 or
(allContinuumSelected(selection,nchan) and numberPixelsInMom8Mask > 0) or # line added 06 Jan 2019
(groups>minGroupsForSFCAdjustment and not tdmSpectrum(channelWidth,nchan))): # line added Aug 22, 2016
if (newMaxTrim > 0):
casalogPost("But re-running findContinuumChannels with new maxTrim")
fCCiteration += 1
result = findContinuumChannels(avgSpectrumNansReplaced,
nBaselineChannels, sigmaFindContinuum, nanmin,
baselineModeB, trimChannels, narrow, verbose, maxTrim,
maxTrimFraction, separator, peakOverMad,
negativeThresholdFactor=negativeThresholdFactor,
dropBaselineChannels=dropBaselineChannels,
madRatioUpperLimit=madRatioUpperLimit,
madRatioLowerLimit=madRatioLowerLimit,
projectCode=projectCode, fCCiteration=fCCiteration,
signalRatioTier1=signalRatioTier1, signalRatioTier2=signalRatioTier2,
sigmaFindContinuumMode=sigmaFindContinuumMode,
enableRejectNarrowInnerWindows=enableRejectNarrowInnerWindows)
continuumChannels,selection,threshold,median,groups,correctionFactor,medianTrue,mad,medianCorrectionFactor,negativeThreshold,lineStrengthFactor,singleChannelPeaksAboveSFC,allGroupsAboveSFC,spectralDiff,trimChannels, useLowBaseline, narrowValueModified, allBaselineChannelsXY, madRatio, useMiddleChannels, signalRatio, rangesDropped = result
RangesDropped += rangesDropped
if verbose: print("aboveBelow run3")
sumAboveMedian, sumBelowMedian, sumRatio, channelsAboveMedian, channelsBelowMedian, channelRatio = \
aboveBelow(avgSpectrumNansReplaced,medianTrue)
selectionPreBluePruning = selection
secondSelectionAdded = '' # used to indicate on the spectrum plot if the heuristic for CAS-11720 was activated to ensure a second region
if meanSpectrumMethod == 'mom0mom8jointMask':
casalogPost('No slope fit attempted because we are using mom0mom8jointMask.')
if ((sFC_adjusted or ((peakOverMad>6 or len(continuumChannels) > 4*nchan/5) and not tdmSpectrum(channelWidth,nchan))) and
len(continuumChannels) > 0 and allowBluePruning):
# New heuristic for Cycle 6: blue pruning (a.k.a. examining blue points)
# Definition: remove any candidate continuum range
# (horizontal cyan lines in the plot) that do not contain any
# initial baseline points (blue points in the plot), and trim
# those ranges that do contain baseline points to the maximal
# extent of the enclosed baseline points, eliminating them if
# they get too narrow.
casalogPost('Activating blue pruning heuristic')
channelSelections = selection.split(separator)
validSelections = []
newContinuumChannels = []
newGroups = 0
checkForGaps = True
# checkForGaps means to look for large portions of a channelSelection that has no
# baseline channels in it, and if found, then split up that selection into pieces.
# This is meant to eliminate the remaining weak lines in a hot core spectrum that
# are below the final threshold.
for i,ccstring in enumerate(channelSelections):
validSelection = False
cc = [int(j) for j in ccstring.split('~')]
selectionWidth = cc[1]+1 - cc[0]
# We set a max threshold mainly to prevent losing too much continuum if the "line" is not real
if tdmSpectrum(channelWidth,nchan):
gapMaxThreshold = int(nchan/4) # was 70
else:
gapMaxThreshold = int(nchan/3) # added for 478chan spw27 in 2018.1.00081.S
if groups == 1:
# Enable detection of single faint narrow line,
# e.g. 42 in project 54 uid___A001_X144_X7b.s21_0.IRD_C_sci.spw29
# 58 MHz 240 chan, 15 chan -> 3.6 km/s at 300 GHz
# 58 MHz 480 chan, 30 chan -> 3.6 km/s
# 58 MHz 960 chan, 60 chan -> 3.6 km/s
# 58 MHz 960 chan, 15 chan -> 3.6 km/s at 75 GHz
gapMinThreshold = int(30*(firstFreq/300e9)*(spwBandwidth/58e6)*(nchan/480.)) # previously was 40
casalogPost('initial gapMinThreshold=%d' % (gapMinThreshold))
if peakOverMad <= 6: # we got into here only because more than 4/5 channels were picked
if gapMinThreshold < 30:
gapMinThreshold = 30 # 30 was set to avoid gaps with no lines
elif gapMinThreshold > 45: #50:
gapMinThreshold = 45 #50 # 50 was set by E2E6.1.00030.S spw16 CO galaxy w/no continuum uid___A002_Xcff05c_X1bd.s28_0.HCG25b_sci.spw16.mfs.I.findcont.residual, but this line is 111 channels wide, so no longer appears to motivate this value
#49 was needed by 2018.1.00635.S spw29 CI1-0
else:
gapMinThreshold = np.max([16,gapMinThreshold]) # original heuristic, Note: can be a large number, even > gapMaxThreshold!
if (gapMinThreshold > gapMaxThreshold):
# this would fix two results on CAS-5290, including 2018.1.01100.S
# but has not been checked in regression
casalogPost('sFC was adjusted and gapMin=%d > gapMax=%d, so setting gapMin=30' % (gapMinThreshold,gapMaxThreshold))
gapMinThreshold = 30 # could also try 0.06*nchan (which is 28 for 478 chan)
if gapMinThreshold > gapMaxThreshold:
casalogPost(' but new gapMin=%d > gapMax=%d, so setting gapMin=%d' % (gapMinThreshold,gapMaxThreshold,gapMaxThreshold-10))
gapMinThreshold = gapMaxThreshold-10
else:
# Must be at least half the width of the group
gapMinThreshold = int(np.max([selectionWidth/2, narrow])) # splitgaps.pdf
if checkForGaps and verbose:
casalogPost("%s Looking for gaps in group %d/%d: %s that are %d < gap < %d" % (projectCode,i+1,len(channelSelections),ccstring,gapMinThreshold, gapMaxThreshold), debug=True)
theseChannels = list(range(cc[0], cc[1]+1))
startChan = -1
stopChan = -1
gaps = []
for chan in theseChannels:
if chan in allBaselineChannelsXY[0]:
validSelection = True # valid if any of the channels appeared in the original baseline
if startChan == -1:
startChan = chan
else:
# If the blue points are contiguous, then chan will now be stopChan+1, so check
# if it is actually a lot more than that (meaning we just crossed a gap). But,
# it must be a big gap relative to the selection width in order to qualify.
if ((stopChan > -1) and (chan-stopChan) > gapMinThreshold and (chan-stopChan) < gapMaxThreshold and checkForGaps):
gaps.append([stopChan+1,chan-1])
else:
if chan-stopChan >= gapMaxThreshold:
print("chan-stopChan = %d >= %d" % (chan-stopChan,gapMaxThreshold))
stopChan = chan
if (len(gaps) == 1 and tdmSpectrum(channelWidth,nchan)):
# check if the max is in here
if np.max(np.abs(avgSpectrumNansReplaced)) > np.max(np.abs(avgSpectrumNansReplaced[gaps[0][0]:gaps[0][1]])):
casalogPost('Removing the one gap found because the peak is not in here')
gaps = []
tooNarrow = False
if validSelection:
if (stopChan - startChan) < narrow-1:
tooNarrow = True
validSelection = False
if len(gaps) == 0:
casalogPost('%s Looking for trimmed channel ranges to drop'%(projectCode))
if stopChan == -1:
stopChan = cc[1]
ccstring = '%d~%d' % (startChan,stopChan)
if validSelection:
newGroups += 1
validSelections.append(ccstring)
newContinuumChannels += list(range(startChan,stopChan+1)) # theseChannels (bug fix May 14, 2020)
elif tooNarrow:
casalogPost('%s Dropping trimmed channel range %s because it is too narrow (<%d).' % (projectCode,ccstring,narrow),debug=True)
else:
ccstring = '%d~%d' % (cc[0],cc[1])
casalogPost('%s Dropping channel range %s because not enough baseline channels are contained and sFC was previously adjusted downwards.' % (projectCode,ccstring),debug=True)
else: # new heuristic on April 7, 2018
casalogPost('%s Splitting channel range %s into %d ranges due to wide gaps in baseline channels.' % (projectCode,ccstring,len(gaps)+1),debug=True)
for ngap,gap in enumerate(gaps):
if ngap == 0:
firstChan = startChan
else:
firstChan = gaps[ngap-1][1]+1
ccstring = '%d~%d' % (firstChan,gap[0]-1)
if gap[0]-firstChan >= narrow:
# The split piece of the range is wide enough to keep it.
newGroups += 1
validSelections.append(ccstring)
newContinuumChannels += list(range(firstChan, gap[0]))
casalogPost(' %s Defining range %s' % (projectCode,ccstring), debug=True)
else:
casalogPost(' %s Skipping range %s because it is too narrow (<%d)' % (projectCode,ccstring,narrow), debug=True)
# Process the final piece
ccstring = '%d~%d' % (gaps[len(gaps)-1][1]+1,stopChan)
if stopChan - gaps[len(gaps)-1][1] >= narrow:
newGroups += 1
validSelections.append(ccstring)
newContinuumChannels += list(range(gaps[len(gaps)-1][1]+1, stopChan+1))
casalogPost(' %s Defining range %s' % (projectCode,ccstring), debug=True)
else:
casalogPost(' %s Skipping range %s because it is too narrow (<%d)' % (projectCode,ccstring,narrow), debug=True)
# end for i,ccstring in enumerate(channelSelections)
if newGroups > 0:
selection = separator.join(validSelections)
groups = newGroups
continuumChannels = newContinuumChannels
else:
casalogPost('%s Restoring dropped channels because no groups are left.' % (projectCode),debug=True)
else:
if not allowBluePruning:
casalogPost("Not examining blue dots (at user request)")
else:
casalogPost("Not examining blue dots because: sFC_adjusted=%s is False or (peakOverMad=%.2f <= 6 and len(chans)=%d <= 4*nchan/5=%d) or it's TDM" % (sFC_adjusted,peakOverMad,len(continuumChannels), 4*nchan/5))
# check if only 1 narrow group found (to activate a fix for CAS-11720 == PIPE-183)
if groups == 1:
if selection == '':
print("This should not happen since PIPE-359 was fixed.")
if not amendMask:
return
channelsInSingleGroup = 0
group0 = [-1,-1]
else:
group0 = [int(j) for j in selection.split(',')[0].split('~')]
channelsInSingleGroup = np.abs(np.diff(group0)) + 1
minPercentInSingleGroup = 5.0
if float(channelsInSingleGroup)/nchan < minPercentInSingleGroup/100:
percentage = 100.*channelsInSingleGroup / nchan
casalogPost(' %s ****** There is only a single channel range found with only %.1f%% (< %.0f%%) of the channels' % (projectCode,percentage,minPercentInSingleGroup))
whichHalf = np.mean(group0) < nchan/2
mylist = findWidestContiguousListInOtherHalf(allBaselineChannelsXY[0], whichHalf, nchan)
if mylist is not None:
groups += 1
secondSelectionAdded = '%d~%d' % (mylist[0],mylist[-1])
if selection == '':
selection = secondSelectionAdded
else:
if mylist[0] > allBaselineChannelsXY[0][-1]:
# add to end of string (to maintain increasing order)
selection += separator + secondSelectionAdded
else:
# add to beginning of string (to maintain increasing order)
selection = secondSelectionAdded + separator + selection
casalogPost(' %s ****** Added a second selection in the other half of the spectrum: %s' % (projectCode,secondSelectionAdded))
else:
casalogPost(' %s ****** No blue points found in the other half of the spectrum' % (projectCode))
else:
# The following Cycle 4+5 logic (on checking for the presence of
# candidate continuum channels in various segments of the
# spectrum in order to decide whether to attempt to remove a first or
# second order baseline) is not used by the mom0mom8jointMask method.
selectedChannels = countChannels(selection)
largestGroup = channelsInLargestGroup(selection)
selections = len(selection.split(separator))
slopeRemoved = False
channelDistancesFromCenter = np.array(continuumChannels)-nchan/2
channelDistancesFromLowerThird = np.array(continuumChannels)-nchan*0.3
channelDistancesFromUpperThird = np.array(continuumChannels)-nchan*0.7
if (len(np.where(channelDistancesFromCenter > 0)[0]) > 0 and
len(np.where(channelDistancesFromCenter < 0)[0]) > 0):
channelsInBothHalves = True
else:
# casalogPost("channelDistancesFromCenter = %s" % (channelDistancesFromCenter))
channelsInBothHalves = False # does not get triggered by case 115, which could use it
if ((len(np.where(channelDistancesFromLowerThird < 0)[0]) > 0) and
(len(np.where(channelDistancesFromUpperThird > 0)[0]) > 0)):
channelsInBothEdgeThirds = True
else:
channelsInBothEdgeThirds = False
# Too many selected windows means there might be a lot of lines,
# so do not attempt a baseline fit.
maxSelections = 4 # 2 # July 27, 2017
# If you select too few channels, you cannot get a reliable fit
minBWFraction = 0.3
if ((selectedChannels > channelFractionForSlopeRemoval*nchan or
(selectedChannels > 0.4*nchan and selections==2 and channelFractionForSlopeRemoval<1) or
# fix for project 00956 spw 25 is to put lower bound of 1 < selections:
(largestGroup>nchan*minBWFraction and 1 < selections <= maxSelections and
channelFractionForSlopeRemoval<1))):
previousResult = continuumChannels,selection,threshold,median,groups,correctionFactor,medianTrue,mad,medianCorrectionFactor,negativeThreshold,lineStrengthFactor,singleChannelPeaksAboveSFC,allGroupsAboveSFC
# remove linear slope from mean spectrum and run it again
index = channelSelectionRangesToIndexArray(selection)
# if quadraticFit and (channelsInBothEdgeThirds or (channelsInBothHalves and selections==1)): # July 27, 2017
quadraticFitTest = quadraticFit and ((channelsInBothEdgeThirds and selections <= 2) or (channelsInBothHalves and selections==1))
if quadraticFitTest:
casalogPost("Fitting quadratic to %d channels (largestGroup=%d,nchan*%.1f=%.1f,selectedChannels=%d)" % (len(index), largestGroup, minBWFraction, nchan*minBWFraction, selectedChannels))
fitResult = polyfit(index, avgSpectrumNansReplaced[index], MAD(avgSpectrumNansReplaced[index]))
order2, slope, intercept, xoffset = fitResult
else:
casalogPost("Fitting slope to %d channels (largestGroup=%d,nchan*%.1f=%.1f,selectedChannels=%d)" % (len(index), largestGroup, minBWFraction, nchan*minBWFraction, selectedChannels))
fitResult = linfit(index, avgSpectrumNansReplaced[index], MAD(avgSpectrumNansReplaced[index]))
slope, intercept = fitResult
if (sFC_factor >= 1.5 and maxTrim==maxTrimDefault and trimChannels=='auto'):
# add these 'and' cases on July 20, 2016
# Do not restore if we have modified maxTrim. This prevents breaking up an FDM
# spectrum with no line detection into multiple smaller continuum windows.
sigmaFindContinuum /= sFC_factor
casalogPost("Restoring sigmaFindContinuum to %f" % (sigmaFindContinuum))
rerun = True
else:
rerun = False
if quadraticFitTest:
casalogPost("Removing quadratic = %g*(x-%f)**2 + %g*(x-%f) + %g" % (order2,xoffset,slope,xoffset,intercept))
myx = np.arange(len(avgSpectrumNansReplaced)) - xoffset
priorMad = MAD(avgSpectrumNansReplaced)
trialSpectrum = avgSpectrumNansReplaced + nanmean(avgSpectrumNansReplaced)-(myx**2*order2 + myx*slope + intercept)
postMad = MAD(trialSpectrum)
if postMad > priorMad:
casalogPost("MAD of spectrum got worse after quadratic removal: %f to %f. Switching to linear fit." % (priorMad, postMad), debug=True)
casalogPost("Fitting slope to %d channels (largestGroup=%d,nchan*%.1f=%.1f,selectedChannels=%d)" % (len(index), largestGroup, minBWFraction, nchan*minBWFraction, selectedChannels))
fitResult = linfit(index, avgSpectrumNansReplaced[index], MAD(avgSpectrumNansReplaced[index]))
slope, intercept = fitResult
if (abs(slope) > minSlopeToRemove):
casalogPost("Removing slope = %g" % (slope))
# Do not remove the offset in order to avoid putting spectrum near zero
avgSpectrumNansReplaced -= np.array(range(len(avgSpectrumNansReplaced)))*slope
slopeRemoved = True
rerun = True
else:
avgSpectrumNansReplaced = trialSpectrum
casalogPost("MAD of spectrum improved after quadratic removal: %f to %f" % (priorMad, postMad), debug=True)
slopeRemoved = True
rerun = True
if lineStrengthFactor < 1.2:
if sigmaFindContinuumMode == 'auto':
if tdmSpectrum(channelWidth,nchan) or sigmaFindContinuum < 3.5: # was 4
sigmaFindContinuum += 0.5
casalogPost("lineStrengthFactor = %f < 1.2 (increasing sigmaFindContinuum by 0.5 to %.1f)" % (lineStrengthFactor,sigmaFindContinuum))
else:
casalogPost("lineStrengthFactor = %f < 1.2 (but not increasing sigmaFindContinuum by 0.5 because it is TDM or already >=4)" % (lineStrengthFactor))
else:
casalogPost("lineStrengthFactor = %f < 1.2 (but not increasing sigmaFindContinuum by 0.5 because it is not 'auto')" % (lineStrengthFactor))
else:
casalogPost("lineStrengthFactor = %f >= 1.2" % (lineStrengthFactor))
else:
if (abs(slope) > minSlopeToRemove):
casalogPost("Removing slope = %g" % (slope))
avgSpectrumNansReplaced -= np.array(range(len(avgSpectrumNansReplaced)))*slope
slopeRemoved = True
rerun = True
# The following helped deliver more continuum bandwidth for HD_142527 spw3, but
# it harmed 2013.1.00518 13co (and probably others) by including wing emission.
# if trimChannels == 'auto': # prevent overzealous trimming July 20, 2016
# maxTrim = maxTrimDefault # prevent overzealous trimming July 20, 2016
discardSlopeResult = False
if rerun: # this is only relevant in Cycle 4 and 5; not used by mom0mom8jointMask
fCCiteration += 1
result = findContinuumChannels(avgSpectrumNansReplaced,
nBaselineChannels, sigmaFindContinuum, nanmin,
baselineModeB, trimChannels, narrow, verbose,
maxTrim, maxTrimFraction, separator, peakOverMad, fitResult,
negativeThresholdFactor=negativeThresholdFactor,
dropBaselineChannels=dropBaselineChannels,
madRatioUpperLimit=madRatioUpperLimit,
madRatioLowerLimit=madRatioLowerLimit,
projectCode=projectCode, fCCiteration=fCCiteration,
signalRatioTier1=signalRatioTier1,
signalRatioTier2=signalRatioTier2,
sigmaFindContinuumMode=sigmaFindContinuumMode,
enableRejectNarrowInnerWindows=enableRejectNarrowInnerWindows)
continuumChannels,selection,threshold,median,groups,correctionFactor,medianTrue,mad,medianCorrectionFactor,negativeThreshold,lineStrengthFactor,singleChannelPeaksAboveSFC,allGroupsAboveSFC,spectralDiff,trimChannels,useLowBaseline, narrowValueModified, allBaselineChannelsXY, madRatio, useMiddleChannels, signalRatio, rangesDropped = result
RangesDropped += rangesDropped
# If we had only one group and only added one or two more group after removing slope, and the
# smallest is small compared to the original group, then discard the new solution.
if (groups <= 3 and previousResult[4] == 1):
counts = countChannelsInRanges(selection)
if (float(min(counts))/max(counts) < 0.2):
casalogPost("*** Restoring result prior to linfit because %d/%d < 0.2***" % (min(counts),max(counts)))
discardSlopeResult = True
continuumChannels,selection,threshold,median,groups,correctionFactor,medianTrue,mad,medianCorrectionFactor,negativeThreshold,lineStrengthFactor,singleChannelPeaksAboveSFC,allGroupsAboveSFC = previousResult
else:
if channelFractionForSlopeRemoval < 1:
casalogPost("No slope fit attempted because selected channels (%d) < %.2f * nchan(%d) or other criteria not met" % (selectedChannels,channelFractionForSlopeRemoval,nchan))
casalogPost(" largestGroup=%d <= nchan*%.1f=%.1f or selections=%d > %d" % (largestGroup,minBWFraction,nchan*minBWFraction,selections,maxSelections))
else:
casalogPost("No slope fit attempted because channelFractionForSlopeRemoval >= 1")
idx = np.where(avgSpectrumNansReplaced < threshold)
madOfPointsBelowThreshold = MAD(avgSpectrumNansReplaced[idx])
if avoidance != '':
# Here we are relying on continuumChannels to match the selection string, so double-check to be sure
if selection != convertChannelListIntoSelection(continuumChannels):
casalogPost('THERE IS A MISMATCH BETWEEN selection AND continuumChannels!!!')
print("selection: ", selection)
print("continuumChannels: ", convertChannelListIntoSelection(continuumChannels))
avoidChannels = convertSelectionIntoChannelList(avoidance)
newContinuumChannels = sorted(list(set(continuumChannels) - set(avoidChannels)))
print("Removing %d channels from list due to avoidance regions" % (len(continuumChannels)-len(newContinuumChannels)))
continuumChannels = newContinuumChannels
selection = convertChannelListIntoSelection(continuumChannels)
groups = selection.count(';')+1
#########################################
# Plot the results (before returning)
#########################################
labelDescs = []
if amendMaskIterationName in ['.extraMask','.onlyExtraMask','.autoLower']:
plt.figure(2)
plt.clf()
if casaVersion >= '5.9' and plt.get_backend()== 'TkAgg':
fig = plt.gcf()
casalogPost('Scaling figsize by 1.4 due to TkAgg')
fig.set_size_inches(8.125*1.4, 6.12*1.4) # 150dpi makes this 1218x918
rows = 1
cols = 1
ax1 = plt.subplot(rows, cols, 1)
if replaceNans:
avgspectrumAboveThreshold = avgSpectrumNansReplaced
else:
avgspectrumAboveThreshold = avgSpectrumNansRemoved
# I have co-opted the edgesUsed field in the meanSpectrum text file to
# hold the numberOfPixelsInMask for the mom0mom8jointMask method, so its
# value will be unpredictable, so we test for the method name too.
if (edgesUsed == 2 or edgesUsed is None or
meanSpectrumMethod == 'mom0mom8jointMask'):
plt.plot(list(range(len(avgspectrumAboveThreshold))),
avgspectrumAboveThreshold, 'r-')
drawYlabel(img, typeOfMeanSpectrum, meanSpectrumMethod, meanSpectrumThreshold,
peakFilterFWHM, fontsize, mask, useThresholdWithMask, normalized)
elif (edgesUsed == 0):
# The upper edge is not used and can have an upward spike
# so don't show it.
casalogPost("Not showing final %d channels of %d" % (skipchan,\
len(avgspectrumAboveThreshold)))
plt.plot(list(range(len(avgspectrumAboveThreshold) - skipchan)),
avgspectrumAboveThreshold[:-skipchan], 'r-')
drawYlabel(img, typeOfMeanSpectrum,meanSpectrumMethod, meanSpectrumThreshold,
peakFilterFWHM, fontsize, mask, useThresholdWithMask)
elif (edgesUsed == 1):
# The lower edge channels are not used and the threshold mean can
# have an upward spike, so don't show the first channel inward from
# there.
casalogPost("Not showing first %d channels of %d" % (skipchan,\
len(avgspectrumAboveThreshold)))
plt.plot(list(range(skipchan, len(avgspectrumAboveThreshold))),
avgspectrumAboveThreshold[skipchan:], 'r-')
drawYlabel(img, typeOfMeanSpectrum,meanSpectrumMethod, meanSpectrumThreshold,
peakFilterFWHM, fontsize, mask, useThresholdWithMask)
highDynamicRange = setYLimitsAvoidingEdgeChannels(avgspectrumAboveThreshold, mad)
if (baselineModeA == 'edge'):
nEdgeChannels = nBaselineChannels/2
if (edgesUsed == 0 or edgesUsed == 2):
plt.plot(list(range(nEdgeChannels)), avgspectrumAboveThreshold[:nEdgeChannels], 'm-', lw=3)
if (edgesUsed == 1 or edgesUsed == 2):
plt.plot(list(range(nchan - nEdgeChannels, nchan)), avgspectrumAboveThreshold[-nEdgeChannels:], 'm-', lw=3)
if plotBaselinePoints:
plt.plot(allBaselineChannelsXY[0], allBaselineChannelsXY[1], 'b.', ms=3, mec='b')
if plotQuadraticPoints and rmStatContQuadratic:
plt.plot(channelsFit, avgspectrumAboveThreshold[channelsFit], 'g.', ms=3, mec='g')
# channelSelections = []
channelSelections = selection.split(separator)
casalogPost('Drawing positive threshold at %g' % (threshold))
plt.plot(plt.xlim(), [threshold, threshold], 'k:')
ylims = plt.ylim()
plt.ylim([ylims[0], np.max([ylims[1], threshold])])
if (negativeThreshold is not None):
plt.plot(plt.xlim(), [negativeThreshold, negativeThreshold], 'k:')
casalogPost('Drawing negative threshold at %g' % (negativeThreshold))
ylims = plt.ylim()
plt.ylim([np.min([ylims[0], negativeThreshold]), ylims[1]])
casalogPost('Drawing observed median as dashed line at %g' % (median))
plt.plot(plt.xlim(), [median, median], 'b--') # observed median (always lower than true for mode='min')
casalogPost('Drawing inferredMedian as solid line at %g' % (medianTrue))
plt.plot(plt.xlim(), [medianTrue, medianTrue], 'k-')
if (baselineModeB == 'edge'):
plt.plot([nEdgeChannels, nEdgeChannels], plt.ylim(), 'k:')
plt.plot([nchan - nEdgeChannels, nchan - nEdgeChannels], plt.ylim(), 'k:')
if (len(continuumChannels) > 0):
labelDescs = plotChannelSelections(ax1, selection, separator,
avgspectrumAboveThreshold, skipchan, medianTrue,
threshold, secondSelectionAdded)
if (fitsTable or img==''):
img = meanSpectrumFile
freqType = ''
else:
freqType = getFreqType(img)
if (source is None):
source = os.path.basename(img)
if (not fitsTable):
source = source.split('_')[0]
if (titleText == ''):
narrowString = pickNarrowString(narrow, len(avgSpectrumNansReplaced), narrowValueModified)
trimString = pickTrimString(len(avgSpectrumNansReplaced), trimChannels, len(avgSpectrumNansReplaced), maxTrim, selection)
if len(projectCode) > 0:
projectCode += ', '
titleText = projectCode + os.path.basename(img) + ' ' + transition
ylim = ExpandYLimitsForLegend()
xlim = [0,nchan-1]
plt.xlim(xlim)
titlesize = np.min([fontsize,int(np.floor(fontsize*100.0/len(titleText)))])
if tdmSpectrum(channelWidth,nchan):
dm = 'TDM'
else:
dm = 'FDM'
if (spw != ''):
label = '(Spw %s) %s Channels (%d)' % (str(spw), dm, nchan)
if singleContinuum:
label = 'SingleCont ' + label
else:
label = '%s Channels (%d)' % (dm,nchan)
if singleContinuum:
label = '(SingleContinuum) ' + label
ax1.set_xlabel(label, size=fontsize)
if nchan < 300:
ax1.xaxis.set_minor_locator(matplotlib.ticker.MultipleLocator(10))
elif nchan < 750:
ax1.xaxis.set_minor_locator(matplotlib.ticker.MultipleLocator(25))
elif nchan < 1500:
ax1.xaxis.set_minor_locator(matplotlib.ticker.MultipleLocator(50))
elif nchan < 3000:
ax1.xaxis.set_minor_locator(matplotlib.ticker.MultipleLocator(100))
else:
ax1.xaxis.set_minor_locator(matplotlib.ticker.MultipleLocator(200))
if casaVersion >= '5.9.9':
titleYoffset = 1.10
else:
titleYoffset = 1.08
plt.text(0.5, titleYoffset, titleText, size=titlesize, ha='center', transform=ax1.transAxes)
plt.ylim(ylim)
# The following line seems to have zero effect.
# ax1.yaxis.set_major_formatter(matplotlib.ticker.FormatStrFormatter('%.2e'))
ax2 = ax1.twiny()
plt.setp(ax1.get_xticklabels(), fontsize=fontsize)
plt.setp(ax1.get_yticklabels(), fontsize=fontsize)
plt.setp(ax2.get_xticklabels(), fontsize=fontsize)
ax2.set_xlim(firstFreq*1e-9,lastFreq*1e-9)
freqRange = np.abs(lastFreq-firstFreq)
power = int(np.log10(freqRange))-9
ax2.xaxis.set_major_locator(matplotlib.ticker.MultipleLocator(10**power))
if (len(ax2.get_xticks()) < 2):
ax2.xaxis.set_major_locator(matplotlib.ticker.MultipleLocator(0.5*10**power))
ax2.xaxis.set_minor_locator(matplotlib.ticker.MultipleLocator(0.1*10**power))
ax2.xaxis.set_major_formatter(matplotlib.ticker.ScalarFormatter(useOffset=False))
aggregateBandwidth = computeBandwidth(selection, channelWidth, 1)
if (channelWidth > 0):
channelWidthString = ', channel width: %g kHz, BW: %g MHz, contBW: %g MHz' % (channelWidth*1e-3, spwBandwidth*1e-6, aggregateBandwidth*1000)
else:
channelWidthString = ''
if (spw != ''):
upperXlabel = r'(Spw %s) $\bf{%s\/Frequency\/(GHz)}$' % (str(spw), freqType) + channelWidthString
else:
upperXlabel = r'$\bf{%s\/Frequency\/(GHz)}$' % freqType + channelWidthString
ax2.set_xlabel(upperXlabel, size=fontsize) # this artist cannot be removed in CASA 5.6
inc = 0.03
i = 1
if (meanSpectrumMethod == 'mom0mom8jointMask' and rmStatContQuadratic):
bottomLegend = 'rm quad: %d/%d, ' % (len(channelsFit), len(avgSpectrumNansReplaced))
elif initialQuadraticRemoved:
bottomLegend = "rm quad: mad improve: %.1f>%.1f, " % (initialQuadraticImprovementRatio,initialQuadraticImprovementThreshold)
else:
bottomLegend = ''
if madRatio is not None:
bottomLegend += "peak/MAD: %.2f, %.2f, signalRatio: %.3f, madRatio: %.3f" % (peakOverMad,peakMinusMedianOverMad,signalRatio,madRatio)
elif useMiddleChannels:
bottomLegend += "peak/MAD: %.2f, %.2f, signalRatio: %.3f, middle chans used" % (peakOverMad,peakMinusMedianOverMad,signalRatio)
else:
bottomLegend += "peak/MAD: %.2f, %.2f, signalRatio: %.3f, madRatio not computed" % (peakOverMad,peakMinusMedianOverMad,signalRatio)
if meanSpectrumMethod == 'mom0mom8jointMask':
bottomLegend += ', pixMom8=%d' % (numberPixelsInMom8Mask)
if bottomLegend != '':
plt.text(0.5, 0.01, bottomLegend, ha='center', size=fontsize,
transform=ax1.transAxes)
effectiveSigma = sigmaFindContinuum*correctionFactor
if meanSpectrumMethod.find('mean') >= 0:
plt.text(0.5, 0.99 - i * inc, 'bl=(%s,%s), narrow=%s, sCube=%.1f, sigmaEff=%.2f*%.2f=%.2f, trim=%s' % (baselineModeA, baselineModeB, narrowString, sigmaCube, sigmaFindContinuum, correctionFactor, effectiveSigma, trimString), transform=ax1.transAxes, ha='center', size=fontsize)
else:
plt.text(
0.5, 0.99-i*inc, ' baselineModeB=%s, narrow=%s, sigmaFC=%.2f*%.2f=%.2f, trim=%s, maxBase=%.0fm' %
(baselineModeB, narrowString, sigmaFindContinuum, correctionFactor, effectiveSigma, trimString, maxBaseline),
transform=ax1.transAxes, ha='center', size=fontsize-1)
i += 1
peak = np.max(avgSpectrumNansReplaced)
peakFeatureSigma = (peak-medianTrue)/mad
maxPeakFeatureSigma = (peak - np.min(avgSpectrumNansReplaced))/MAD(avgSpectrumNansReplaced)
# casalogPost("initial peakFeatureSigma = (%f-%f)/%f = %f, maxPeakFeatureSigma=%f" % (peak,medianTrue,mad,peakFeatureSigma,maxPeakFeatureSigma))
if (signalRatio > signalRatioTier1 or (signalRatio > signalRatioTier2 and peakOverMad<5)) and peakFeatureSigma > maxPeakFeatureSigma:
mad *= peakFeatureSigma/maxPeakFeatureSigma
casalogPost('Limiting peakFeatureSigma from %f to %f, reset MAD to %f' % (peakFeatureSigma,maxPeakFeatureSigma,mad))
peakFeatureSigma = maxPeakFeatureSigma
areaString = 'limited max: %.1f*mad=%.4f; %d ranges; ' % (peakFeatureSigma, peak, len(channelSelections))
else:
casalogPost("maxPeakFeatureSigma=%f" % (maxPeakFeatureSigma))
areaString = 'max: %.1f*mad=%.4f; %d ranges; ' % (peakFeatureSigma, peak, len(channelSelections))
if (fullLegend):
plt.text(0.5, 0.99 - i * inc, 'rms=MAD*1.4826: of baseline chans = %f, scaled by %.1f for all chans = %f' % (mad, correctionFactor, mad * correctionFactor),
transform=ax1.transAxes, ha='center', size=fontsize)
i += 1
plt.text(0.017, 0.99 - i * inc, 'lineStrength factor: %.2f' % (lineStrengthFactor), transform=ax1.transAxes, ha='left', size=fontsize)
plt.text(0.983, 0.99 - i * inc, 'MAD*1.4826: of points below upper dotted line = %f' % (madOfPointsBelowThreshold),
transform=ax1.transAxes, ha='right', size=fontsize)
i += 1
plt.text(0.5, 0.99 - i * inc, 'median: of %d baseline chans = %f, offset by %.1f*MAD for all chans = %f' % (nBaselineChannels, median, medianCorrectionFactor, medianTrue - median),
transform=ax1.transAxes, ha='center', size=fontsize)
i += 1
plt.text(0.5, 0.99 - i * inc, 'chans>median: %d (sum=%.4f), chans<median: %d (sum=%.4f), ratio: %.2f (%.2f)' % (channelsAboveMedian, sumAboveMedian, channelsBelowMedian, sumBelowMedian, channelRatio, sumRatio),
transform=ax1.transAxes, ha='center', size=fontsize-1)
if (negativeThreshold is not None):
plt.text(0.5, 0.99 - i * inc, 'mad: %.3g; levs: %.3g, %.3g (dot); median: %.3g (solid), medmin: %.3g (dash)' % (mad, threshold, negativeThreshold, medianTrue, median), transform=ax1.transAxes, ha='center', size=fontsize - 1)
else:
plt.text(0.5, 0.99 - i * inc, 'mad: %.3g; threshold: %.3g (dot); median: %.3g (solid), medmin: %.3g (dash)' % (mad, threshold, medianTrue, median),
transform=ax1.transAxes, ha='center', size=fontsize)
i += 1
if meanSpectrumMethod == 'mom0mom8jointMask':
if pbBasedMask:
areaString += 'pixels in pb-based mask: %g' % (numberPixelsInJointMask)
casalogPost('%s pixels in pb-based mask: %g' % (projectCode, numberPixelsInJointMask))
else:
if regionsPruned > 0:
# some regions were pruned
areaString += 'pixels in pruned mask: %g' % (numberPixelsInJointMask)
else:
areaString += 'pixels in joint mask: %g' % (numberPixelsInJointMask)
casalogPost('%s pixels in joint mask: %g' % (projectCode, numberPixelsInJointMask))
elif (centralArcsec == 'auto'):
areaString += 'mean over area: (unknown)'
elif (centralArcsec < 0):
areaString += 'mean over area: whole field (%.2fMpix)' % (megapixels)
else:
areaString += 'mean over: central box of radius %.1f" (%.2fMpix)' % (centralArcsec,megapixels)
labelDesc = plt.text(0.5, 0.99 - i * inc, areaString, transform=ax1.transAxes, ha='center', size=fontsize - 1)
labelDescs.append(labelDesc)
if (meanSpectrumMethodMessage != ''):
if casaVersion >= '5.9.9':
msmm_ylabel = -0.12
else:
msmm_ylabel = -0.10
plt.text(0.5, msmm_ylabel, meanSpectrumMethodMessage,
transform=ax1.transAxes, ha='center', size=fontsize)
finalLine = ''
if (len(mask) > 0):
if (percentagePixelsNotMasked > 0):
finalLine += 'mask=%s (%.2f%% pixels)' % (os.path.basename(mask), percentagePixelsNotMasked)
else:
finalLine += 'mask=%s' % (os.path.basename(mask))
i += 1
plt.text(0.5, 0.99 - i * inc, finalLine, transform=ax1.transAxes, ha='center', size=fontsize - 1)
finalLine = ''
if (slope is not None):
if discardSlopeResult:
discarded = ' (result discarded)'
elif slopeRemoved:
discarded = ' (removed)'
else:
discarded = ' (not removed)'
if quadraticFitTest:
finalLine += 'quadratic fit: %g*(x-%g)**2+%g*(x-%g)+%g %s' % (roundFigures(order2,3),roundFigures(xoffset,4),roundFigures(slope,3),roundFigures(xoffset,4),roundFigures(intercept,3),discarded)
else:
finalLine += 'linear slope: %g %s' % (roundFigures(slope,3),discarded)
i += 1
# This is the line to use to put the legend in the top group of lines.
# plt.text(0.5, 0.99-i*inc, finalLine, transform=ax1.transAxes, ha='center', size=fontsize)
plt.text(0.5, 0.04, finalLine, transform=ax1.transAxes,
ha='center', size=fontsize)
gigabytes = getMemorySize()/(1024.**3)
if not regressionTest:
# Write CVS version to plot legend
plt.text(1.06, -0.005 - 2 * inc, ' '.join(version().split()[1:4]), size=8,
transform=ax1.transAxes, ha='right')
# Write CASA version to plot legend
casaText = "CASA "+casaVersion+' (%.0f GB'%gigabytes
else:
casaText = '(%.0f GB' % gigabytes
byteLimit = byteLimit/(1024.**3)
if maxMemory < 0 or byteLimit == gigabytes:
casaText += ')'
else:
casaText += ', but limited to %.0fGB)' % (byteLimit)
plt.text(-0.03, -0.005 - 2 * inc, casaText, size=8, transform=ax1.transAxes, ha='left')
if (plotAtmosphere != '' and img != meanSpectrumFile):
if (plotAtmosphere == 'tsky'):
value = 'tsky'
else:
value = 'transmission'
freqs, atm = CalcAtmTransmissionForImage(img, imageInfo, chanInfo, airmass, pwv, value=value)
casalogPost("freqs: min=%g, max=%g n=%d" % (np.min(freqs),np.max(freqs), len(freqs)))
atmRange = 0.5 # how much of the y-axis should it take up
yrange = ylim[1]-ylim[0]
if (value == 'transmission'):
atmRescaled = ylim[0] + 0.3*yrange + atm*atmRange*yrange
print("atmRescaled: min=%f, max=%f" % (np.min(atmRescaled), np.max(atmRescaled)))
plt.text(1.015, 0.3, '0%', color='m', transform=ax1.transAxes, ha='left', va='center')
plt.text(1.015, 0.8, '100%', color='m', transform=ax1.transAxes, ha='left', va='center')
plt.text(1.03, 0.55, 'Atmospheric Transmission', color='m', ha='center', va='center',
rotation=90, transform=ax1.transAxes, size=11)
for yt in np.arange(0.3, 0.81, 0.1):
xtickTrans,ytickTrans = np.array([[1,1.01],[yt,yt]])
line = matplotlib.lines.Line2D(xtickTrans,ytickTrans,color='m',transform=ax2.transAxes)
ax1.add_line(line)
line.set_clip_on(False)
else:
atmRescaled = ylim[0] + 0.3*yrange + atm*atmRange*yrange/300.
plt.text(1.015, 0.3, '0', color='m', transform=ax1.transAxes, ha='left', va='center')
plt.text(1.015, 0.8, '300', color='m', transform=ax1.transAxes, ha='left', va='center')
plt.text(1.03, 0.55, 'Sky temperature (K)', color='m', ha='center', va='center',
rotation=90, transform=ax1.transAxes, size=11)
for yt in np.arange(0.3, 0.81, 0.1):
xtickTrans,ytickTrans = np.array([[1,1.01],[yt,yt]])
line = matplotlib.lines.Line2D(xtickTrans,ytickTrans,color='m',transform=ax2.transAxes)
ax1.add_line(line)
line.set_clip_on(False)
plt.text(1.06, 0.55, '(%.1f mm PWV, 1.5 airmass)' % pwv, color='m', ha='center', va='center',
rotation=90, transform=ax1.transAxes, size=11)
# plt.plot(freqs, atmRescaled, 'w.', ms=0) # need this to finalize the ylim value
ylim = plt.ylim()
yrange = ylim[1]-ylim[0]
if (value == 'transmission'):
atmRescaled = ylim[0] + 0.3*yrange + atm*atmRange*yrange
else:
atmRescaled = ylim[0] + 0.3*yrange + atm*atmRange*yrange/300.
plt.plot(freqs, atmRescaled, 'm-')
plt.ylim(ylim) # restore the prior value (needed for CASA 5.0)
plt.draw()
if (png == ''):
if pngBasename:
png = os.path.basename(img) + amendMaskIterationName
else:
png = img + amendMaskIterationName
if outdir != '':
png = os.path.join(outdir,os.path.basename(png))
transition = transition.replace('(','_').replace(')','_').replace(' ','_').replace(',','')
if (len(transition) > 0):
transition = '.' + transition
narrowString = pickNarrowString(narrow, len(avgSpectrumNansReplaced), narrowValueModified) # this is used later
trimString = pickTrimString(len(avgSpectrumNansReplaced), trimChannels, len(avgSpectrumNansReplaced), maxTrim, selection)
if userJointMask == '':
png += '.meanSpectrum.%s.%s.%s.%.2fsigma.narrow%s.trim%s%s' % (meanSpectrumMethod, baselineModeA, baselineModeB, sigmaFindContinuum, narrowString, trimString, transition)
elif userJointMask.find('.amendedJointMask') > 0:
png += '.meanSpectrum.amendedJointMask.%s.%s.%.2fsigma.narrow%s.trim%s%s' % (baselineModeA, baselineModeB, sigmaFindContinuum, narrowString, trimString, transition)
else:
png += '.meanSpectrum.userJointMask.%s.%s.%.2fsigma.narrow%s.trim%s%s' % (baselineModeA, baselineModeB, sigmaFindContinuum, narrowString, trimString, transition)
if maxMemory > 0:
png += '.%.0fGB' % (maxMemory)
if overwrite:
png += '.overwriteTrue.png'
else: # change the following to '.png' after my test of July 20, 2016
png += '.png'
pngdir = os.path.dirname(png)
if (len(pngdir) < 1):
pngdir = '.'
if (not os.access(pngdir, os.W_OK) and pngdir != '.'):
casalogPost("No permission to write to specified directory: %s. Will try alternateDirectory." % pngdir)
if (len(alternateDirectory) < 1):
alternateDirectory = '.'
png = alternateDirectory + '/' + os.path.basename(png)
pngdir = alternateDirectory
print("png = ", png)
if (not os.access(pngdir, os.W_OK)):
casalogPost("No permission to write to alternateDirectory. Will not save the plot.")
else:
plt.savefig(png, dpi=dpi)
casalogPost("Wrote png = %s" % (png))
donetime = timeUtilities.time()
casalogPost("%.1f sec elapsed in runFindContinuum" % (donetime-startTime))
baselineMAD = mad
if amendMaskIterationName in ['.extraMask','.onlyExtraMask','.autoLower']:
# go back to original figure now
plt.close(2)
plt.figure(1)
return(selection, png, slope, channelWidth, nchan, useLowBaseline, mom0snrs, mom8snrs, useMiddleChannels,
selectionPreBluePruning, sigmaFindContinuum, jointMask, avgspectrumAboveThreshold, medianTrue,
labelDescs, ax1, ax2, threshold, areaString, RangesDropped, effectiveSigma, baselineMAD, upperXlabel,
allBaselineChannelsXY)
# end of runFindContinuum
[docs]def invertChannelRanges(invertstring, nchan=0, startchan=0, vis='', spw='',
separator=';'):
"""
Takes a CASA channel selection string and inverts it.
-Todd Hunter
"""
myspw = spw
checkForMultipleSpw = invertstring.split(',')
mystring = ''
if (vis != ''):
mymsmd = createCasaTool(msmdtool)
mymsmd.open(vis)
for c in range(len(checkForMultipleSpw)):
checkspw = checkForMultipleSpw[c].split(':')
if (len(checkspw) > 1 and spw==''):
spw = checkspw[0]
checkForMultipleSpw[c] = checkspw[1]
if (vis != ''):
nchan = mymsmd.nchan(int(spw))
# print "Setting nchan = %d for spw %s" % (nchan,str(spw))
goodstring = checkForMultipleSpw[c]
goodranges = goodstring.split(separator)
if (str(spw) != ''):
mystring += str(spw) + ':'
if (int(goodranges[0].split('~')[0]) > startchan):
mystring += '%d~%d%s' % (startchan,int(goodranges[0].split('~')[0])-1, separator)
multiWindows = False
for g in range(len(goodranges)-1):
r = goodranges[g]
s = goodranges[g+1]
mystring += '%d~%d' % (int(r.split('~')[1])+1, int(s.split('~')[0])-1)
multiWindows = True
if (g < len(goodranges)-2):
mystring += separator
if (int(goodranges[-1].split('~')[-1]) < startchan+nchan-1):
if (multiWindows):
mystring += separator
mystring += '%d~%d' % (int(goodranges[-1].split('~')[-1])+1, startchan+nchan-1)
if (c < len(checkForMultipleSpw)-1):
mystring += ','
spw = myspw # reset to the initial value
if (vis != ''):
mymsmd.close()
return(mystring)
[docs]def plotChannelSelections(ax1, selection, separator, avgspectrumAboveThreshold,
skipchan, medianTrue,
threshold, secondSelectionAdded='', lineColor='c'):
"""
avgspectrumAboveThreshold: computed in runFindContinuum
medianTrue, threshold: computed by findContinuumChannels
Returns labelDescs: list of plot artists that may be removed and replaced later
"""
labelDescs = []
channelSelections = selection.split(separator)
# ylevel = np.mean(avgspectrumAboveThreshold) # Cycle 0-5 heuristic
peak = np.max(avgspectrumAboveThreshold[skipchan:-skipchan])
if (peak-medianTrue) > 1.5*(threshold-medianTrue):
ylevel = threshold + (threshold-medianTrue)*np.log10((peak-medianTrue)/(threshold-medianTrue))
print("Setting cyan level (%f) to just above the positive threshold dotted line" % (ylevel))
else:
ylevel = threshold - 0.5*(threshold-medianTrue)
print("Setting cyan level to half the positive threshold dotted line")
yoffset = ylevel + 0.04*(plt.ylim()[1] - plt.ylim()[0])
for i,ccstring in enumerate(channelSelections):
cc = [int(j) for j in ccstring.split('~')]
labelDesc = ax1.plot(cc, np.ones(len(cc))*ylevel, '-', color=lineColor, lw=2)
labelDescs.append(labelDesc)
mystring = ccstring
if i==1 and secondSelectionAdded != '':
# indicate if the heuristic for CAS-11720 was activated to ensure a second region
mystring += ' added'
labelDesc = ax1.text(np.mean(cc), yoffset, mystring, va='bottom', ha='center',size=8,rotation=90)
labelDescs.append(labelDesc)
return labelDescs
[docs]def findWidestContiguousListInOtherHalf(channels, lowerHalf, nchan):
"""
Given a list of channels, find the longest contiguous list in the first half or second half
example:
CASA <43>: fc.findWidestContiguousListInOtherHalf([2,3,7,8,9,12,13,14,15], True, 20)
Out[43]: [12, 13, 14, 15]
channels: list of blue points channel numbers
lowerHalf: a Boolean (True for lower half, False for upper half)
nchan: total channels in spectrum
"""
contiguousLists = splitListIntoContiguousLists(sorted(channels))
mylengths = []
widestList = None
widestListLength = 0
for j,contiguousList in enumerate(contiguousLists):
mylengths.append(0)
if (lowerHalf and contiguousList[0] >= nchan/2) or (not lowerHalf and contiguousList[0] < nchan/2):
mylengths[j] = len(contiguousList)
if mylengths[j] > widestListLength:
widestListLength = mylengths[j]
widestList = contiguousList
return widestList
[docs]def findWidestContiguousListInChannelRange(channels, channelRange, continuumChannels):
"""
Given a list of channels, find the longest contiguous list in the first half or second half
example:
CASA <43>: fc.findWidestContiguousListInOtherHalf([2,3,7,8,9,12,13,14,15], True, 20)
Out[43]: [12, 13, 14, 15]
channels: list of blue points channel numbers
channelRange: range of channels to look for widest set of blue points
continuumChannels: current selection of continuuum channels
"""
contiguousLists = splitListIntoContiguousLists(sorted(channels))
currentRange = continuumChannels[-1]-continuumChannels[0] + 1
mylengths = []
widestList = None
widestListLength = 0
startchan, endchan = channelRange
for j,contiguousList in enumerate(contiguousLists):
mylengths.append(0)
if (contiguousList[0] >= startchan) and (contiguousList[-1] < endchan):
spreadFactor = np.max([contiguousList[-1]-continuumChannels[0]+1,continuumChannels[-1]-contiguousList[0]+1])/currentRange
mylengths[j] = len(contiguousList) * spreadFactor
casalogPost('Possible region: %s, %d*%f = %f' % (contiguousList, len(contiguousList), spreadFactor, mylengths[j]))
if mylengths[j] > widestListLength:
widestListLength = mylengths[j]
widestList = contiguousList
return widestList
[docs]def aboveBelow(avgSpectrumNansReplaced, threshold):
"""
This function is called by runFindContinuum.
Given an array of values (i.e. a spectrum) and a threshold value, this
function computes and returns 6 items:
* the number of channels above that threshold (group 1)
* the number of channels below that threshold (group 2)
* the ratio of these numbers (i.e., #ChanAboveThreshold / #ChanBelowThreshold))
* sum of the intensities in group (1)
* sum of the intensities in group (2)
* the ratio of these sums (i.e., FluxAboveThreshold / FluxBelowThreshold)
(This value is not currently used by the calling function.)
Example with 24 channels where 3 have positive spikes, and rest are zero:
3 2 3 channelsAbove = 3
+threshold=+1 ........................ sumAbove = (3-1) + (2-1) + (3-1) = 5
zero --------------------------- channelsBelow = 21
-threshold=-1 ........................ sumBelow = 21*(1-0) = 21
"""
y = avgSpectrumNansReplaced
aboveMedian = np.where(avgSpectrumNansReplaced > threshold)
belowMedian = np.where(avgSpectrumNansReplaced < threshold)
sumAboveMedian = np.sum(-threshold+avgSpectrumNansReplaced[aboveMedian])
sumBelowMedian = np.sum(threshold-avgSpectrumNansReplaced[belowMedian])
channelsAboveMedian = len(aboveMedian[0])
channelsBelowMedian = len(belowMedian[0])
channelRatio = channelsAboveMedian*1.0/channelsBelowMedian
sumRatio = sumAboveMedian/sumBelowMedian # this value is unused by the calling function
return(sumAboveMedian, sumBelowMedian, sumRatio,
channelsAboveMedian, channelsBelowMedian, channelRatio)
[docs]def writeMeanSpectrum(meanSpectrumFile, frequency, avgspectrum,
avgSpectrumNansReplaced, threshold, nchan, edgesUsed=0,
nanmin=0, centralArcsec='auto', mask='', iteration=0):
"""
This function is called by meanSpectrum.
Writes out the mean spectrum (and the key parameters used to create it),
so that it can quickly be restored. This allows the manual user to quickly
experiment with different parameters of findContinuumChannels applied to
the same mean spectrum.
Units are Hz and Jy/beam.
meanSpectrumFile: name of file to create
threshold: this is interpreted as mom0threshold for meanSpectrumMethod='mom0mom8jointMask'
edgesUsed: this is interpreted as numberPixelsInMask for meanSpectrumMethod='mom0mom8jointMask'
nanmin: this is interpreted as mom8threshold for meanSpectrumMethod='mom0mom8jointMask'
frequency: list or array of frequencies
Returns: None
"""
if len(meanSpectrumFile) == 0:
print("WARNING: blank name requested for meanSpectrumFile!!")
return
f = open(meanSpectrumFile, 'w')
if centralArcsec == 'mom0mom8jointMask':
field1 = 'mom0threshold'
field2 = 'numberPixelsInMask'
field4 = 'mom8threshold'
else:
field1 = 'threshold'
field2 = 'edgesUsed'
field4 = 'nanmin'
if (iteration == 0):
f.write('#%s %s nchan %s centralArcsec=%s %s\n' % (field1,field2,field4,str(centralArcsec),mask))
else:
f.write('#threshold edgesUsed nchan nanmin centralArcsec=auto %s %s\n' % (str(centralArcsec),mask))
# field1 field2 field4
f.write('%.10f %g %g %.10f\n' % (threshold, edgesUsed, nchan, nanmin))
f.write('#chan freq(Hz) avgSpectrum avgSpectrumNansReplaced\n')
for i in range(len(avgspectrum)):
f.write('%d %.1f %.10f %.10f\n' % (i, frequency[i], avgspectrum[i], avgSpectrumNansReplaced[i]))
casalogPost('Wrote %s' % meanSpectrumFile, debug=True)
f.close()
[docs]def findContinuumChannels(spectrum, nBaselineChannels=16, sigmaFindContinuum=3,
nanmin=None, baselineMode='min', trimChannels='auto',
narrow='auto', verbose=False, maxTrim=maxTrimDefault,
maxTrimFraction=1.0, separator=';', peakOverMad=0, fitResult=None,
maxGroupsForMaxTrimAdjustment=3, lowHighBaselineThreshold=1.5,
lineSNRThreshold=20, negativeThresholdFactor=1.15,
dropBaselineChannels=2.0, madRatioUpperLimit=1.5,
madRatioLowerLimit=1.15, projectCode='', fCCiteration=0,
signalRatioTier1=0.965, signalRatioTier2=0.94, sigmaFindContinuumMode='auto',
enableRejectNarrowInnerWindows=True):
"""
This function is called by runFindContinuum.
Trys to find continuum channels in a spectrum, based on a threshold or
some number of edge channels and their median and standard deviation.
Inputs:
spectrum: a one-dimensional array of intensity values
nBaselineChannels: number of channels over which to compute standard deviation/MAD
sigmaFindContinuum: value to multiply the standard deviation by then add
to median to get threshold. Default = 3.
narrow: the minimum number of channels in a contiguous block to accept
if 0<narrow<1, then it is interpreted as the fraction of all
channels within identified blocks
nanmin: the value that NaNs were replaced by in previous steps
baselineMode: 'min' or 'edge', 'edge' will use nBaselineChannels/2 from each edge
trimChannels: if integer, use that number of channels. If float between
0 and 1, then use that fraction of channels in each contiguous list
(rounding up). If 'auto', then use 0.1 but not more than maxTrim channels.
The 'auto' mode can also cause trimChannels to be set to 13 or 6 in certain
circumstances (when moderately strong lines are present).
maxTrim: if trimChannels='auto', this is the max channels to trim per group for TDM spws; it is automatically scaled upward for FDM spws.
maxTrimFraction: in trimChannels='auto', the max fraction of channels to trim per group
separator: the character to use to separate groups of channels in the string returned
negativeThresholdFactor: scale the nominal negative threshold by this factor (to adjust
sensitivity to absorption features: smaller values=more sensitive)
dropBaselineChannels: percentage of extreme values to drop in baseline mode 'min'
madRatioUpperLimit, madRatioLowerLimit: if ratio of MADs is between these values, then
apply dropBaselineChannels when defining the MAD of the baseline range
fitResult: coefficients from linear or quadratic fit, only relevant when meanSpectrumMethod is
not 'mom0mom8jointMask'
signalRatioTier1: threshold for signalRatio, above which we desensitize the level to
consider line emission in order to avoid small differences in noise levels from
affecting the result (e.g. that occur between serial and parallel tclean cubes)
signalRatio=1 means: no lines seen, while closer to zero: more lines seen
signalRatioTier2: second threshold for signalRatio, used for FDM spws (nchan>192) and
for cases of peakOverMad < 5. Should be < signalRatioTier1.
sigmaFindContinuumMode: if 'auto', then do not desensitize with signalRatioTier1
Returns:
1 list of channels to use (separated by the specified separator)
2 list converted to ms channel selection syntax
3 positive threshold used
4 median of the baseline-defining channels
5 number of groups found
6 correctionFactor used
7 inferred true median
8 scaled MAD of the baseline-defining channels
9 correction factor applied to get the inferred true median
10 negative threshold used
11 lineStrength factor
12 singleChannelPeaksAboveSFC: how many cases where only a single channel exceeds the threshold
13 allGroupsAboveSFC
14 spectralDiff (percentage of median)
15 value of trimChannels parameter
16 Boolean describing whether the low values were used as the baseline
17 value of the narrow parameter
18 tuple containing the channel numbers of the baseline channels, and their respective y-axis values
19 value of madRatio: MAD/MAD_after_dropping_extreme_channels
This value will be larger if there are many tall single channel peaks.
20 number of ranges it dropped (or would have dropped if it was enabled)
"""
if (fitResult is not None):
myx = np.arange(len(spectrum), dtype=np.float64)
if (len(fitResult) > 2):
myx -= fitResult[3]
originalSpectrum = spectrum + (fitResult[0]*myx**2 + fitResult[1]*myx + fitResult[2]) - nanmean(spectrum)
casalog.post("min/max spectrum = %f, %f" % (np.min(spectrum), np.max(spectrum)))
casalog.post("min/max originalSpectrum = %f, %f" % (np.min(originalSpectrum), np.max(originalSpectrum)))
else:
originalSpectrum = spectrum + fitResult[0]*myx
else:
originalSpectrum = spectrum
if (narrow == 'auto'):
narrow = pickNarrow(len(spectrum))
autoNarrow = True
else:
autoNarrow = False
npts = len(spectrum)
percentile = 100.0*nBaselineChannels/npts
correctionFactor = sigmaCorrectionFactor(baselineMode, npts, percentile)
sigmaEffective = sigmaFindContinuum*correctionFactor
if (fitResult is not None):
if (len(fitResult) > 2):
casalogPost("****** starting findContinuumChannels (%d) (polynomial=%g*(x-%.2f)**2+%g*(x-%.2f)+%g) ***********" % (fCCiteration, fitResult[0], fitResult[3], fitResult[1], fitResult[3], fitResult[2]))
else:
casalogPost("****** starting findContinuumChannels (%d) (slope=%g) ***********" % (fCCiteration, fitResult[0]))
else:
casalogPost("****** starting findContinuumChannels (%d) with nBaselineChannels=%d ***********" % (fCCiteration,nBaselineChannels))
casalogPost("Using sigmaFindContinuum=%.2f, sigmaEffective=%.1f, percentile=%.0f for mode=%s, channels=%d/%d" % (sigmaFindContinuum, sigmaEffective, percentile, baselineMode, nBaselineChannels, len(spectrum)))
if (baselineMode == 'edge'):
# pick n channels on both edges
lowerChannels = spectrum[:nBaselineChannels/2]
upperChannels = spectrum[-nBaselineChannels/2:]
intensityAllBaselineChannels = list(lowerChannels) + list(upperChannels)
allBaselineXChannels = list(range(0, nBaselineChannels/2)) + \
list(range(len(spectrum) - nBaselineChannels/2, len(spectrum)))
if (np.std(lowerChannels) == 0):
mad = MAD(upperChannels)
median = nanmedian(upperChannels)
casalogPost("edge method: Dropping lower channels from median and std calculations")
elif (np.std(upperChannels) == 0):
mad = MAD(lowerChannels)
median = nanmedian(lowerChannels)
casalogPost("edge method: Dropping upper channels from median and std calculations")
else:
mad = MAD(intensityAllBaselineChannels)
median = nanmedian(intensityAllBaselineChannels)
useLowBaseline = True
else:
# Pick the n channels with the n lowest values (or highest if those
# have smallest MAD), but ignore edge channels inward for as long they
# are identical to themselves (i.e. avoid the
# effect of TDM edge flagging.)
myspectrum = spectrum
allBaselineXChannels = np.array(range(len(myspectrum)))
# the following could also be len(spectrum), since their lengths are identical
if (len(originalSpectrum) > 10): # and len(originalSpectrum) <= 128):
# Was opened up to all data (not just TDM) in Cycle 6
# picks up some continuum in self-absorbed area in Cha-MMs1_CS in 200-channel spw
if False:
# ALMA Cycle 4+5
idx = np.where((originalSpectrum != originalSpectrum[0]) * (originalSpectrum != originalSpectrum[-1]))
else:
# ALMA Cycle 6
edgeValuedChannels = np.where((originalSpectrum == originalSpectrum[0]) | (originalSpectrum == originalSpectrum[-1]))[0]
edgeValuedChannelsLists = splitListIntoContiguousLists(edgeValuedChannels)
print("edgeValuedChannelsLists: ", edgeValuedChannelsLists)
idx = np.array(range(edgeValuedChannelsLists[0][-1] + 1, edgeValuedChannelsLists[-1][0]))
if len(idx) > 0:
allBaselineXChannels = idx
myspectrum = spectrum[idx]
casalogPost('Avoided %d edge channels of %d when computing min channels' % (len(spectrum)-len(myspectrum), len(spectrum)),debug=True)
if len(spectrum)-len(myspectrum) > 0:
casalogPost("using channels %d-%d" % (idx[0],idx[-1]),debug=True)
allBaselineXChannelsOriginal = allBaselineXChannels
# Sort by intensity: myspectrum is often only a subset of spectrum, so
# the channel numbers in original spectrum must be tracked separately
# in the variable: allBaselineXChannels
idx = np.argsort(myspectrum)
intensityAllBaselineChannels = myspectrum[idx[:nBaselineChannels]]
allBaselineOriginalChannels = originalSpectrum[idx[:nBaselineChannels]]
highestChannels = myspectrum[idx[-nBaselineChannels:]]
medianOfAllChannels = nanmedian(myspectrum)
mad0 = MAD(intensityAllBaselineChannels)
mad1 = MAD(highestChannels)
middleChannels = myspectrum[idx[nBaselineChannels:-nBaselineChannels]]
madMiddleChannels = MAD(middleChannels)
# Introduced the lowHighBaselineThreshold factor on Aug 31, 2016 for CAS-8938
whichBaseline = np.argmin([mad0, lowHighBaselineThreshold*mad1, lowHighBaselineThreshold*madMiddleChannels])
if (whichBaseline == 0):
useBaseline = 'low'
elif (whichBaseline == 1):
useBaseline = 'high'
else:
useBaseline = 'middle'
# In the following if/elif/else, we convert the tri-valued variable useBaseline into two
# Booleans for legacy purposes (i.e before useMiddleChannels was considered as an option).
if (useBaseline == 'high'):
# This is the absorption line case
casalogPost("%s Using highest %d channels as baseline because low:mid:high = %g:%g:%g" % (projectCode,nBaselineChannels,mad0,madMiddleChannels,mad1), debug=True)
intensityAllBaselineChannels = highestChannels[::-1] # reversed it so that first channel is highest value
allBaselineXChannels = allBaselineXChannels[idx][-nBaselineChannels:]
allBaselineXChannels = allBaselineXChannels[::-1] # reversed it so that first channel is highest value
mad0 = MAD(intensityAllBaselineChannels)
useLowBaseline = False
useMiddleChannels = False
elif useBaseline == 'middle':
# This case is needed when there is a mix of emission and absorption lines,
# as in Band 10 Orion = 2016.1.00970.S spw25,31.
casalogPost("%s Using middle %d channels as baseline because low:mid:high = %g:%g:%g" % (projectCode, len(middleChannels),mad0,madMiddleChannels,mad1), debug=True)
intensityAllBaselineChannels = middleChannels
allBaselineXChannels = allBaselineXChannels[idx][nBaselineChannels:-nBaselineChannels]
dropBaselineChannels = 0
mad0 = MAD(intensityAllBaselineChannels)
useLowBaseline = False
useMiddleChannels = True
else:
# This is the emission line case
casalogPost("%s Using lowest %d channels as baseline because low:mid:high = %g:%g:%g" % (projectCode,nBaselineChannels,mad0,madMiddleChannels,mad1), debug=True)
useLowBaseline = True
useMiddleChannels = False
allBaselineXChannels = allBaselineXChannels[idx][:nBaselineChannels]
# allBaselineXChannels = idx[:nBaselineChannels]
casalogPost("Median of all channels = %f, MAD of selected baseline channels = %f" % (medianOfAllChannels,mad0))
madRatio = None
if dropBaselineChannels > 0:
dropExtremeChannels = int(int(len(idx)*dropBaselineChannels)/100)
if dropExtremeChannels > 0:
intensityAllBaselineChannelsDropExtremeChannels = myspectrum[idx[dropExtremeChannels:nBaselineChannels+dropExtremeChannels]]
allBaselineXChannelsDropExtremeChannels = allBaselineXChannelsOriginal[idx[dropExtremeChannels:nBaselineChannels+dropExtremeChannels]]
mad0_dropExtremeChannels = MAD(intensityAllBaselineChannelsDropExtremeChannels)
if mad0_dropExtremeChannels > 0:
# prevent division by zero error
madRatio = mad0/mad0_dropExtremeChannels
if madRatioLowerLimit < madRatio < madRatioUpperLimit:
# more than 1.2 means there was a significant improvement; more than 1.5 means something unexpected about the statistics
casalogPost("****** Dropping most extreme %d = %.1f%% of channels when computing the MAD, since it reduces the mad by a factor of x=%.2f (%.2f<x<%.2f)" % (dropExtremeChannels, dropBaselineChannels, madRatio, madRatioLowerLimit, madRatioUpperLimit))
intensityAllBaselineChannels = intensityAllBaselineChannelsDropExtremeChannels
# print("len(allBaselineXChannels)=%d, len(idx)=%d, dropExtremeChannels=%d, nBaselineChannels+dropExtremeChannels=%d" % (len(allBaselineXChannels), len(idx), dropExtremeChannels, nBaselineChannels+dropExtremeChannels))
allBaselineXChannels = allBaselineXChannelsDropExtremeChannels
allBaselineOriginalChannels = originalSpectrum[allBaselineXChannelsDropExtremeChannels] # allBaselineXChannels[idx][dropExtremeChannels:nBaselineChannels+dropExtremeChannels]]
else:
casalogPost("**** Not dropping most extreme channels when computing the MAD, since the change in MAD of %.2f is not within %.2f<x<%.2f" % (madRatio, madRatioLowerLimit, madRatioUpperLimit))
casalogPost("min method: computing MAD and median of %d channels used as the baseline" % (len(intensityAllBaselineChannels)))
mad = MAD(intensityAllBaselineChannels)
madOriginal = MAD(allBaselineOriginalChannels)
# casalogPost("MAD of all baseline channels = %f (%s)" % (mad,intensityAllBaselineChannels))
casalogPost("MAD of all baseline channels = %f" % (mad))
if (fitResult is not None):
casalogPost("MAD of original baseline channels (before removal of fit) = %f" % (madOriginal))
if (mad < 1e-17 or madOriginal < 1e-17):
casalogPost("min method: avoiding blocks of identical-valued channels")
if (len(originalSpectrum) > 10):
myspectrum = spectrum[np.where((originalSpectrum != originalSpectrum[0]) * (originalSpectrum != originalSpectrum[-1]))]
else: # original logic, prior to linear fit removal
myspectrum = spectrum[np.where(spectrum != intensityAllBaselineChannels[0])]
idx = np.argsort(myspectrum)
intensityAllBaselineChannels = myspectrum[idx[:nBaselineChannels]]
# allBaselineXChannels = idx[:nBaselineChannels]
for i in range(len(idx)):
casalogPost("idx[%d] = %d" % (i,idx[i]))
casalogPost("len(allBaselineXChannels)=%d, shape(idx)=%s, nBaselineChannels=%d" % (len(allBaselineXChannels), str(np.shape(idx)), nBaselineChannels))
allBaselineXChannels = allBaselineXChannels[idx][:nBaselineChannels]
casalogPost(" computing MAD and median of %d channels used as the baseline" % (len(intensityAllBaselineChannels)))
mad = MAD(intensityAllBaselineChannels)
mad = MAD(intensityAllBaselineChannels)
median = nanmedian(intensityAllBaselineChannels)
casalogPost("min method: median intensity of %d channels used as the baseline: %f, mad: %f" % (len(intensityAllBaselineChannels), median, mad))
# signalRatio will be 1.0 if no lines present and 0.25 if half the channels have lines, etc.
signalRatio = (1.0 - 1.0*len(np.where(np.abs(spectrum-median)>(sigmaEffective*mad*2))[0]) / len(spectrum))**2
if signalRatio >= 0.99 or ((signalRatio > 0.98) and (15 > peakOverMad > 10)):
# If nearly no lines are found, then look a bit deeper to allow the possibility of signalRatio to drop enough to
# go below 0.965, and thereby prevent the desensitizing that comes later (see line 2542).
weakLineFactor = 1.35 #1.39 gives 0.982 on 305 # 1.5 gives 1.0 on 305
casalogPost("Initial signalRatio: %f, peakOverMad=%f, Re-assessing." % (signalRatio,peakOverMad))
signalRatio = (1.0 - 1.0*len(np.where(np.abs(spectrum-median)>(sigmaEffective*mad*weakLineFactor))[0]) / len(spectrum))**2
maxdiff = np.max(np.abs(spectrum-median))
casalogPost("max(abs(spectrum-median)) = %f while sigmaEffective(%f)*mad(%f)*%.2f=%f" % (maxdiff,sigmaEffective,mad,weakLineFactor,sigmaEffective*mad*weakLineFactor))
originalMedian = np.median(originalSpectrum)
# Should not divide by post-baseline-fit median since it may be close to 0
spectralDiff = 100*np.median(np.abs(np.diff(spectrum)))/originalMedian
spectralDiff2 = 100*np.median(np.abs(np.diff(spectrum,n=2)))/originalMedian
casalogPost("signalRatio=%f, spectralDiff = %f and spectralDiff2=%f percent of the median" % (signalRatio, spectralDiff,spectralDiff2))
lineStrengthFactor = 1.0/signalRatio
if (spectralDiff2 < 0.65 and npts > 192 and signalRatio<signalRatioTier2): # used to be <0.95 but regression 321 has 0.949 so use 0.94
# This appears to be a channel-averaged FDM spectrum with lots of real line emission.
# So, don't allow the median to be raised, and reduce the mad to lower the threshold.
# page 15: G11.92_B7.ms_spw3 yields spectralDiff2=0.6027
# We can also get into here if there is a large slope in the mean spectrum, so we
# counter that by removing a linear slope before evaluating lineSNR.
casalogPost('The spectral difference (n=2) is rather small, so set signalRatio=0 to reduce the baseline level.',debug=True)
signalRatio = 0
if True:
# note: this slope removal differs from the one in runFindContinuum because
# it always acts upon the whole spectrum, not just the potential baseline windows.
print("Removing linear slope for purposes of computing lineSNR.")
x = np.arange(len(spectrum))
slope, intercept = linfit(x, spectrum, MAD(spectrum))
newspectrum = spectrum - x*slope
newmad = MAD(newspectrum)
# Avoid high edge channels from creating false high SNR
lineSNR = (np.max(newspectrum[1:-1])-np.median(newspectrum))/newmad
else:
# Avoid high edge channels from creating false high SNR
lineSNR = (np.max(spectrum[1:-1])-median)/mad
casalogPost('lineSNR = %f' % lineSNR)
if (lineSNR > lineSNRThreshold):
casalogPost('The lineSNR > %d, so scaling the mad by 1/3 to reduce the threshold.' % lineSNRThreshold, debug=True)
mad *= 0.33
if (trimChannels == 'auto'):
trimChannels = 6
casalogPost('Setting trimChannels to %d.' % (trimChannels))
else:
casalogPost('Not reducing mad by 1/3: npts=%d, signalRatio=%.3f, spectralDiff2=%.2f' % (npts,signalRatio,spectralDiff2),debug=True)
if useMiddleChannels:
medianTrue = median
else:
# Do not use signalRatio here as the 5th parameter anymore, because we may have set it to zero just above,
# but 1/lineStrengthFactor is the same value as before it was set to zero. Setting it to zero causes
# problems on non-hot core FDM, like spw29 of 2015.1.00581.S, uid://A001/X2f6/X16b (CAS-11671). Not
# setting it to zero changes the selection on hot cores, some giving more channels, some giving fewer,
# but not drastically either direction. Largest change is on spw22 of uid://A001/X2f6/X265 (page 245).
medianTrue = medianCorrected(baselineMode, percentile, median, mad,
1.0/lineStrengthFactor, useLowBaseline)
if medianTrue < np.min(spectrum):
casalogPost("**** PIPE-525: Corrected median is less than all data points. Using true median instead.")
medianTrue = np.median(spectrum)
peakFeatureSigma = (np.max(spectrum)-medianTrue)/mad
# De-sensitize the threshold in order to avoid small differences in noise levels
# (e.g. that occur between serial and parallel tclean cubes)
# from producing drastically different continuum selections.
if ((signalRatio > signalRatioTier1) or (signalRatio > signalRatioTier2 and peakOverMad < 5)) and (sigmaFindContinuumMode in ['auto','autolower']):
casalogPost('findContinuumChannels: desensitizing because %f>%f or (%f>%f and %f<5)' % (signalRatio,signalRatioTier1,signalRatio,signalRatioTier2,peakOverMad))
maxPeakFeatureSigma = (np.max(spectrum) - np.min(spectrum))/MAD(spectrum)
if peakFeatureSigma > maxPeakFeatureSigma:
oldmad = 1*mad
mad *= peakFeatureSigma/maxPeakFeatureSigma
casalogPost('findContinuumChannels: Limiting peakFeatureSigma from %f to %f, reset MAD from %f to %f' % (peakFeatureSigma,maxPeakFeatureSigma,oldmad,mad))
peakFeatureSigma = maxPeakFeatureSigma
else:
casalogPost('findContinuumChannels: not limiting peakFeatureSigma because %f > %f' % (peakFeatureSigma,maxPeakFeatureSigma))
threshold = sigmaEffective*mad + medianTrue
# Use a (default=15%) lower negative threshold to help prevent false identification of absorption features.
negativeThreshold = -negativeThresholdFactor*sigmaEffective*mad + medianTrue
casalogPost("MAD = %f, median = %f, trueMedian=%f, signalRatio=%f" % (mad, median, medianTrue, signalRatio))
casalogPost("findContinuumChannels: computed threshold = %f, medianTrue=%f" % (threshold, medianTrue))
channels = np.where(spectrum < threshold)[0]
if (negativeThreshold is not None):
channels2 = np.where(spectrum > negativeThreshold)[0]
channels = np.intersect1d(channels,channels2)
# for CAS-8059: remove channels that are equal to the minimum if all
# channels from it toward the nearest edge are also equal to the minimum:
channels = list(channels)
if (abs(originalSpectrum[np.min(channels)] - np.min(originalSpectrum)) < abs(1e-10*np.min(originalSpectrum))):
lastmin = np.min(channels)
channels.remove(lastmin)
removed = 1
casalogPost("Checking channels %d-%d" % (np.min(channels),np.max(channels)))
for c in range(np.min(channels),np.max(channels)):
mydiff = abs(originalSpectrum[c] - np.min(originalSpectrum))
mycrit = abs(1e-10*np.min(originalSpectrum))
if (mydiff > mycrit):
break
if c in channels:
channels.remove(c)
removed += 1
casalogPost("Removed %d channels on low channel edge that were at the minimum." % (removed))
# Now come in from the upper side
if (abs(originalSpectrum[np.max(channels)] - np.min(originalSpectrum)) < abs(1e-10*np.min(originalSpectrum))):
lastmin = np.max(channels)
channels.remove(lastmin)
removed = 1
casalog.post("Checking channels %d-%d" % (np.max(channels),np.min(channels)))
for c in range(np.max(channels),np.min(channels)-1,-1):
mydiff = abs(originalSpectrum[c] - np.min(originalSpectrum))
mycrit = abs(1e-10*np.min(originalSpectrum))
if (mydiff > mycrit):
break
if c in channels:
channels.remove(c)
removed += 1
casalogPost("Removed %d channels on high channel edge that were at the minimum." % (removed))
peakChannels = np.where(spectrum > threshold)[0]
peakChannelsLists = splitListIntoContiguousLists(peakChannels)
widthOfWidestFeature = maxLengthOfLists(peakChannelsLists)
casalogPost("Width of widest feature = %d (length spectrum = %d)" % (widthOfWidestFeature, len(spectrum)))
# C4R2 had signalRatio < 0.6 and spectralDiff2 < 1.2 but this yielded only 1 channel
# of continuum on NGC6334I spw25 when memory expanded to 256GB.
if (signalRatio > 0 and signalRatio < 0.925 and spectralDiff2 < 1.3 and
len(spectrum) > 1000 and trimChannels=='auto' and
widthOfWidestFeature < len(spectrum)/8):
# This is meant to prevent rich hot cores from returning only 1
# or 2 channels of continuum. signalRatio>0 is to avoid conflict
# with the earlier heuristic above where it is set to zero.
trimChannels = 13
if autoNarrow:
narrow = 2
casalogPost('Setting trimChannels=%d, narrow=%s since many lines appear to be present (signalRatio=%f).' % (trimChannels,str(narrow), signalRatio))
else:
casalogPost('Not changing trimChannels from %s: signalRatio=%f, spectralDiff2=%f' % (str(trimChannels), signalRatio, spectralDiff2))
peakMultiChannelsLists = splitListIntoContiguousListsAndRejectNarrow(peakChannels, narrow=2)
allGroupsAboveSFC = len(peakChannelsLists)
singleChannelPeaksAboveSFC = allGroupsAboveSFC - len(peakMultiChannelsLists)
selection = convertChannelListIntoSelection(channels)
casalogPost("Found %d potential continuum channels: %s" % (len(channels), str(selection)))
if (len(channels) == 0):
selection = ''
groups = 0
else:
channels = splitListIntoContiguousListsAndRejectZeroStd(channels, spectrum, nanmin, verbose=verbose)
if verbose:
print("channels = ", channels)
selection = convertChannelListIntoSelection(channels,separator=separator)
groups = len(selection.split(separator))
casalogPost("Found %d channels in %d groups after rejecting zero std: %s" % (len(channels), groups, str(selection)))
if (len(channels) == 0):
selection = ''
else:
# casalogPost("Calling splitListIntoContiguousListsAndTrim(totalChannels=%s, channels=%s, trimChannels=%s, maxTrim=%d, maxTrimFraction=%f)" % (str(npts), str(channels), str(trimChannels), maxTrim, maxTrimFraction))
originalChannels = channels[:]
channels = splitListIntoContiguousListsAndTrim(npts, channels,
trimChannels, maxTrim, maxTrimFraction, verbose)
selection = convertChannelListIntoSelection(channels)
if selection == '': # fix for PIPE-359
casalogPost("WARNING: all channels trimmed, which is quite rare. Reverting to maxTrim=0.1.")
maxTrim = 0.1
channels = splitListIntoContiguousListsAndTrim(npts, originalChannels,
trimChannels, maxTrim, maxTrimFraction, verbose)
selection = convertChannelListIntoSelection(channels)
groups = len(selection.split(separator))
if (groups > maxGroupsForMaxTrimAdjustment and trimChannels=='auto'
and maxTrim>maxTrimDefault):
maxTrim = maxTrimDefault
casalogPost("Restoring maxTrim=%d because groups now > %d" % (maxTrim,maxGroupsForMaxTrimAdjustment))
# if verbose:
# casalogPost("Calling splitListIntoContiguousListsAndTrim(totalChannels=%s, channels=%s, trimChannels=%s, maxTrim=%d, maxTrimFraction=%f)" % (str(npts), str(channels), str(trimChannels), maxTrim, maxTrimFraction))
trialChannels = splitListIntoContiguousListsAndTrim(npts, channels,
trimChannels, maxTrim, maxTrimFraction, verbose)
if len(trialChannels) == 0:
casalogPost("Zero channels left after trimming. Reverting to prior selection.")
else:
channels = trialChannels
if verbose:
print("channels = ", channels)
selection = convertChannelListIntoSelection(channels)
groups = len(selection.split(separator))
if verbose:
print("Found %d groups of channels = " % (groups), channels)
if (groups > 1):
# if verbose:
# casalogPost("Calling splitListIntoContiguousListsAndRejectNarrow(channels=%s, narrow=%s)" % (str(channels), str(narrow)))
# else:
# casalogPost("Calling splitListIntoContiguousListsAndRejectNarrow(narrow=%s)" % (str(narrow)))
trialChannels = splitListIntoContiguousListsAndRejectNarrow(channels, narrow)
if (len(trialChannels) > 0):
channels = trialChannels
casalogPost("Found %d channels after trimming %s channels and rejecting narrow groups." % (len(channels),str(trimChannels)))
selection = convertChannelListIntoSelection(channels)
groups = len(selection.split(separator))
else:
casalogPost("Not rejecting narrow groups since there is only %d group!" % (groups))
casalogPost("Found %d continuum channels in %d groups: %s" % (len(channels), groups, selection))
potentialChannels, messages, rangesDropped = rejectNarrowInnerWindowsChannels(channels, fCCiteration)
if enableRejectNarrowInnerWindows:
channels = potentialChannels
for message in messages:
casalogPost(message)
else:
if rangesDropped > 0:
casalogPost("rejectNarrowInnerWindows wanted to remove %d ranges, but it was not enabled" % (rangesDropped))
selection = convertChannelListIntoSelection(channels)
groups = len(selection.split(separator))
casalogPost("Final: found %d continuum channels (sFC=%.2f) in %d groups: %s" % (len(channels), sigmaFindContinuum, groups, selection))
return(channels, selection, threshold, median, groups, correctionFactor,
medianTrue, mad, computeMedianCorrectionFactor(baselineMode, percentile)*signalRatio,
negativeThreshold, lineStrengthFactor, singleChannelPeaksAboveSFC,
allGroupsAboveSFC, [spectralDiff, spectralDiff2], trimChannels,
useLowBaseline, narrow, [allBaselineXChannels,intensityAllBaselineChannels],
madRatio, useMiddleChannels, signalRatio, rangesDropped)
[docs]def rejectNarrowInnerWindowsChannels(channels, fCCiteration=0):
"""
This function is called by findContinuumChannels.
If there are 3-15 groups of channels, then remove any inner window
that is narrower than both edge windows.
Returns: a list of channels
"""
mylists = splitListIntoContiguousLists(channels)
groups = len(mylists)
messages = []
rangesDropped = 0
# if (groups > 2 and groups < 8): C4R2 heuristic
if (groups > 2 and groups < 16): # Cycle 5 onward
channels = []
lenFirstGroup = len(mylists[0])
lenLastGroup = len(mylists[-1])
channels += mylists[0] # start with the first window
if (groups < 8):
widthFactor = 1
else:
# this is to prevent unnecessarily trimming narrow windows, e.g. in a hot core case
widthFactor = 0.2
# The final group needs to be included in the output channels list, and it is safe to
# simply included it in the loop because it will always pass the test.
for group in range(1,groups):
# if (len(mylists[group]) >= np.mean([lenFirstGroup,lenLastGroup])*widthFactor): # or len(mylists[group]) >= lenLastGroup*widthFactor):
widthThreshold = np.min([lenFirstGroup,lenLastGroup])
if (len(mylists[group]) >= (widthThreshold*widthFactor)):
channels += mylists[group]
if group < groups-1:
# Don't report the final group because it is not really under consideration here
messages.append('fCCiter%d: Keeping channel group %d because it is wider than %.1f*width of both edge groups' % (fCCiteration,group,widthFactor))
else:
rangesDropped += 1
messages.append('fCCiter%d: Dropping channel group %d because it is narrower (%d) than %.1f*width (%d) of both edge groups' % (fCCiteration, group, len(mylists[group]), widthFactor, widthThreshold*widthFactor))
if rangesDropped == 0:
messages.append('fCCiter%d: No channel groups dropped by rejectNarrowInnerWindowsChannels' % (fCCiteration))
else:
messages.append('fCCiter%d: Number of channel groups (%d) disqualifies it for rejectNarrowInnerWindowsChannels' % (fCCiteration,groups))
return(channels, messages, rangesDropped)
[docs]def splitListIntoContiguousListsAndRejectNarrow(channels, narrow=3):
"""
This function is called by findContinuumChannels.
Split a list of channels into contiguous lists, and reject those that
have a small number of channels.
narrow: if >=1, then interpret it as the minimum number of channels that
a group must have in order to survive
if <1, then interpret it as the minimum fraction of channels that
a group must have relative to the total number of channels
Returns: a new single list (as an array)
Example: [1,2,3,4,6,7,9,10,11] --> [ 1, 2, 3, 4, 9, 10, 11]
"""
length = len(channels)
mylists = splitListIntoContiguousLists(channels)
channels = []
for mylist in mylists:
if (narrow < 1):
if (len(mylist) <= fraction*length):
continue
elif (len(mylist) < narrow):
continue
channels += mylist
return(np.array(channels))
[docs]def splitListIntoContiguousListsAndTrim(totalChannels, channels, trimChannels=0.1,
maxTrim=maxTrimDefault,
maxTrimFraction=1.0, verbose=False):
"""
This function is called by findContinuumChannels.
Split a list of channels into contiguous lists, and trim some number
of channels from each edge.
totalChannels: number of channels in the spectrum
channels: a list of channels selected for potential continuum
trimChannels: if integer, use that number of channels. If float between
0 and 1, then use that fraction of channels in each contiguous list
(rounding up). If 'auto', then use 0.1 but not more than maxTrim
channels and not more than maxTrimFraction of channels.
maxTrim: used in 'auto' mode
Returns: a new single list
"""
if type(trimChannels) != str:
if (trimChannels <= 0):
return(np.array(channels))
length = len(channels)
trimChannelsMode = trimChannels
medianWidthOfRanges = np.median(countChannelsInRanges(convertChannelListIntoSelection(channels)))
if (trimChannels == 'auto'):
trimChannels = pickAutoTrimChannels(totalChannels, length, maxTrim, medianWidthOfRanges)
mylists = splitListIntoContiguousLists(channels)
channels = []
trimLimitForEdgeRegion = 3
for i,mylist in enumerate(mylists):
trimChan = trimChannels
if verbose:
print("trimChan=%d, Checking list = " % (trimChan), mylist)
if (trimChannels < 1):
trimChan = int(np.ceil(len(mylist)*trimChannels))
if verbose:
print("since trimChannels=%s<1; reset trimChan=%d" % (str(trimChannels),trimChan))
if (trimChannelsMode == 'auto' and 1.0*trimChan/len(mylist) > maxTrimFraction):
trimChan = int(np.floor(maxTrimFraction*len(mylist)))
if verbose:
print("trimChan for this window = %d" % (trimChan))
if (len(mylist) < 1+trimChan*2):
if (len(mylists) == 1):
# If there was only one list of 1 or 2 channels, then don't trim it away!
channels += mylist[:1]
continue
# Limit the trimming of the edge closest to the edge of the spw to 3 channels,
# in order to preserve bandwidth.
if (i==0 and trimChan > trimLimitForEdgeRegion):
if (len(mylists)==1):
# It is the only window so limit the trim on the far edge too
channels += mylist[trimLimitForEdgeRegion:-trimLimitForEdgeRegion]
else:
channels += mylist[trimLimitForEdgeRegion:-trimChan]
elif (i==len(mylists)-1 and trimChan > trimLimitForEdgeRegion):
channels += mylist[trimChan:-trimLimitForEdgeRegion]
else:
# It is not an edge window, or it is an edge window and trimChan<=3
channels += mylist[trimChan:-trimChan]
return(np.array(channels))
[docs]def maxLengthOfLists(lists):
"""
This function is called by findContinuumChannels.
lists: a list if lists
Returns: an integer that is the length of the longest list
"""
maxLength = 0
for a in lists:
if (len(a) > maxLength):
maxLength = len(a)
return maxLength
[docs]def pickAutoTrimChannels(totalChannels, length, maxTrim, medianWidthOfRanges):
"""
This function is called by splitListIntoContiguousListsAndTrim and pickTrimString.
Automatic choice of number of trimChannels as a function of the number
of potential continuum channels in an spw.
length: number of channels in the list of potential continuum channels
Returns: an integer or floating point value
"""
trimChannels = 0.1
casalogPost('median width of potential continuum ranges = %.1f' % (medianWidthOfRanges))
if (length*trimChannels > maxTrim and medianWidthOfRanges > maxTrim*1.5 and totalChannels > 256):
casalogPost("pickAutoTrimChannels(): set trimChannels = %d because %.0f > %d and totalChannels=%d>256" % (maxTrim,length*trimChannels,maxTrim,totalChannels))
trimChannels = maxTrim
else:
casalogPost("pickAutoTrimChannels(): set trimChannels = %.1f because %.0f <= %d or %.1f <= %f or totalChannels=%d <= 256" % (trimChannels,length*trimChannels,maxTrim,medianWidthOfRanges,maxTrim*1.5,totalChannels))
return(trimChannels)
[docs]def pickTrimString(totalChannels, trimChannels, length, maxTrim, selection):
"""
This function is called by runFindContinuum.
Generate a string describing the setting of the trimChannels parameter.
totalChannels: total number of points in spectrum
trimChannels: string (e.g. 'auto') or integer (e.g. 20) or float (e.g. 0.1)
length: number of channels in the spectrum (now unused)
selection: continuum channel selection string
Returns: a string
"""
if (trimChannels=='auto'):
contSelectionLength = np.sum(countChannelsInRanges(selection))
medianWidthOfRanges = np.median(countChannelsInRanges(selection))
# trimString = 'auto_max=%g' % pickAutoTrimChannels(length, maxTrim, medianWidthOfRanges)
trimString = 'auto_max=%g' % pickAutoTrimChannels(totalChannels, contSelectionLength, maxTrim, medianWidthOfRanges)
else:
trimString = '%g' % trimChannels
return(trimString)
[docs]def pickNarrowString(narrow, length, narrowValueModified=None):
"""
This function is called by runFindContinuum.
Generate a string describing the setting of the narrow parameter.
Returns: a string
"""
if (narrow=='auto'):
myNarrow = pickNarrow(length)
if (narrowValueModified is None or myNarrow == narrowValueModified):
narrowString = 'auto=%g' % myNarrow
else:
narrowString = 'auto=%g' % narrowValueModified
else:
narrowString = '%g' % narrow
return(narrowString)
[docs]def pickNarrow(length):
"""
This function is called by pickNarrowString and findContinuumChannels.
Automatically picks a setting for the narrow parameter of
findContinuumChannels() based on the number of channels in the spectrum.
Returns: an integer
Examples: This formula results in the following values:
length: 64,128,240,480,960,1920,3840,7680:
return: 2, 3, 3, 3, 3, 4, 4, 4 (ceil(log10)) **** current function ****
2, 2, 2, 3, 3, 3, 4, 4 (round_half_up(log10))
1, 2, 2, 2, 2, 3, 3, 3 (floor(log10))
5, 5, 6, 7, 7, 8, 9, 9 (ceil(log))
4, 5, 5, 6, 7, 8, 8, 9 (round_half_up(log))
4, 4, 5, 6, 6, 7, 8, 8 (floor(log))
"""
return(int(np.ceil(np.log10(length))))
[docs]def sigmaCorrectionFactor(baselineMode, npts, percentile):
"""
This function is called by findContinuumChannels.
Computes the correction factor for the fact that the measured rms (or MAD)
on the N%ile lowest points of a datastream will be less than the true rms
(or MAD) of all points.
Returns: a value between 0 and 1, which will be used to reduce the
estimate of the population sigma based on the sample sigma
"""
edgeValue = (npts/128.)**0.08
if (baselineMode == 'edge'):
return(edgeValue)
value = edgeValue*2.8*(percentile/10.)**-0.25
casalogPost("sigmaCorrectionFactor using percentile = %g to get sCF=%g" % (percentile, value), debug=True)
return(value)
[docs]def getImageInfo(img, returnBeamAreaPerChannel=False):
"""
This function is called by findContinuum and meanSpectrum.
Extract the beam and pixel information from a CASA image.
This was copied from getFitsBeam in analysisUtils.py.
Returns: A list of 12 things: bmaj, bmin, bpa, cdelt1, cdelt2,
naxis1, naxis2, frequency, shape, crval1, crval2, maxBaseline
Beam angles are in arcseconds (bpa in degrees), crvals are in radians
Frequency is in GHz and is the central frequency
MaxBaseline is in meters, inferred from freq and beamsize (will be zero for .residual images)
"""
ARCSEC_PER_RAD = 206264.80624709636
c_mks = 2.99792458e8
if (os.path.exists(img) == False):
print("image not found: ", img)
return
myia = iatool()
myia.open(img)
mydict = myia.restoringbeam()
if 'major' in mydict or 'beams' in mydict:
myqa = qatool()
if 'major' in mydict:
# single beam case
bmaj = myqa.convert(mydict['major'], 'arcsec')['value']
bmin = myqa.convert(mydict['minor'], 'arcsec')['value']
bpa = myqa.convert(mydict['positionangle'], 'deg')['value']
elif 'beams' in mydict:
# perplane beams
beams = mydict['beams']
major = []
minor = []
sinpa = []
cospa = []
for chan_beam in beams.values():
for chan_pol_beam in chan_beam.values():
major.append(myqa.convert(chan_pol_beam['major'], 'arcsec')['value'])
minor.append(myqa.convert(chan_pol_beam['minor'], 'arcsec')['value'])
sinpa.append(np.sin(myqa.convert(chan_pol_beam['positionangle'], 'rad')['value']))
cospa.append(np.cos(myqa.convert(chan_pol_beam['positionangle'], 'rad')['value']))
if returnBeamAreaPerChannel:
return np.array(major)*np.array(minor)
bmaj = np.median(major)
bmin = np.median(minor)
bpa = np.degrees(np.arctan2(np.median(sinpa), np.median(cospa)))
else:
print("Unrecognized beam dictionary.")
return
else:
bmaj = 0
bmin = 0
bpa = 0
# if 'mask' not in img:
# print("Warning: No beam found in header.")
naxis1 = myia.shape()[0]
naxis2 = myia.shape()[1]
axis = findSpectralAxis(myia)
mycs = myia.coordsys()
myqa = qatool()
restfreq = myqa.convert(mycs.restfrequency(), 'Hz')['value'][0]
cdelt1 = mycs.increment()['numeric'][0] * ARCSEC_PER_RAD # arcsec
cdelt2 = mycs.increment()['numeric'][1] * ARCSEC_PER_RAD # arcsec
crval1 = mycs.referencevalue()['numeric'][0] # radian
crval2 = mycs.referencevalue()['numeric'][1] # radian
deltaFreq = mycs.increment()['numeric'][axis]
frequency = mycs.referencevalue()['numeric'][axis]
frequencyGHz = frequency * 1e-9
mycs.done()
bunit = myia.brightnessunit()
velocityWidth = abs(c_mks * 0.001 * deltaFreq / frequency)
shape = myia.shape()
myia.close()
myqa.done()
if bmaj > 0:
maxBaseline = c_mks / ((1e9*frequencyGHz*(bmaj*bmin)**0.5)/ARCSEC_PER_RAD)
else:
maxBaseline = 0
return([bmaj,bmin,bpa,cdelt1,cdelt2,naxis1,naxis2,frequencyGHz,shape,crval1,crval2,maxBaseline])
[docs]def numberOfChannelsInCube(img, returnFreqs=False, returnChannelWidth=False,
verbose=False):
"""
This function is called by findContinuum, cubeLSRKToTopo,
computeStatisticalSpectrumFromMask, and meanSpectrum.
Finds the number of channels in a CASA image cube.
returnFreqs: if True, then also return the frequency of the
first and last channel (in Hz)
returnChannelWidth: if True, then also return the channel width (in Hz)
verbose: if True, then print the frequencies of first and last channel
-Todd Hunter
"""
if (not os.path.exists(img)):
print("Image not found.")
return
myia = iatool()
myia.open(img)
axis = findSpectralAxis(myia)
naxes = len(myia.shape())
nchan = myia.shape()[axis]
mycs = myia.coordsys()
cdelt = mycs.increment()['numeric'][axis]
pixel = [0]*naxes
firstFreq = mycs.toworld(pixel, format='n')['numeric'][axis]
pixel[axis] = nchan-1
lastFreq = mycs.toworld(pixel, format='n')['numeric'][axis]
mycs.done()
myia.close()
if (returnFreqs):
if (returnChannelWidth):
return(nchan,firstFreq,lastFreq,cdelt)
else:
return(nchan,firstFreq,lastFreq)
else:
if (returnChannelWidth):
return(nchan,cdelt)
else:
return(nchan)
[docs]def nanmean(a, axis=0):
"""
This function is called by findContinuumChannels, runFindContinuum, and avgOverCube.
Takes the mean of an array, ignoring the nan entries
"""
if list(map(int, np.__version__.split('.')[:3])) < [1, 8, 1]:
return scipy_nanmean(a, axis)
else:
return np.nanmean(a, axis)
def _nanmedian(arr1d, preop=None): # This only works on 1d arrays
"""
Private function for rank a arrays. Compute the median ignoring Nan.
This function is called by nanmedian(), which is in turn called by MAD.
Parameters
----------
arr1d : ndarray
Input array, of rank 1.
Results
-------
m : float
The median.
"""
x = arr1d.copy()
c = np.isnan(x)
s = np.where(c)[0]
if s.size == x.size:
warnings.warn("All-NaN slice encountered", RuntimeWarning)
return np.nan
elif s.size != 0:
# select non-nans at end of array
enonan = x[-s.size:][~c[-s.size:]]
# fill nans in beginning of array with non-nans of end
x[s[:enonan.size]] = enonan
# slice nans away
x = x[:-s.size]
if preop:
x = preop(x)
return np.median(x, overwrite_input=True)
[docs]def findSpectralAxis(img):
"""
This function is called by computeStatisticalSpectrumFromMask, getImageInfo,
findContinuum and numberOfChannelsInCube.
Finds the spectral axis number of an image tool instance, or an image.
img: string or iatool instance
"""
if (type(img) == str):
myia = iatool()
myia.open(img)
needToClose = True
else:
myia = img
needToClose = False
mycs = myia.coordsys()
try:
iax = mycs.findaxisbyname("spectral")
except:
print("ERROR: can't find spectral axis. Assuming it is 3.")
iax = 3
mycs.done()
if needToClose: myia.close()
return iax
[docs]def countUnmaskedPixels(img, useImstat=True):
"""
This function is called by meanSpectrumFromMom0Mom8JointMask.
Returns number of unmasked pixels in an multi-dimensional image, i.e.
where the internal mask is True.
Total pixels: spatial * spectral * Stokes
Todd Hunter
"""
if useImstat:
npix = imstat(img, listit=imstatListit, verbose=imstatVerbose)['npts']
if type(npix) == list or type(npix) == np.ndarray:
if len(npix) == 0:
npix = 0
else:
npix = int(npix[0])
return npix
else:
myia = iatool()
myia.open(img)
maskdata = myia.getregion(getmask=True)
myia.close()
idx = np.where(maskdata==0)[0]
maskedPixels = len(idx)
pixels = np.prod(np.shape(maskdata))
return pixels-maskedPixels
[docs]def countPixelsAboveZero(img, useImstat=True, value=0):
"""
This function is called by meanSpectrumFromMom0Mom8JointMask.
Returns number of pixels > a specified threshold.
value: threshold (default=0)
Total pixels: spatial * spectral * Stokes
Todd Hunter
"""
if not os.path.exists(img):
print("Could not find image: ", img)
return
if useImstat:
npix = imstat(img, mask='"%s">0'%(img), listit=imstatListit, verbose=imstatVerbose)['npts']
if type(npix) == list or type(npix) == np.ndarray:
if len(npix) == 0:
npix = 0
else:
npix = int(npix[0])
return npix
else:
myia = iatool()
myia.open(img)
data = myia.getregion()
myia.close()
idx = np.where(data>value)[0]
pixels = len(idx)
return pixels
[docs]def flattenMask(mask, outfile='', overwrite=True):
"""
Takes a multi-channel CASA image mask and propagates all pixels to a
new single plane image mask
outfile: default name = <mask>.flattened
- Todd Hunter
"""
if (not os.path.exists(mask)):
print("Could not find mask image.")
return
if (outfile == ''):
outfile = mask + '.flattened'
axis = findSpectralAxis(mask)
imcollapse(mask, outfile=outfile, function='max', axes=axis,
overwrite=overwrite)
return outfile
[docs]def imagePercentileNoMask(img, percentiles):
"""
percentiles: a single value or a list
Returns: a single value or a list
"""
myia = iatool()
myia.open(img)
mymask = myia.getregion(getmask=True)
pixels = myia.getregion(getmask=False)
myia.close()
if type(percentiles) != list and type(percentiles) != np.ndarray:
value = scoreatpercentile(pixels[np.where(mymask > 0.5)], percentile)
return value
value = []
for percentile in percentiles:
value.append(scoreatpercentile(pixels[np.where(mymask > 0.5)], percentile))
return value
[docs]def meanSpectrumFromMom0Mom8JointMask(cube, imageInfo, nchan, pbcube=None, psfcube=None, minbeamfrac=0.3,
projectCode='', overwriteMoments=False,
overwriteMasks=True, phase2=True,
normalizeByMAD=True, minPixelsInJointMask=3,
initialQuadraticImprovementThreshold=1.6, userJointMask='',
snrThreshold=23, mom0minsnr=MOM0MINSNR_DEFAULT,
mom8minsnr=MOM8MINSNR_DEFAULT, rmStatContQuadratic=True,
bidirectionalMaskPhase2=False, outdir='',
avoidExtremaInNoiseCalcForJointMask=False, momentdir='',
statistic='mean'):
"""
This function is called by runFindContinuum when meanSpectrumMethod='mom0mom8jointMask'.
This is the new heuristic for Cycle 6 which creates the moment 0 and moment 8 images
for a cube, takes their union and determines the mean spectrum by calling
computeStatisticalSpectrumFromMask(), which uses ia.getprofile.
pbcube: if not specified, then assume '.residual' should be replaced in the name by '.pb'
overwriteMoments: rebuild the mom0 and mom8 images even if they already exist
overwriteMasks: rebuild the mom0 and mom8 mask images even if they already exist
phase2: if True, then run a second phase if SNR is high
minPixelsInJointMask: if fewer than these pixels are found, then use all pixels above pb=0.3
userJointMask: if specified, use this joint mask instead of computing one; if it is a cube mask,
as from tclean, then this function will form a flattened version first, and then use that
snrThreshold: if SNR is higher than this, and phase2==True, then run a phase 2 mask calculation
mom0minsnr: sets the threshold for the mom0 image (i.e. median + mom0minsnr*scaledMAD of the whole image)
mom8minsnr: sets the threshold for the mom8 image (i.e. median + mom8minsnr*scaledMAD of thw whole image)
rmStatContQuadratic: if True, then do not remove the old-style quadratic in this function in
favor of the new-style removal elsewhere
bidirectionalMaskPhase2: True=extend mask to negative values beyond threshold; False=Cycle6 behavior
outdir: directory to write the mom0, mom8, masks, .dat and mean spectrum text files
avoidExtremaInNoiseCalcForJointMask: experimental Boolean to avoid pixels <5%ile and >95%ile
in the chauvenet imstat of mom0 and mom8 images
momentdir: alternate directory to look for existing mom0 and mom8 images
Returns: 13 things:
* 1) the meanSpectrum
* 2) a Boolean which states whether normalization was applied
* 3) the number of pixels in the mask
* 4) a Boolean for whether the mask reverted to pb-based
* 5) a Boolean for whether a quadratic was removed
* 6) the initialQuadraticImprovementRatio
* 7) the list of 3 mom0snrs
* 8) the list of 3 mom8snrs
* 9) the number of regions pruned
* 10) number of pixels in moment 8 mask
* 11) mom0peak (Jy*km/s)
* 12) mom8peak (Jy)
* 13) name of jointMask file
"""
print("momentdir = ", momentdir)
overwritePhase2 = True
# Look for the .pb image if it was not specified
if pbcube is None:
pbcube = locatePBCube(cube)
# Look for the .psf image if it was not specified
if psfcube is None:
if cube.find('.residual') >= 0:
if os.path.islink(cube):
psfcube = os.readlink(cube).replace('.residual','.psf')
else:
psfcube = cube.replace('.residual','.psf')
if not os.path.exists(psfcube):
psfcube = None
casalogPost('psfcube = %s' % (psfcube))
if momentdir == '':
if outdir == '':
mom0 = cube+'.mom0'
mom8 = cube+'.mom8'
else:
mom0 = os.path.join(outdir,os.path.basename(cube)+'.mom0')
mom8 = os.path.join(outdir,os.path.basename(cube)+'.mom8')
else:
mom0 = os.path.join(outdir,os.path.basename(cube)+'.mom0')
mom8 = os.path.join(outdir,os.path.basename(cube)+'.mom8')
if not os.path.exists(mom0):
firstParentDir = os.path.dirname(cube).split()[-1]
if momentdir[0] == '.':
momentdir = os.path.join(os.getcwd(),momentdir)
mom0src = os.path.join(momentdir, outdir, os.path.basename(cube)+'.mom0')
print("Looking for mom0 at: ", mom0src)
if os.path.exists(mom0src):
casalogPost('Creating symlink to moment 0')
os.symlink(mom0src, mom0)
if not os.path.exists(mom8):
mom8src = os.path.join(momentdir, outdir, os.path.basename(cube)+'.mom8')
if os.path.exists(mom8src):
casalogPost('Creating symlink to moment 8')
os.symlink(mom8src, mom8)
if os.path.exists(mom0):
if overwriteMoments:
os.system('rm -rf ' + mom0)
else:
mom0info = getImageInfo(mom0)
# check if RA/Dec images sizes match
if imageInfo[5] == mom0info[5] and imageInfo[6] == mom0info[6]:
print("Re-using existing moment0 image at %s" % (mom0))
else:
print("Mismatch between cube and mom0 image, rebuilding mom0")
os.system('rm -rf ' + mom0)
if not os.path.exists(mom0):
print("Could not find: ", mom0)
casalogPost("Running immoments('%s', moments=[0], outfile='%s')" % (cube, mom0))
immoments(cube, moments=[0], outfile=mom0)
if os.path.exists(mom8):
if overwriteMoments:
os.system('rm -rf ' + mom8)
else:
mom8info = getImageInfo(mom8)
# check if RA/Dec images sizes match
if imageInfo[5] == mom8info[5] and imageInfo[6] == mom8info[6]:
print("Re-using existing moment8 image at %s" % (mom8))
else:
print("Mismatch between cube and mom8 image, rebuilding mom8")
os.system('rm -rf ' + mom8)
if not os.path.exists(mom8):
casalogPost("Running immoments('%s', moments=[8], outfile='%s')" % (cube, mom8))
# immoments(cube, moments=[8], outfile=mom8)
immoments(cube, moments=[8], outfile=mom8, mask='"%s"!=0'%(cube))
pbBasedMask = False
mom0mask = mom0+'.mask_bi'
mom0mask2 = mom0+'.mask2_bi'
mom8mask = mom8+'.mask_bi'
mom8mask2 = mom8+'.mask2_bi'
if outdir == '':
jointMask = cube+'.joint.mask'
jointMask2 = cube+'.joint.mask2'
else:
jointMask = os.path.join(outdir, os.path.basename(cube)+'.joint.mask')
jointMask2 = os.path.join(outdir, os.path.basename(cube)+'.joint.mask2')
lowerAnnulusLevel = None
higherAnnulusLevel = None
# sevenMeter = False
# if sevenMeter:
# snrThreshold = 20
# else:
# snrThreshold = 25
regionsPruned = 0
# if overwriteMasks or not os.path.exists(mom0mask) or not os.path.exists(mom8mask):
if (overwriteMasks or not os.path.exists(mom0mask) or not os.path.exists(mom8mask)) and userJointMask == '':
#####################
# Build Moment 0 mask
#####################
nptsInCube = countUnmaskedPixels(cube)
cmd = 'rm -rf %s.mask*' % (mom0)
print("running: %s" % cmd)
os.system(cmd)
classicResult = imstat(mom0, listit=imstatListit, verbose=imstatVerbose)
if avoidExtremaInNoiseCalcForJointMask:
avoidLow, avoidHigh = imagePercentileNoMask(mom0, [5,95])
mask = '"%s" > %.9f && "%s" < %.9f' % (mom0, avoidLow, mom0, avoidHigh)
print("running imstat('%s', algorithm='chauvenet', maxiter=5, mask='%s')" % (mom0,mask))
result = imstat(mom0, algorithm='chauvenet', maxiter=5, listit=imstatListit, verbose=imstatVerbose,
mask=mask)
else:
print("running imstat('%s', algorithm='chauvenet', maxiter=5)" % (mom0))
result = imstat(mom0, algorithm='chauvenet', maxiter=5, listit=imstatListit, verbose=imstatVerbose)
mom0peak = classicResult['max'][0]
mom0snr = classicResult['max'][0]/result['medabsdevmed'][0]
scaledMAD = result['medabsdevmed'][0]*1.4826
if False:
mom0sigma = mom0minsnr
else:
if mom0minsnr == MOM0MINSNR_DEFAULT:
mom0sigma = np.max([mom0minsnr,oneEvent(nptsInCube,1)])
else:
mom0sigma = mom0minsnr
casalogPost("+++ nptsInCube=%d, initial mom0sigma=%f, scaledMAD=%f, mom0median=%f" % (nptsInCube, mom0sigma, scaledMAD, result['median'][0]))
mom0threshold = mom0sigma*scaledMAD + result['median'][0]
mom0min = classicResult['min'][0]
mom0max = classicResult['max'][0]
# the test against mom0max is to prevent infinite loop
while (mom0min >= mom0threshold and mom0threshold < mom0max):
# Then there will be no points that satisfy subsequent mask, so raise the threshold so that at least some points are considered to be signal-free
mom0sigma += 1
print(" %e < %e: increasing mom0sigma to %d = %f" % (mom0min,mom0threshold,mom0sigma,mom0sigma*scaledMAD))
mom0threshold = mom0sigma*scaledMAD + result['median'][0]
mask = '"%s" > %.9f || "%s" < -%.9f' % (mom0, mom0threshold, mom0, mom0threshold)
print("applying mask to mom0: ", mask)
# mom0.mask_chauv_bi: this image will have the true values of the image where it is not masked
imsubimage(mom0, mask=mask, outfile=mom0+'.mask_chauv_bi')
# mom0.mask_bi: this image will have the value of 1.0 where it is not masked
makemask(mode='copy', inpimage=mom0+'.mask_chauv_bi',
inpmask=mom0+'.mask_chauv_bi:mask0', output=mom0mask)
#####################
# Build Moment 8 mask
#####################
cmd = 'rm -rf %s.mask*' % (mom8)
print("running: %s" % cmd)
os.system(cmd)
classicResult = imstat(mom8, listit=imstatListit, verbose=imstatVerbose)
if avoidExtremaInNoiseCalcForJointMask:
avoidLow8, avoidHigh8 = imagePercentileNoMask(mom8, [5,95])
mask = '"%s" > %.9f && "%s" < %.9f' % (mom8, avoidLow8, mom8, avoidHigh8)
print("running imstat('%s', algorithm='chauvenet', maxiter=5, mask='%s')" % (mom8,mask))
result = imstat(mom8, algorithm='chauvenet', maxiter=5, listit=imstatListit, verbose=imstatVerbose,
mask=mask)
else:
result = imstat(mom8, algorithm='chauvenet', maxiter=5, listit=imstatListit, verbose=imstatVerbose)
mom8peak = classicResult['max'][0]
myMAD8 = result['medabsdevmed'][0]
scaledMAD = result['medabsdevmed'][0]*1.4826
print("+++mom8 scaled MAD = %f" % (scaledMAD))
if scaledMAD <= 0:
scaledMAD = result['sigma'][0]
myMAD8 = scaledMAD / 1.4826
casalogPost("Using sigma = %g instead of scaled MAD, which was zero." % (scaledMAD))
mom8snr = classicResult['max'][0]/myMAD8
if False:
oneEventPoints = 0.05
if mom8minsnr == MOM8MINSNR_DEFAULT:
mom8sigma = np.max([mom8minsnr,oneEvent(nptsInCube//nchan,oneEventPoints)])
else:
mom8sigma = mom8minsnr
casalogPost("Choosing %f from mom8minsnr=%f, oneEvent=%f" % (mom8sigma, mom8minsnr, oneEvent(nptsInCube//nchan,oneEventPoints)))
else:
# Cycle 7
oneEventPoints = 1
if mom8minsnr == MOM8MINSNR_DEFAULT:
mom8sigma = np.max([mom8minsnr,oneEvent(nptsInCube,oneEventPoints)])
else:
mom8sigma = mom8minsnr
casalogPost("Choosing %f from mom8minsnr=%f, oneEvent=%f" % (mom8sigma, mom8minsnr, oneEvent(nptsInCube,oneEventPoints)))
mom8min = classicResult['min'][0]
mom8max = classicResult['max'][0]
mom8threshold = mom8sigma*scaledMAD + result['median'][0]
# the test against mom8max is to prevent infinite loop
casalogPost('++++++ Initial mom8sigma = %f, scaledMAD=%f, mom8median=%f, mom8min=%g, mom8max=%g, mom8threshold=%g' % (mom8sigma, scaledMAD, result['median'][0], mom8min, mom8max, mom8threshold))
while (mom8min >= mom8threshold and mom8threshold < mom8max):
# Then there will be no points that satisfy subsequent mask, so raise the threshold so that at least some points are considered to be signal-free
mom8sigma += 1
mom8threshold = mom8sigma*scaledMAD + result['median'][0]
casalogPost(" %e>%e: increasing mom8sigma to %d: %f" % (mom8min,mom8threshold,mom8sigma,mom8sigma*scaledMAD))
mask = '"%s" > %.9f || "%s" < -%.9f' % (mom8, mom8threshold, mom8, mom8threshold)
print("applying mask to mom8: ", mask)
# mom8.mask_chauv_bi: this image will have the true values of the image where it is not masked
imsubimage(mom8, mask=mask, outfile=mom8+'.mask_chauv_bi')
# mom8.mask_bi: this image will have the true values of the image where it is not masked
makemask(mode='copy', inpimage=mom8+'.mask_chauv_bi',
inpmask=mom8+'.mask_chauv_bi:mask0', output=mom8mask)
numberPixelsInMom8Mask = countPixelsAboveZero(mom8mask)
####################
# Build joint mask
####################
os.system('rm -rf %s' % (jointMask))
print("Running makemask(inpimage='%s', mode='copy', inpmask=['%s','%s'], output='%s')" % (mom0, mom0mask, mom8mask, jointMask))
makemask(inpimage=mom0, mode='copy', inpmask=[mom0mask, mom8mask],
output=jointMask)
if minbeamfrac > 0:
regionsPruned = pruneMask(jointMask, psfcube, minbeamfrac)
pixelsInMask = imstat(jointMask, listit=imstatListit, verbose=imstatVerbose)['max'][0] > 0.5
jointMask1 = jointMask
snr = np.max([mom0snr,mom8snr])
mom0snr2 = None
mom8snr2 = None
mom0snr3 = None
mom8snr3 = None
mom0sigma2 = None
mom8sigma2 = None
if os.path.exists(jointMask2):
# remove any old version to avoid confusion if phase2 is not run
os.system('rm -rf %s*' % (jointMask2))
if phase2 and snr > snrThreshold:
casalogPost("Doing phase 2 mask calculation because one or both SNR > %.0f (mom0=%f,mom8=%f)" % (snrThreshold,mom0snr,mom8snr))
if (overwritePhase2 or not os.path.exists(mom0mask2)
or not os.path.exists(mom8mask2)) and pixelsInMask:
####################################################################
# Recompute statistics using pixels outside the initial coarse mask
# because it will likely yield a lower MAD value.
#################################################
# Build Mom 0 mask
##################
os.system('rm -rf %s.mask2*' % (mom0))
print("running imstat('%s', mask='%s'<0.5)" % (mom0,jointMask))
classicResult = imstat(mom0, mask='"%s"<0.5'%jointMask, listit=imstatListit, verbose=imstatVerbose)
print("running imstat('%s', algorithm='chauvenet', maxiter=5, mask='%s'<0.5)" % (mom0,jointMask))
result = imstat(mom0, algorithm='chauvenet', maxiter=5, mask='"%s"<0.5'%jointMask, listit=imstatListit, verbose=imstatVerbose)
# Compute SNR (peak/MAD) outside of phase 1 mask
mom0snr2 = classicResult['max'][0]/result['medabsdevmed'][0]
scaledMAD = result['medabsdevmed'][0]*1.4826
resultPositive = imstat(mom0, algorithm='chauvenet', maxiter=5, mask='"%s"<0.5'%jointMask, listit=imstatListit, verbose=imstatVerbose)
if len(resultPositive['medabsdevmed']) == 0:
print("WARNING: zero-length results from mom0")
# reduce the sigma somewhat:
if False:
mom0sigma2 = mom0minsnr-1
else: # Cycle 7
if mom0minsnr == MOM0MINSNR_DEFAULT:
mom0sigma2 = np.max([mom0minsnr-1,oneEvent(nptsInCube,0.5)])
else:
mom0sigma2 = mom0minsnr-1
mom0threshold = mom0sigma2*scaledMAD + result['median'][0]
casalogPost('++++++ phase 2 mom0sigma2=%f, npts=%d, scaledMAD=%f, median=%f' % (mom0sigma2, result['npts'][0], scaledMAD, result['median'][0]))
casalogPost('++++++ phase 2 mom0threshold = %f' % mom0threshold)
if bidirectionalMaskPhase2:
mask = '"%s" > %.9f || "%s" < -%.9f' % (mom0, mom0threshold, mom0, mom0threshold)
else:
mask = '"%s" > %f' % (mom0, mom0threshold) # Cycle 6 release
imsubimage(mom0, mask=mask, outfile=mom0+'.mask2_chauv')
makemask(mode='copy', inpimage=mom0+'.mask2_chauv',
inpmask=mom0+'.mask2_chauv:mask0', output=mom0mask2)
##################
# Build Mom 8 mask
##################
cmd = 'rm -rf %s.mask2*' % (mom8)
print("Running: ", cmd)
os.system(cmd)
classicResult = imstat(mom8, mask='"%s"<0.5'%jointMask, listit=imstatListit, verbose=imstatVerbose)
print("Running imstat('%s',algorithm='chauvenet',maxiter=5,mask='\"%s\"<0.5, listit=%s, verbose=%s)" % (mom8, jointMask, imstatListit, imstatVerbose))
result = imstat(mom8, algorithm='chauvenet', maxiter=5, mask='"%s"<0.5'%jointMask, listit=imstatListit, verbose=imstatVerbose)
myMAD8 = result['medabsdevmed'][0]
if myMAD8 <= 0.0:
# mom8 images can potentially have a majority of identical 0, if immoments(mask) does not avoid 0.0
myMAD8 = result['sigma'][0]/1.4826
mom8snr2 = classicResult['max'][0]/myMAD8
if len(result['medabsdevmed']) == 0:
print("WARNING: zero-length results from mom8")
scaledMAD = result['medabsdevmed'][0]*1.4826
if mom8minsnr == MOM8MINSNR_DEFAULT:
mom8sigma2 = np.max([mom8minsnr,oneEvent(nptsInCube,0.5)])
# mom8sigma2 = np.max([mom8minsnr,oneEvent(nptsInCube//nchan,oneEventPoints)]) # was 0.5
else:
mom8sigma2 = mom8minsnr
mom8threshold = mom8sigma2*scaledMAD + result['median'][0]
casalogPost('++++++ phase 2 mom8sigma2=%f, npts=%d, scaledMAD=%f, median=%f' % (mom8sigma2, result['npts'][0], scaledMAD, result['median'][0]))
casalogPost('++++++ phase 2 mom8threshold = %f' % mom8threshold)
if bidirectionalMaskPhase2:
mask = '"%s" > %.9f || "%s" < -%.9f' % (mom8, mom8threshold, mom8, mom8threshold)
else:
mask = '"%s" > %f' % (mom8, mom8threshold) # Cycle 6 release
print("Running imsubimage('%s', mask='%s', outfile='%s')" % (mom8, mask, mom8+'.mask2_chauv'))
imsubimage(mom8, mask=mask, outfile=mom8+'.mask2_chauv')
print("Running makemask(mode='copy', inpimage='%s', inpmask='%s', output='%s')" % (mom8+'.mask2_chauv', mom8+'.mask2_chauv:mask0', mom8mask2))
myia = iatool()
myia.open(mom8+'.mask2_chauv')
if myia.maskhandler('default')[0] != '':
mom8mask2exists = True
else:
casalogPost('++++++ phase 2 mom8.mask2_chauv has no mask (no qualifying pixels).')
mom8mask2exists = False
myia.close()
if mom8mask2exists:
makemask(mode='copy', inpimage=mom8+'.mask2_chauv',
inpmask=mom8+'.mask2_chauv:mask0', output=mom8mask2)
inpmask = [mom0mask2, mom8mask2]
else:
inpmask = [mom0mask2]
##########################
# Build second joint mask
##########################
makemask(inpimage=mom0, mode='copy', inpmask=inpmask,
output=jointMask2)
if minbeamfrac > 0:
regionsPruned = pruneMask(jointMask2, psfcube, minbeamfrac)
jointMask = jointMask2
pixelsInMask = imstat(jointMask2, listit=imstatListit, verbose=imstatVerbose)['max'][0] > 0.5
# Compute SNR (peak/MAD) outside of phase 2 mask
classicResult = imstat(mom0, mask='"%s"<0.5'%jointMask, listit=imstatListit, verbose=imstatVerbose)
result = imstat(mom0, algorithm='chauvenet', maxiter=5, mask='"%s"<0.5'%jointMask, listit=imstatListit, verbose=imstatVerbose)
mom0snr3 = classicResult['max'][0]/result['medabsdevmed'][0]
classicResult = imstat(mom8, mask='"%s"<0.5'%jointMask, listit=imstatListit, verbose=imstatVerbose)
result = imstat(mom8, algorithm='chauvenet', maxiter=5, mask='"%s"<0.5'%jointMask, listit=imstatListit, verbose=imstatVerbose)
myMAD8 = result['medabsdevmed'][0]
if myMAD8 < 0.0:
# mom8 images can potentially have a majority of identical 0, if immoments(mask) does not avoid 0.0
myMAD8 = result['sigma'][0] / 1.4826
mom8snr3 = classicResult['max'][0] / myMAD8
else:
print("Not doing phase 2 because both SNR < %.0f (%f,%f)" % (snrThreshold,mom0snr,mom8snr))
os.system('rm -rf %s.mask2*' % (mom0))
os.system('rm -rf %s.mask2*' % (mom8))
if outdir == '':
meanSpectrumFile = cube+'.meanSpectrum.mom0mom8jointMask'
else:
meanSpectrumFile = os.path.join(outdir,os.path.basename(cube)+'.meanSpectrum.mom0mom8jointMask')
mom0snrs = [mom0snr, mom0snr2, mom0snr3]
mom8snrs = [mom8snr, mom8snr2, mom8snr3]
# if no pixels were found in the mask, then build one from PB annulus, or use whole image if no PB is available.
numberPixelsInMask = countPixelsAboveZero(jointMask)
if not pixelsInMask or numberPixelsInMask < minPixelsInJointMask:
pbBasedMask = True
os.system('rm -rf %s' % (jointMask))
myMask1chan = jointMask + '.1chan'
if pbcube is None:
casalogPost("No pb was passed into %s and .residual does not appear in cube name. Using whole image." % (__file__),debug=True)
# imsubimage(cube, chans='1', mask='"%s">-1000'%(cube), outfile=myMask1chan, overwrite=True)
immath(cube, chans='1', mode='evalexpr', expr='iif(IM0>-1000, 1.0, 0.0)', outfile=jointMask)
else:
casalogPost("Because less than %d pixels were in the joint mask, building pb-based mask from %s" % (minPixelsInJointMask,pbcube), debug=True)
lowerAnnulusLevel, higherAnnulusLevel = findOuterAnnulusForPBCube(pbcube, imstatListit, imstatVerbose)
imsubimage(pbcube, chans='1', mask='"%s">%f' % (pbcube,higherAnnulusLevel), outfile=myMask1chan, overwrite=True)
print("Done imsubimage, made ", myMask1chan)
makemask(mode='copy', inpimage=myMask1chan, overwrite=True,
inpmask=myMask1chan+':mask0', output=jointMask)
else:
classicResult = imstat(mom0, listit=imstatListit, verbose=imstatVerbose)
mom0peak = classicResult['max'][0]
result = imstat(mom0, algorithm='chauvenet', maxiter=5, listit=imstatListit, verbose=imstatVerbose)
mom0snr = classicResult['max'][0]/result['medabsdevmed'][0]
mom0snrs = [mom0snr,None,None]
mom0threshold = 0
classicResult = imstat(mom8, listit=imstatListit, verbose=imstatVerbose)
mom8peak = classicResult['max'][0]
myMAD8 = result['medabsdevmed'][0]
mom8snr = classicResult['max'][0]/myMAD8
mom8snrs = [mom8snr,None,None]
mom8threshold = 0
if os.path.exists(mom8mask):
numberPixelsInMom8Mask = countPixelsAboveZero(mom8mask)
elif os.path.exists(userJointMask):
numberPixelsInMom8Mask = countPixelsAboveZero(userJointMask)
casalogPost("Using userJointMask to set numberPixelsInMom8Mask = %d" % (numberPixelsInMom8Mask))
else:
numberPixelsInMom8Mask = 0
casalogPost("Neither the mom8mask nor the userJointMask exists. Setting numberPixelsInMom8Mask = %d" % (numberPixelsInMom8Mask))
if userJointMask != '':
if numberOfChannelsInCube(userJointMask) > 1:
jointMask = userJointMask + '.flattened'
if os.path.exists(jointMask):
print("Using existing flattened version of userJointMask")
else:
print("Running flattenMask('%s', '%s')" % (userJointMask, jointMask))
flattenMask(userJointMask, jointMask)
else:
jointMask = userJointMask
if outdir == '':
if userJointMask.find('.amendedJointMask') > 0: # special case name
meanSpectrumFile = cube+'.meanSpectrum.amendedJointMask'
else:
meanSpectrumFile = cube+'.meanSpectrum.userJointMask'
else:
if userJointMask.find('.amendedJointMask') > 0: # special case name
meanSpectrumFile = os.path.join(outdir,os.path.basename(cube)+'.meanSpectrum.amendedJointMask')
else:
meanSpectrumFile = os.path.join(outdir,os.path.basename(cube)+'.meanSpectrum.userJointMask')
# print("++++++++++++++++ Setting meanSpectrumFile = %s" % (meanSpectrumFile))
computeFirstSpectrum = False
if computeFirstSpectrum:
channels, frequency, intensity, normalized = computeStatisticalSpectrumFromMask(cube, jointMask1, pbcube, imageInfo, statistic, normalizeByMAD, projectCode, higherAnnulusLevel, lowerAnnulusLevel, outdir, jointMask)
writeMeanSpectrum(meanSpectrumFile+'_bidirectional', frequency, intensity,
intensity, mom0threshold,
nchan, numberPixelsInMask, mom8threshold, centralArcsec='mom0mom8jointMask',
mask=False, iteration=0)
channels, frequency, intensity, normalized = computeStatisticalSpectrumFromMask(cube, jointMask, pbcube, imageInfo, statistic, normalizeByMAD, projectCode, higherAnnulusLevel, lowerAnnulusLevel, outdir, jointMask)
numberPixelsInMask = countPixelsAboveZero(jointMask)
if MAD(intensity) == 0.0:
# Fix for CAS-11960
pbBasedMask = True
os.system('rm -rf %s' % (jointMask))
myMask1chan = jointMask + '.1chan'
casalogPost("Because less than %d pixels were in the joint mask, building pb-based mask from %s" % (minPixelsInJointMask,pbcube), debug=True)
lowerAnnulusLevel, higherAnnulusLevel = findOuterAnnulusForPBCube(pbcube, imstatListit, imstatVerbose)
imsubimage(pbcube, chans='1', mask='"%s">%f' % (pbcube,higherAnnulusLevel), outfile=myMask1chan, overwrite=True)
print("Done imsubimage, made ", myMask1chan)
makemask(mode='copy', inpimage=myMask1chan, overwrite=True,
inpmask=myMask1chan+':mask0', output=jointMask)
channels, frequency, intensity, normalized = computeStatisticalSpectrumFromMask(cube, jointMask, pbcube, imageInfo, statistic, normalizeByMAD, projectCode, higherAnnulusLevel, lowerAnnulusLevel, outdir, jointMask)
numberPixelsInMask = countPixelsAboveZero(jointMask)
initialQuadraticRemoved = False
initialQuadraticImprovementRatio = 1.0
elif not rmStatContQuadratic:
intensity, initialQuadraticRemoved, initialQuadraticImprovementRatio = removeInitialQuadraticIfNeeded(intensity,initialQuadraticImprovementThreshold)
else:
initialQuadraticRemoved = False
initialQuadraticImprovementRatio = 1.0
writeMeanSpectrum(meanSpectrumFile, frequency, intensity,
intensity, mom0threshold,
nchan, numberPixelsInMask, mom8threshold, centralArcsec='mom0mom8jointMask',
mask=initialQuadraticRemoved, iteration=0)
return intensity, normalized, numberPixelsInMask, pbBasedMask, initialQuadraticRemoved, initialQuadraticImprovementRatio, mom0snrs, mom8snrs, regionsPruned, numberPixelsInMom8Mask, mom0peak, mom8peak, jointMask
[docs]def locatePBCube(cube):
if cube.find('.residual') >= 0:
if os.path.islink(cube):
pbcube = os.readlink(cube).replace('.residual','.pb')
else:
pbcube = cube.replace('.residual','.pb')
if not os.path.exists(pbcube):
pbcube = None
elif cube.find('.image') >= 0:
# need this section for it to work on a clean cube instead of dirty cube
if os.path.islink(cube):
pbcube = os.readlink(cube).replace('.image','.pb')
else:
pbcube = cube.replace('.image','.pb')
if not os.path.exists(pbcube):
pbcube = None
else:
pbcube = None
return pbcube
[docs]def oneEvent(npts, events=1.0, positive=True, verbose=False):
"""
This function is called by meanSpectrumFromMom0Mom8JointMask.
For a specified size of a Gaussian population of data, compute the sigma
that gives just less than one event, by using scipy.special.erfinv.
Inputs:
positive: if True, then only considers positive events.
events: how many events to allow
verbose: passed to sigmaEvent
Return:
sigma: floating point value
"""
odds = events/float(npts)
if positive:
odds *= 2
sigma = (2**0.5)*(scipy.special.erfinv(1-odds))
return(sigma)
[docs]def findOuterAnnulusForPBCube(pbcube, imstatListit=False, imstatVerbose=False):
"""
Given a PB cube, finds the minimum sensitivity value, then computes the
corresponding higher value to form an annulus. Returns 0.2-0.3 for normal
images that have not been mitigated. The factor of 1.15 below will effectively
mimic a corresponding higher range. For example:
CASA <247>: au.gaussianBeamResponse(au.gaussianBeamOffset(0.5)/1.15, fwhm=1)
Out[247]: 0.59207684245045789
CASA <248>: au.gaussianBeamResponse(au.gaussianBeamOffset(0.8)/1.15, fwhm=1)
Out[248]: 0.8447381483786558
"""
lowerAnnulusLevel = imstat(pbcube, listit=imstatListit, verbose=imstatVerbose)['min'][0]
higherAnnulusLevel = gaussianBeamResponse(gaussianBeamOffset(lowerAnnulusLevel)/1.15, fwhm=1)
return np.max([0.21,lowerAnnulusLevel]), higherAnnulusLevel
[docs]def computeStatisticalSpectrumFromMask(cube, jointmask, pbimg=None, imageInfo=None,
statistic='mean', normalizeByMAD=False,
projectCode='', higherAnnulusLevel=None, lowerAnnulusLevel=None,
outdir='', jointMaskForNormalize=None):
"""
New heuristic for Cycle 6 pipeline. It is called by meanSpectrumFromMom0Mom8JointMask
Uses ia.getprofile to compute the mean spectrum of a cube within a
masked area.
jointmask: a 2D or 3D mask image indicating which pixels shall be used
or an expression with < or > along with a 2D or 3D mask image
jointMaskForNormalize: a 2D or 3D mask image to be used when normalizeByMAD==True
pbimg: the path to the primary beam of this cube, or a 2D primary beam with an expression
statistic: passed to ia.getprofile via the 'function' parameter
normalizeByMAD: if True, then create the inverse of the jointmask and
normalize the spectrum by the spectrum of 'xmadm' (scaled MAD)
computed on the inverse mask
Returns: three arrays: channels, freqs(Hz), intensities, and a Boolean
which says if normalization was applied
"""
chanInfo = numberOfChannelsInCube(cube, returnChannelWidth=True, returnFreqs=True)
nchan, firstFreq, lastFreq, channelWidth = chanInfo # freqs are in Hz
frequency = np.linspace(firstFreq, lastFreq, nchan) # lsrk
myia = iatool()
myia.open(cube)
casalogPost("ia.open('%s')" % (cube))
axis = findSpectralAxis(myia)
casalogPost("Using jointmask = %s" % (jointmask))
if jointmask == '' or jointmask.find('>') > 0 or jointmask.find('<') > 0:
# an expression was given
jointmaskQuoted = jointmask
else:
# a filename was given
jointmaskQuoted = '"'+jointmask+'">0'
if pbimg != '':
if pbimg is not None:
if pbimg.find('"') < 0:
# a plain pbimg was given
pbimgExpression = '"%s">0.21' % (pbimg)
else:
pbimgExpression = pbimg
jointmaskQuoted += ' && ' + pbimgExpression
casalogPost("Running ia.getprofile(axis=%d, function='%s', mask='%s', stretch=True)" % (axis, statistic, jointmaskQuoted))
casalogPost(" on the cube: %s" % (cube))
avgIntensity = myia.getprofile(axis=axis, function=statistic, mask=jointmaskQuoted, stretch=True)['values']
myia.close()
# Note: I tried putting this stage here, but a strong line at spw center (2016.1.00484.L)
# CS5-4 in spw21, will cause it to over-remove the quadratic, leaving a bowl which prevents
# identifying the line in the later stage, and puts it in the continuum selection.
# avgIntensity, initialQuadraticRemoved, initialQuadraticImprovementRatio = removeInitialQuadraticIfNeeded(avgIntensity,initialQuadraticImprovementThreshold)
normalized = False # start off by assuming it won't get normalized
if normalizeByMAD:
if pbimg is None:
casalogPost("pbimg is None, will not attempt to use the sensitivity annulus")
pbimgExists = False
elif not os.path.exists(pbimg.lstrip('"').split('"')[0]): # remove any > < expression
casalogPost("Could not find primary beam cube: %s" % (pbimg))
pbimgExists = False
else:
pbimgExists = True
if not pbimgExists:
if jointMaskForNormalize in ['',None]:
outsideMaskQuoted = ''
else:
casalogPost("Computing potential normalization spectrum from outside the joint mask (to remove atmospheric features). jointmask=%s." % (jointMaskForNormalize))
casalogPost("**** Number of unmasked pixels in jointmask = %s" % (countUnmaskedPixels(jointMaskForNormalize)))
if jointMaskForNormalize.find('.joint.') > 0:
outsideMask = jointMaskForNormalize.replace('.joint.','.inverseJoint.')
else:
outsideMask = jointMaskForNormalize + '.inverseJoint'
print("Generating outsideMask = %s" % (outsideMask))
mask = '"' + jointMaskForNormalize + '"<0.5'
print("Running imsubimage('%s', mask='%s', outfile='%s')" % (jointMaskForNormalize, mask, outsideMask))
os.system('rm -rf '+outsideMask)
imsubimage(jointMaskForNormalize, mask=mask, outfile=outsideMask)
print("**** Using unmasked pixels outside = ", countUnmaskedPixels(outsideMask))
outsideMaskQuoted = '"'+outsideMask+'"'
else:
# Use the annulus region from 0.21-0.3, unless image does not go out that far, in
# which case we use a comparable annulus that begins at a higher level
if higherAnnulusLevel is None:
pbimg = pbimg.lstrip('"').split('"')[0]
print("Running fc.findOuterAnnulusForPbcube('%s',False,False)" % (pbimg))
lowerAnnulusLevel, higherAnnulusLevel = findOuterAnnulusForPBCube(pbimg, imstatListit, imstatVerbose)
casalogPost("Computing potential normalization spectrum from MAD of an outer annulus <%.2f (to remove atmospheric features)." % (higherAnnulusLevel))
if jointMaskForNormalize not in ['', None]:
outsideMaskQuoted = '"%s">%g && "%s"<%g && "%s"<0.5'%(pbimg,lowerAnnulusLevel,pbimg,higherAnnulusLevel,jointMaskForNormalize)
else:
outsideMaskQuoted = '"%s">%g && "%s"<%g'%(pbimg,lowerAnnulusLevel,pbimg,higherAnnulusLevel)
myia.open(cube)
casalogPost("Running ia.getprofile(axis=%d, function='xmadm', mask='%s', stretch=True)" % (axis, outsideMaskQuoted))
xmadm = myia.getprofile(axis=axis, function='xmadm', mask=outsideMaskQuoted, stretch=True)['values']
# The following is only needed for debugging
if True:
avgIntensityOutsideMask = myia.getprofile(axis=axis, function='mean', mask=outsideMaskQuoted, stretch=True)['values']
else:
avgIntensityOutsideMask = avgIntensity*0.0
myia.close()
originalMAD = MAD(avgIntensity)
originalMedian = np.median(avgIntensity)
offset = np.max([0,-np.min(avgIntensity)])
avgIntensityOffset = avgIntensity + offset
avgIntensityNormToZero = np.max(avgIntensityOffset)/np.median(avgIntensityOffset) - 1 # this is a scalar
xmadmNormToZero = xmadm - np.min(xmadm) # this is a vector
normalizationFactor = 1 + xmadmNormToZero * avgIntensityNormToZero/np.max(xmadmNormToZero) # this is a vector
mymin = np.min(normalizationFactor)
mymax = np.max(normalizationFactor)
# 2018-04-05: avoid dividing by numbers near zero and thus inserting spikes
if mymin < 0.05*mymax:
normalizationFactor += 0.05*mymax - mymin # 1*np.median(normalizationFactor)
# if mymin < 0.5:
# normalizationFactor += 0.5-mymin
myMAD = MAD(normalizationFactor)
avgIntensityNormalized = avgIntensityOffset / normalizationFactor - offset
# Proposed modification 2018-04-05 to avoid producing spectra with negative medians:
myMedian = np.median(avgIntensityNormalized)
newMADestimate = MAD(avgIntensityNormalized) # could raise this to account for processing
avgIntensityNormalized = (avgIntensityNormalized-myMedian)*originalMAD/newMADestimate + originalMedian
# Here is where you would put writeXmadmFile()
# Here we use the highest common MAD because the number of pixels may
# be drastically different between the mask area and the
# outside-the-mask area. We only need to try to remove the effect
# atmospheric lines if they dominate the signal spectrum.
highestMAD = np.max([MAD(avgIntensity), MAD(xmadm)])
casalogPost("peak(avgIntensity)=%f, MAD(avgIntensity)=%f, MAD(xmadm)=%f" % (np.max(avgIntensity),MAD(avgIntensity),MAD(xmadm)),debug=True)
# This definition of peakOverMad will be high if there is strong continuum
# or if there is a strong line. Removing the median from the peak would
# eliminate the sensitivity to continuum emission.
peakOverMAD_signal = np.max(avgIntensity) / highestMAD
peakOverMAD_xmadm = np.max(xmadm) / highestMAD
applyNormalizationThreshold = 3.2 # was initially 3.5
if False:
# This method was attempted to try to resolve the 308 vs. 312, 355, 372
# discrepancy but caused too many other poor results.
peakOverMAD_signal = (np.max(avgIntensity)-np.median(avgIntensity)) / MAD(avgIntensity)
peakOverMAD_xmadm = (np.max(xmadm)-np.median(xmadm)) / MAD(xmadm)
applyNormalizationThreshold = 1.4
if tdmSpectrum(channelWidth,nchan):
applyNormalizationThreshold *= 4
projectCode = projectCode + ' '
if peakOverMAD_xmadm > peakOverMAD_signal/applyNormalizationThreshold:
avgIntensity = avgIntensityNormalized
casalogPost('%sApplying normalization because peak/MAD of xmadm spectrum %f > (peak/MAD of signal %.3f/%.2f=%.3f)' % (projectCode, peakOverMAD_xmadm,peakOverMAD_signal,applyNormalizationThreshold,peakOverMAD_signal/applyNormalizationThreshold))
normalized = True
else:
casalogPost('%sRejecting normalization because peak/MAD of xmadm spectrum %f <= (peak/MAD of signal %.3f/%.2f=%.3f)' % (projectCode, peakOverMAD_xmadm, peakOverMAD_signal, applyNormalizationThreshold, peakOverMAD_signal/applyNormalizationThreshold))
else:
casalogPost('Not-computing normalization because atmospheric variation considered too small')
channels = list(range(len(avgIntensity)))
if nchan != len(channels):
print("Discrepant number of channels!")
return np.array(channels), frequency, avgIntensity, normalized
[docs]def create_casa_quantity(myqatool,value,unit):
"""
This function is called by CalcAtmTransmissionForImage, frames, and lsrkToRest.
A wrapper to handle the changing ways in which casa quantities are invoked.
Todd Hunter
"""
if 'casac' in locals():
if (type(casac.Quantity) != type): # casa 4.x
myqa = myqatool.quantity(value, unit)
else: # casa 3.x
myqa = casac.Quantity(value, unit)
else:
# This is CASA 6 (same as 4.x and 5.x)
myqa = myqatool.quantity(value,unit)
return(myqa)
[docs]def MAD(a, c=0.6745, axis=0):
"""
This function is called by removeInitialQuadraticIfNeeded, findContinuumChannels,
meanSpectrum, computeStatisticalSpectrumFromMask, plotStatisticalSpectrumFromMask
and runFindContinuum.
Median Absolute Deviation along given axis of an array:
median(abs(a - median(a))) / c
c = 0.6745 is the constant to convert from MAD to std
"""
a = np.asarray(a, np.float64)
m = nanmedian(a, axis=axis,
preop=lambda x:
np.fabs(np.subtract(x, np.median(x, axis=None), out=x), out=x))
return m / c
[docs]def splitListIntoContiguousLists(mylist):
"""
This function is called by findContinuumChannels.
Called by copyweights. See also splitListIntoHomogenousLists.
Converts [1,2,3,5,6,7] into [[1,2,3],[5,6,7]], etc.
-Todd Hunter
"""
mylists = []
if (len(mylist) < 1):
return(mylists)
newlist = [mylist[0]]
for i in range(1,len(mylist)):
if (mylist[i-1] != mylist[i]-1):
mylists.append(newlist)
newlist = [mylist[i]]
else:
newlist.append(mylist[i])
mylists.append(newlist)
return(mylists)
[docs]def splitListIntoContiguousListsAndRejectZeroStd(channels, values, nanmin=None, verbose=False):
"""
This function is called by findContinuumChannels.
Takes a list of numerical values, splits into contiguous lists and
removes those that have zero standard deviation in their associated values.
Note that values *must* hold the values of the whole array, while channels
can be a subset.
If nanmin is specified, then reject lists that contain more than 3
appearances of this value.
"""
if verbose:
print("splitListIntoContiguousListsAndRejectZeroStd: len(values)=%d, len(channels)=%d" % (len(values), len(channels)))
values = np.array(values)
mylists = splitListIntoContiguousLists(channels)
channels = []
for i,mylist in enumerate(mylists):
mystd = np.std(values[mylist])
if (mystd > 1e-17): # avoid blocks of identically-zero values
if (nanmin is not None):
minvalues = len(np.where(values[i] == nanmin)[0])
if (float(minvalues)/len(mylist) > 0.1 and minvalues > 3):
print("Rejecting list %d with multiple min values (%d)" % (i,minvalues))
continue
channels += mylist
return(np.array(channels))
[docs]def convertChannelListIntoSelection(channels, trim=0, separator=';'):
"""
This function is called by findContinuumChannels.
Converts a list of channels into casa selection string syntax.
channels: a list of channels
trim: the number of channels to trim off of each edge of a block of channels
"""
selection = ''
firstList = True
if (len(channels) > 0):
mylists = splitListIntoContiguousLists(channels)
for mylist in mylists:
if (mylist[0]+trim <= mylist[-1]-trim):
if (not firstList):
selection += separator
selection += '%d~%d' % (mylist[0]+trim, mylist[-1]-trim)
firstList = False
return(selection)
[docs]def convertSelectionIntoChannelList(selection):
"""
Converts an arbitrary CASA selection string into an array of channel numbers
'0~1;4~6' -- > [0,4,5]
Ignores any initial spw number that preceeds a colon. Will fail if there is more than one colon.
"""
if selection.find(':') >= 0:
spw, selection = selection.split(':')
selections = selection.split(';')
mylist = []
for myrange in selections:
if myrange.find('~') > 0:
[c0,c1] = myrange.split('~')
mylist += range(int(c0),int(c1)+1)
else: # single value
mylist += [int(myrange)]
return np.array(mylist)
[docs]def CalcAtmTransmissionForImage(img, imageInfo, chanInfo='', airmass=1.5, pwv=-1,
spectralaxis=-1, value='transmission', P=-1, H=-1,
T=-1, altitude=-1):
"""
This function is called by atmosphereVariation.
Supported telescopes are VLA and ALMA (needed for default weather and PWV)
img: name of CASA image
value: 'transmission' or 'tsky'
chanInfo: a list containing nchan, firstFreqHz, lastFreqHz, channelWidthHz
pwv: in mm
P: in mbar
H: in percent
T: in Kelvin
Returns:
2 arrays: frequencies (in GHz) and values (Kelvin, or transmission: 0..1)
"""
if not os.path.isdir(img):
# Input was a spectrum rather than an image
print("chanInfo: ", chanInfo)
if (chanInfo[1] < 60e9):
telescopeName = 'ALMA'
else:
telescopeName = 'VLA'
else:
telescopeName = getTelescope(img)
freqs = np.linspace(chanInfo[1]*1e-9,chanInfo[2]*1e-9,chanInfo[0])
numchan = len(freqs)
lsrkwidth = (chanInfo[2] - chanInfo[1])/(numchan-1)
result = cubeLSRKToTopo(img, imageInfo, nchan=numchan, f0=chanInfo[1], f1=chanInfo[2], chanwidth=lsrkwidth)
if (result is None):
topofreqs = freqs
else:
topoWidth = (result[1]-result[0])/(numchan-1)
topofreqs = np.linspace(result[0], result[1], chanInfo[0]) * 1e-9
casalogPost("Converted LSRK range,width (%f-%f,%f) to TOPO (%f-%f,%f) over %d channels" % (chanInfo[1]*1e-9, chanInfo[2]*1e-9,lsrkwidth,topofreqs[0],topofreqs[-1],topoWidth,numchan))
P0 = 1000.0 # mbar
H0 = 20.0 # percent
T0 = 273.0 # Kelvin
if (telescopeName.find('ALMA') >= 0 or telescopeName.find('ACA') >= 0):
pwv0 = 1.0
P0 = 563.0
H0 = 20.0
T0 = 273.0
altitude0 = 5059
elif (telescopeName.find('VLA') >= 0):
P0 = 786.0
pwv0 = 5.0
altitude0 = 2124
else:
pwv0 = 10.0
altitude0 = 0
if (pwv < 0):
pwv = pwv0
if (T < 0):
T = T0
if (H < 0):
H = H0
if (P < 0):
P = P0
if (altitude < 0):
altitude = altitude0
tropical = 1
midLatitudeSummer = 2
midLatitudeWinter = 3
reffreq = 0.5*(topofreqs[int(numchan/2)-1]+topofreqs[int(numchan/2)])
# reffreq = np.mean(topofreqs)
numchanModel = numchan*1
chansepModel = (topofreqs[-1]-topofreqs[0])/(numchanModel-1)
nbands = 1
myqa = qatool()
fCenter = create_casa_quantity(myqa, reffreq, 'GHz')
fResolution = create_casa_quantity(myqa, chansepModel, 'GHz')
fWidth = create_casa_quantity(myqa, numchanModel*chansepModel, 'GHz')
myat = attool()
myat.initAtmProfile(humidity=H, temperature=create_casa_quantity(myqa,T,"K"),
altitude=create_casa_quantity(myqa,altitude,"m"),
pressure=create_casa_quantity(myqa,P,'mbar'),atmType=midLatitudeWinter)
myat.initSpectralWindow(nbands, fCenter, fWidth, fResolution)
myat.setUserWH2O(create_casa_quantity(myqa, pwv, 'mm'))
# myat.setAirMass() # This does not affect the opacity, but it does effect TebbSky, so do it manually.
myqa.done()
dry = np.array(myat.getDryOpacitySpec(0)[1])
wet = np.array(myat.getWetOpacitySpec(0)[1]['value'])
TebbSky = myat.getTebbSkySpec(spwid=0)[1]['value']
# readback the values to be sure they got set
if (myat.getRefFreq()['unit'] != 'GHz'):
casalogPost("There is a unit mismatch for refFreq in the atm code.")
if (myat.getChanSep()['unit'] != 'MHz'):
casalogPost("There is a unit mismatch for chanSep in the atm code.")
numchanModel = myat.getNumChan()
freq0 = myat.getChanFreq(0)['value']
freq1 = myat.getChanFreq(numchanModel-1)['value']
# We keep the original LSRK freqs for overlay on the LSRK spectrum, but associate
# the transmission values from the equivalent TOPO freqs
newfreqs = np.linspace(freqs[0], freqs[-1], numchanModel) # fix for SCOPS-4815
transmission = np.exp(-airmass*(wet+dry))
TebbSky *= (1-np.exp(-airmass*(wet+dry)))/(1-np.exp(-wet-dry))
if value=='transmission':
values = transmission
else:
values = TebbSky
del myat
return(newfreqs, values)
[docs]def mjdToUT(mjd, use_metool=True, prec=6):
"""
This function is called by getDateObs.
Converts an MJD value to a UT date and time string
such as '2012-03-14 00:00:00 UT'
use_metool: whether or not to use the CASA measures tool if running from CASA.
This parameter is simply for testing the non-casa calculation.
-Todd Hunter
"""
mjdsec = mjd*86400
utstring = mjdSecondsToMJDandUT(mjdsec, use_metool, prec=prec)[1]
return(utstring)
[docs]def mjdSecondsToMJDandUT(mjdsec, debug=False, prec=6, delimiter='-'):
"""
This function is called by mjdToUT.
Converts a value of MJD seconds into MJD, and into a UT date/time string.
prec: 6 means HH:MM:SS, 7 means HH:MM:SS.S
example: (56000.0, '2012-03-14 00:00:00 UT')
Caveat: only works for a scalar input value
"""
myme = metool()
today = myme.epoch('utc','today')
mjd = np.array(mjdsec) / 86400.
today['m0']['value'] = mjd
myqa = qatool()
hhmmss = myqa.time(today['m0'], prec=prec)[0]
date = myqa.splitdate(today['m0'])
myqa.done()
myme.done()
utstring = "%s%s%02d%s%02d %s UT" % (date['year'],delimiter,date['month'],delimiter,
date['monthday'],hhmmss)
return(mjd, utstring)
[docs]def cubeLSRKToTopo(img, imageInfo, freqrange='', prec=4, verbose=False,
nchan=None, f0=None, f1=None, chanwidth=None,
vis=''):
"""
This function is called by CalcAtmTransmissionForImage and writeContDat.
Reads the date of observation, central RA and Dec,
and observatory from an image cube and then calls lsrkToTopo to
return the specified frequency range in TOPO.
freqrange: desired range of frequencies (empty string or list = whole cube)
floating point list of two frequencies, or a delimited string
(delimiter = ',', '~' or space)
prec: in fractions of Hz (only used to display the value when verbose=True)
vis: read date of observation from the specified measurement set (instead of from img)
"""
if img != '':
if getFreqType(img).upper() == 'TOPO':
print("This cube is purportedly already in TOPO.")
return
if (nchan is None or f0 is None or f1 is None or chanwidth is None):
if img != '':
nchan,f0,f1,chanwidth = numberOfChannelsInCube(img, returnFreqs=True, returnChannelWidth=True)
if len(freqrange) == 0:
startFreq = f0
stopFreq = f1
elif (type(freqrange) == str):
if (freqrange.find(',') > 0):
freqrange = [parseFrequencyArgument(i) for i in freqrange.split(',')]
elif (freqrange.find('~') > 0):
freqrange = [parseFrequencyArgument(i) for i in freqrange.split('~')]
else:
freqrange = [parseFrequencyArgument(i) for i in freqrange.split()]
startFreq, stopFreq = freqrange
else:
startFreq, stopFreq = freqrange
ra,dec = rad2radec(imageInfo[9], imageInfo[10], delimiter=' ', verbose=False).split()
myia = iatool()
myia.open(img)
equinox = getEquinox(img,myia)
observatory = getTelescope(img,myia)
if vis == '':
datestring = getDateObs(img,myia)
else:
if not os.path.exists(vis):
print("Measurement set does not exist!")
return
datestring = getObservationStartDate(vis)
myia.close()
f0 = lsrkToTopo(startFreq, datestring, ra, dec, equinox, observatory, prec, verbose)
f1 = lsrkToTopo(stopFreq, datestring, ra, dec, equinox, observatory, prec, verbose)
return(np.array([f0,f1]))
[docs]def getObservationStartDate(vis, obsid=0, delimiter='-', measuresToolFormat=True):
"""
Uses the tb tool to read the start time of the observation and reports the date.
See also getObservationStartDay.
Returns: '2013-01-31 07:36:01 UT'
measuresToolFormat: if True, then return '2013/01/31/07:36:01', suitable for lsrkToTopo
-Todd Hunter
"""
mjdsec = getObservationStart(vis, obsid)
if (mjdsec is None):
return
obsdateString = mjdToUT(mjdsec/86400.)
if (delimiter != '-'):
obsdateString = obsdateString.replace('-', delimiter)
if measuresToolFormat:
return(obsdateString.replace(' UT','').replace(delimiter,'/').replace(' ','/'))
else:
return(obsdateString)
[docs]def rad2radec(ra=0,dec=0, prec=5, verbose=True, component=0,
replaceDecDotsWithColons=True, hmsdms=False, delimiter=', ',
prependEquinox=False, hmdm=False):
"""
This function is called by cubeLSRKToTopo.
Convert a position in RA/Dec from radians to sexagesimal string which
is comma-delimited, e.g. '20:10:49.01, +057:17:44.806'.
The position can either be entered as scalars via the 'ra' and 'dec'
parameters, as a tuple via the 'ra' parameter, or as an array of shape (2,1)
via the 'ra' parameter.
replaceDecDotsWithColons: replace dots with colons as the Declination d/m/s delimiter
hmsdms: produce output of format: '20h10m49.01s, +057d17m44.806s'
hmdm: produce output of format: '20h10m49.01, +057d17m44.806' (for simobserve)
delimiter: the character to use to delimit the RA and Dec strings output
prependEquinox: if True, insert "J2000" before coordinates (i.e. for clean or simobserve)
"""
if (type(ra) == tuple or type(ra) == list or type(ra) == np.ndarray):
if (len(ra) == 2):
dec = ra[1] # must come first before ra is redefined
ra = ra[0]
else:
ra = ra[0]
dec = dec[0]
if (np.shape(ra) == (2,1)):
dec = ra[1][0]
ra = ra[0][0]
myqa = qatool()
myra = myqa.formxxx('%.12frad'%ra,format='hms',prec=prec+1)
mydec = myqa.formxxx('%.12frad'%dec,format='dms',prec=prec-1)
if replaceDecDotsWithColons:
mydec = mydec.replace('.',':',2)
if (len(mydec.split(':')[0]) > 3):
mydec = mydec[0] + mydec[2:]
mystring = '%s, %s' % (myra, mydec)
myqa.done()
if (hmsdms):
mystring = convertColonDelimitersToHMSDMS(mystring)
if (prependEquinox):
mystring = "J2000 " + mystring
elif (hmdm):
mystring = convertColonDelimitersToHMSDMS(mystring, s=False)
if (prependEquinox):
mystring = "J2000 " + mystring
if (delimiter != ', '):
mystring = mystring.replace(', ', delimiter)
if (verbose):
print(mystring)
return(mystring)
[docs]def convertColonDelimitersToHMSDMS(mystring, s=True, usePeriodsForDeclination=False):
"""
This function is called by rad2radec.
Converts HH:MM:SS.SSS, +DD:MM:SS.SSS to HHhMMmSS.SSSs, +DDdMMmSS.SSSs
or HH:MM:SS.SSS +DD:MM:SS.SSS to HHhMMmSS.SSSs +DDdMMmSS.SSSs
or HH:MM:SS.SSS, +DD:MM:SS.SSS to HHhMMmSS.SSSs, +DD.MM.SS.SSS
or HH:MM:SS.SSS +DD:MM:SS.SSS to HHhMMmSS.SSSs +DD.MM.SS.SSS
s: whether or not to include the trailing 's' in both axes
"""
colons = len(mystring.split(':'))
if (colons < 5 and (mystring.strip().find(' ')>0 or mystring.find(',')>0)):
print("Insufficient number of colons (%d) to proceed (need 4)" % (colons-1))
return
if (usePeriodsForDeclination):
decdeg = '.'
decmin = '.'
decsec = ''
else:
decdeg = 'd'
decmin = 'm'
decsec = 's'
if (s):
outstring = mystring.strip(' ').replace(':','h',1).replace(':','m',1).replace(',','s,',1).replace(':',decdeg,1).replace(':',decmin,1) + decsec
if (',' not in mystring):
outstring = outstring.replace(' ', 's ',1)
else:
outstring = mystring.strip(' ').replace(':','h',1).replace(':','m',1).replace(':',decdeg,1).replace(':',decmin,1)
return(outstring)
[docs]def lsrkToTopo(lsrkFrequency, datestring, ra, dec, equinox='J2000',
observatory='ALMA', prec=4, verbose=False):
"""
This function is called by cubeLSRKToTopo.
Converts an LSRKfrequency and observing date/direction
to the corresponding frequency in the TOPO frame.
Inputs:
lsrkFrequency: floating point value in Hz or GHz, or a string with units
datestring: "YYYY/MM/DD/HH:MM:SS" (format = image header keyword 'date-obs')
ra: string "HH:MM:SS.SSSS"
dec: string "DD.MM.SS.SSSS" or "DD:MM:SS.SSSS" (colons will be replaced with .)
prec: only used to display the value when verbose=True
Returns: the TOPO frequency in Hz
"""
velocityLSRK = 0 # does not matter what it is, just needs to be same in both calls
restFreqHz = lsrkToRest(lsrkFrequency, velocityLSRK, datestring, ra, dec, equinox,
observatory, prec, verbose)
topoFrequencyHz = restToTopo(restFreqHz, velocityLSRK, datestring, ra, dec, equinox,
observatory, verbose=verbose)
return topoFrequencyHz
[docs]def lsrkToRest(lsrkFrequency, velocityLSRK, datestring, ra, dec,
equinox='J2000', observatory='ALMA', prec=4, verbose=True):
"""
This function is called by lsrkToTopo.
Converts an LSRK frequency, LSRK velocity, and observing date/direction
to the corresponding frequency in the rest frame.
Inputs:
lsrkFrequency: floating point value in Hz or GHz, or a string with units
velocityLSRK: floating point value in km/s
datestring: "YYYY/MM/DD/HH:MM:SS" (format = image header keyword 'date-obs')
ra: string "HH:MM:SS.SSSS"
dec: string "DD.MM.SS.SSSS" or "DD:MM:SS.SSSS" (colons will be replaced with .)
prec: only used to display the value when verbose=True
Returns: the Rest frequency in Hz
"""
if (dec.find(':') >= 0):
dec = dec.replace(':','.')
if verbose:
print("Warning: replacing colons with decimals in the dec field.")
freqGHz = parseFrequencyArgumentToGHz(lsrkFrequency)
myqa = qatool()
myme = metool()
velocityRadio = create_casa_quantity(myqa,velocityLSRK,"km/s")
position = myme.direction(equinox, ra, dec)
obstime = myme.epoch('TAI', datestring)
dopp = myme.doppler("RADIO",velocityRadio)
radialVelocityLSRK = myme.toradialvelocity("LSRK",dopp)
myme.doframe(position)
myme.doframe(myme.observatory(observatory))
myme.doframe(obstime)
rvelRad = myme.measure(radialVelocityLSRK,'LSRK')
doppRad = myme.todoppler('RADIO', rvelRad)
freqRad = myme.torestfrequency(myme.frequency('LSRK',str(freqGHz)+'GHz'), dopp)
myqa.done()
myme.done()
return freqRad['m0']['value']
[docs]def restToTopo(restFrequency, velocityLSRK, datestring, ra, dec, equinox='J2000',
observatory='ALMA', veltype='radio', verbose=False):
"""
This function is called by lsrkToTopo.
Converts a rest frequency, LSRK velocity, and observing date/direction
to the corresponding frequency in the TOPO frame.
Inputs:
restFrequency: floating point value in Hz or GHz, or a string with units
velocityLSRK: floating point value in km/s
datestring: "YYYY/MM/DD/HH:MM:SS"
ra: string "HH:MM:SS.SSSS"
dec: string "DD.MM.SS.SSSS" or "DD:MM:SS.SSSS" (colons will be replaced with .)
prec: only used to display the value when verbose=True
Returns: the TOPO frequency in Hz
"""
topoFreqHz, diff1, diff2 = frames(velocityLSRK, datestring, ra, dec, equinox,
observatory, verbose=verbose,
restFreq=restFrequency, veltype=veltype)
return topoFreqHz
[docs]def frames(velocity=286.7, datestring="2005/11/01/00:00:00",
ra="05:35:28.105", dec="-069.16.10.99", equinox="J2000",
observatory="ALMA", prec=4, verbose=True, myme='', myqa='',
restFreq=345.79599, veltype='optical'):
"""
This function is called by restToTopo.
Converts an optical velocity into barycentric, LSRK and TOPO frames.
Converts a radio LSRK velocity into TOPO frame.
Inputs:
velocity: in km/s
datestring: "YYYY/MM/DD/HH:MM:SS"
ra: "05:35:28.105"
dec: "-069.16.10.99"
equinox: "J2000"
observatory: "ALMA"
prec: precision to display (digits to the right of the decimal point)
veltype: 'radio' or 'optical'
restFreq: in Hz, GHz or a string with units
Returns:
* TOPO frequency in Hz
* difference between LSRK-TOPO in km/sec
* difference between LSRK-TOPO in Hz
"""
localme = False
localqa = False
if (myme == ''):
myme = metool()
localme = True
if (myqa == ''):
myqa = qatool()
localqa = True
if (dec.find(':') >= 0):
dec = dec.replace(':','.')
position = myme.direction(equinox, ra, dec)
obstime = myme.epoch('TAI', datestring)
if (veltype.lower().find('opt') == 0):
velOpt = create_casa_quantity(myqa,velocity,"km/s")
dopp = myme.doppler("OPTICAL",velOpt)
# CASA doesn't do Helio, but difference to Bary is hopefully small
rvelOpt = myme.toradialvelocity("BARY",dopp)
elif (veltype.lower().find('rad') == 0):
rvelOpt = myme.radialvelocity('LSRK',str(velocity)+'km/s')
else:
print("veltype must be 'rad'io or 'opt'ical")
return
myme.doframe(position)
myme.doframe(myme.observatory(observatory))
myme.doframe(obstime)
myme.showframe()
rvelRad = myme.measure(rvelOpt,'LSRK')
doppRad = myme.todoppler("RADIO",rvelRad)
restFreq = parseFrequencyArgumentToGHz(restFreq)
freqRad = myme.tofrequency('LSRK',doppRad, myme.frequency('rest',str(restFreq)+'GHz'))
myqa = qatool()
lsrk = myqa.tos(rvelRad['m0'],prec=prec)
rvelTop = myme.measure(rvelOpt,'TOPO')
doppTop = myme.todoppler("RADIO",rvelTop)
freqTop = myme.tofrequency('TOPO', doppTop, myme.frequency('rest',str(restFreq)+'GHz'))
if (localme):
myme.done()
topo = myqa.tos(rvelTop['m0'],prec=prec)
if (localqa):
myqa.done()
velocityDifference = 0.001*(rvelRad['m0']['value']-rvelTop['m0']['value'])
frequencyDifference = freqRad['m0']['value'] - freqTop['m0']['value']
return(freqTop['m0']['value'], velocityDifference, frequencyDifference)
[docs]def parseFrequencyArgumentToGHz(bandwidth):
"""
This function is called by frames and lsrkToRest.
Converts a frequency string into floating point in GHz, based on the units.
If the units are not present, then the value is assumed to be GHz if less
than 1000.
"""
value = parseFrequencyArgument(bandwidth)
if (value > 1000):
value *= 1e-9
return(value)
[docs]def parseFrequencyArgument(bandwidth):
"""
This function is called by parseFrequencyArgumentToGHz, topoFreqToChannel and cubeLSRKToTopo.
Converts a string frequency into floating point in Hz, based on the units.
If the units are not present, then the value is simply converted to float.
"""
bandwidth = str(bandwidth)
ghz = bandwidth.lower().find('ghz')
mhz = bandwidth.lower().find('mhz')
khz = bandwidth.lower().find('khz')
hz = bandwidth.lower().find('hz')
if (ghz>0):
bandwidth = 1e9*float(bandwidth[:ghz])
elif (mhz>0):
bandwidth = 1e6*float(bandwidth[:mhz])
elif (khz>0):
bandwidth = 1e3*float(bandwidth[:khz])
elif (hz>0):
bandwidth = float(bandwidth[:hz])
else:
bandwidth = float(bandwidth)
return(bandwidth)
[docs]def channelSelectionRangesToIndexArray(selection, separator=';'):
"""
This function is called by runFindContinuum.
Convert a channel selection range string to integer array of indices.
e.g.: '3~8;10~11' -> [3,4,5,6,7,8,10,11]
"""
index = []
for s in selection.split(separator):
a,b = s.split('~')
index += list(range(int(a), int(b)+1, 1))
return np.array(index)
[docs]def linfit(x, y, yerror, pinit=[0,0]):
"""
This function is called by atmosphereVariation in Cycle 4+5+6 and by
runFindContinuum in Cycle 4+5.
Basic linear function fitter with error bars in y-axis data points.
Uses scipy.optimize.leastsq(). Accepts either lists or arrays.
Example:
lf = au.linfit()
lf.linfit(x, y, yerror, pinit)
Input:
x, y: x and y axis values
yerror: uncertainty in the y-axis values (vector or scalar)
pinit contains the initial guess of [slope, intercept]
Output:
The fit result as: [slope, y-intercept]
"""
x = np.array(x)
y = np.array(y)
if (type(yerror) != list and type(yerror) != np.ndarray):
yerror = np.ones(len(x)) * yerror
fitfunc = lambda p, x: p[1] + p[0]*x
errfunc = lambda p,x,y,err: (y-fitfunc(p,x))/(err**2)
out = scipy.optimize.leastsq(errfunc, pinit, args=(x,y,yerror/y), full_output=1)
p = out[0]
covar = out[1]
return(p)
[docs]def polyfit(x, y, yerror, pinit=[0,0,0,0]):
"""
This function is called by removeInitialQuadraticIfNeeded in Cycle 6, and by
runFindContinuum in Cycle 4+5.
Basic second-order polynomial function fitter with error bars in y-axis data points.
Uses scipy.optimize.leastsq(). Accepts either lists or arrays.
Input:
x, y: x and y axis values
yerror: uncertainty in the y-axis values (vector or scalar)
pinit contains the initial guess of [slope, intercept]
y = order2coeff*(x-xoffset)**2 + slope*(x-xoffset) + y-intercept
Output:
The fit result as: [xoffset, order2coeff, slope, y-intercept]
"""
x = np.array(x)
y = np.array(y)
pinit[2] = np.mean(y)
pinit[3] = x[int(len(x)/2)]
if (type(yerror) != list and type(yerror) != np.ndarray):
yerror = np.ones(len(x)) * yerror
fitfunc = lambda p, x: p[2] + p[1]*(x-p[3]) + p[0]*(x-p[3])**2
errfunc = lambda p,x,y,err: (y-fitfunc(p,x))/(err**2)
out = scipy.optimize.leastsq(errfunc, pinit, args=(x,y,yerror/y), full_output=1)
p = out[0]
covar = out[1]
return(p)
[docs]def channelsInLargestGroup(selection):
"""
This function is called by runFindContinuum.
Returns the number of channels in the largest group of channels in a
CASA selection string.
"""
if (selection == ''):
return 0
ranges = selection.split(';')
largest = 0
for r in ranges:
c0 = int(r.split('~')[0])
c1 = int(r.split('~')[1])
channels = c1-c0+1
if (channels > largest):
largest = channels
return largest
[docs]def countChannelsInRanges(channels, separator=';'):
"""
This function is called by runFindContinuum.
Counts the number of channels in one spw of a CASA channel selection string
and return a list of numbers.
e.g. "5~20;30~40" yields [16,11]
"""
tokens = channels.split(separator)
count = []
for i in range(len(tokens)):
c0 = int(tokens[i].split('~')[0])
c1 = int(tokens[i].split('~')[1])
count.append(c1-c0+1)
return count
[docs]def countChannels(channels):
"""
This function is called by runFindContinuum.
Counts the number of channels in a CASA channel selection string.
If multiple spws are found, then it returns a dictionary of counts:
e.g. "1:5~20;30~40" yields 27; or '6~30' yields 25
"1:5~20;30~40,2:6~30" yields {1:27, 2:25}
"""
if (channels == ''):
return 0
tokens = channels.split(',')
nspw = len(tokens)
count = {}
for i in range(nspw):
string = tokens[i].split(':')
if (len(string) == 2):
spw,string = string
else:
string = string[0]
spw = 0
ranges = string.split(';')
for r in ranges:
c0 = int(r.split('~')[0])
c1 = int(r.split('~')[1])
if (c0 > c1):
casalogPost("Invalid channel range: c0 > c1 (%d > %d)" % (c0,c1))
return
channels = [1+int(r.split('~')[1])-int(r.split('~')[0]) for r in ranges]
count[spw] = np.sum(channels)
if (nspw == 1):
count = count[spw]
return(count)
[docs]def grep(filename, arg):
"""
This function is called only by maskArgumentMismatch and centralArcsecArgumentMismatch.
Runs grep for string <arg> in a subprocess and reports stdout and stderr.
"""
process = subprocess.Popen(['grep', '-n', arg, filename], stdout=subprocess.PIPE)
stdout, stderr = process.communicate()
return stdout, stderr
[docs]def topoFreqToChannel(freqlist, vis, spw, mymsmd=''):
"""
++++ This function is used by pipeline in writeContDat()
Converts a python floating point list of topocentric frequency ranges to
channel ranges in the specified ms.
freqlist: [150.45e9,151.67e9] or [150.45, 151.67]
spw: integer ID
vis: reads the channel frequencies from this measurement set for the specified spw
Returns: a python list of channels
"""
needToClose = False
if mymsmd == '':
needToClose = True
mymsmd = msmdtool()
mymsmd.open(vis)
chanfreqs = mymsmd.chanfreqs(spw)
width = np.abs(mymsmd.chanwidths(spw))[0]
if needToClose:
mymsmd.close()
if type(freqlist) != list and type(freqlist) != np.ndarray:
freqlist = [freqlist]
channels = []
for freq in freqlist:
freq = parseFrequencyArgument(freq)
f0 = np.min(chanfreqs) - 0.5*width
f1 = np.max(chanfreqs) + 0.5*width
if freq < f0 or freq > f1:
chanoff = np.min([np.abs(f0-freq),np.abs(freq-f1)]) / width
print("frequency %.6f GHz not within spw %d: %.6f - %.6f GHz (off by %.2f channels)" % (freq*1e-9, spw, f0*1e-9, f1*1e-9, chanoff))
if freq < f0:
freq = f0
elif freq > f1:
freq = f1
print("Setting frequency to last channel: %f" % (freq))
diffs = np.abs(chanfreqs-freq)
channels.append(np.argmin(diffs))
return channels
[docs]def topoFreqRangeListToChannel(contdatline='', vispath='./', spw=-1, freqlist='', vis='', mymsmd='',
returnFlatlist=False, writeChannelsInIncreasingOrder=True):
"""
++++ This function is used by pipeline in writeContDat()
Converts a semicolon-delimited string list of topocentric frequency ranges to
channel ranges in the specified ms.
contdatline: line cut-and-pasted from .dat file (e.g. 'my.ms 150.45~151.67GHz;151.45~152.67GHz')
vispath: set the path to the measurement set whose name was read from contdatline
freqlist: '150.45~151.67GHz;151.45~152.67GHz'
spw: integer ID or string ID
writeChannelsInIncreasingOrder: if True, then ensure that the list is in increasing order
Returns: a string like: '29:134~136;200~204'
if flatlist==True, then return [134,136,200,204]
"""
if contdatline == '' and freqlist == '':
print("Need to specify either contdatline or freqlist.")
return
if contdatline == '' and vis == '':
print("Need to specify either contdatline or vis.")
return
if contdatline != '':
vis, freqlist = contdatline.split()
if vispath != './':
vis = os.path.join(vispath,vis)
if (spw == -1 or spw == ''):
print("spw must be specified")
return
freqlist = freqlist.split(';')
chanlist = ''
myqa = qatool()
flatlist = []
spw = int(spw)
mymsmd = msmdtool()
mymsmd.open(vis)
for r in freqlist:
freqs = r.split('~')
if len(freqs) == 2:
myfreq = myqa.quantity(freqs[1])
f1 = myqa.convert(myfreq, 'Hz')['value']
unit = myfreq['unit']
f0 = myqa.convert(myqa.quantity(freqs[0]+unit), 'Hz')['value']
print("Running topoFreqToChannel([%f,%f], '%s', %d)" % (f0,f1,vis,spw))
chans = topoFreqToChannel([f0,f1], vis, spw, mymsmd)
chanlist += '%d~%d' % (np.min(chans), np.max(chans))
if r != freqlist[-1]:
chanlist += ';'
flatlist.append(np.min(chans))
flatlist.append(np.max(chans))
myqa.done()
mymsmd.close()
if returnFlatlist:
return flatlist
else:
if writeChannelsInIncreasingOrder:
cselections = chanlist.split(';')
if len(cselections) > 1:
if int(cselections[0].split('~')[0]) > int(cselections[1].split('~')[0]):
print("Reversing order of output channel list (due to lower sideband spw).")
cselections.reverse()
chanlist = ';'.join(cselections)
chanlist = '%d:' % (spw) + chanlist
return chanlist
[docs]def gaussianBeamOffset(response=0.5, fwhm=1.0):
"""
++++ This function is used by pipeline in computeStatisticalSpectrumFromMask().
Computes the radius at which the response of a Gaussian
beam drops to the specified level. For the inverse
function, see gaussianBeamResponse.
"""
if (response <= 0 or response > 1):
print("response must be between 0..1")
return
radius = (fwhm/2.3548)*(-2*np.log(response))**0.5
return(radius)
[docs]def gaussianBeamResponse(radius, fwhm):
"""
++++ This function is used by pipeline in computeStatisticalSpectrumFromMask().
Computes the gain at the specified radial offset from the center
of a Gaussian beam. For the inverse function, see gaussianBeamOffset.
Required inputs:
radius: in arcseconds
fwhm: in arcseconds
"""
sigma = fwhm/2.3548
gain = np.exp(-((radius/sigma)**2)*0.5)
return(gain)
[docs]def imageSNRAnnulus(mymomDiff, jointMaskTest, jointMaskTestWithAnnulus, applyMaskToAll=True):
# Need 2 calls to get peak anywhere outside the joint mask, but median from annulus, and MAD the lower of the two possibilities
ignore2, momDiffPeakOutside, ignore0, ignore1 = imageSNR(mymomDiff, mask=jointMaskTest,
useAnnulus=False, returnAllStats=True, applyMaskToAll=applyMaskToAll)
ignore1, ignore2, medianForSNR, madForSNR = imageSNR(mymomDiff, mask=jointMaskTest, maskWithAnnulus=jointMaskTestWithAnnulus,
useAnnulus=True, returnAllStats=True, applyMaskToAll=applyMaskToAll)
momDiffSNROutside = (momDiffPeakOutside - medianForSNR) / madForSNR
return momDiffSNROutside, momDiffPeakOutside
[docs]def imageSNR(img, axes=[], mask='', maskWithAnnulus='', useAnnulus=False, verbose=False, applyMaskToAll=False,
returnAllStats=True, returnFractionalDifference=False, usepb=''):
"""
Uses imstat (which automatically applies the internal mask, i.e avoids the black edges
of the image) to compute (max-median)/scaledMAD
axes: passed to imstat (use [0,1,2] to get a vector SNR per channel)
img: a 2D image or a cube
mask: additional mask image (or expression) to apply when computing everything but max
maskWithAnnulus: additional mask image (or expression) to use for median and MAD
if useAnnulus==True (but always take the smaller of the 2 possible MADs)
applyMaskToAll: also apply the provided mask when computing the max (default=False=anywhere)
returnAllStats: if True, then also return the peak, median and scaledMAD where the
peak has *not* had the median subtracted (in contrast to the SNR)
returnFractionalDifference: if True, then also return the fraction of negative pixels
usepb: (optional) name of a single-plane pb image to include in the mask (at >0.23 level)
-ToddHunter
"""
if usepb != '':
mask += ' && "%s">0.23' % (usepb)
stats = imstat(img, axes=axes, mask=mask, listit=imstatListit)
if useAnnulus:
statsAnnulus = imstat(img, axes=axes, mask=maskWithAnnulus, listit=imstatListit)
if not applyMaskToAll:
stats_nomask = imstat(img, listit=imstatListit, axes=axes)
# replace max with the max over the whole image
stats['max'] = stats_nomask['max']
if useAnnulus:
# use the median from the annulus
casalogPost("median = %f, median from annulus = %f; scMAD = %f, scMAD from annulus = %f" % (stats['median'],statsAnnulus['median'],stats['medabsdevmed']/.6745,statsAnnulus['medabsdevmed']/.6745))
stats['median'] = statsAnnulus['median']
# pick the lower of the two possible MADs; otherwise, do not use the rest of the statsAnnulus results
if statsAnnulus['medabsdevmed'] > 0 and stats['medabsdevmed'] > 0:
# keep it as an array of length 1
stats['medabsdevmed'] = np.array([np.min([statsAnnulus['medabsdevmed'], stats['medabsdevmed']])])
elif statsAnnulus['medabsdevmed'] > 0: # this means that stats['medabsdevmed'] is zero, so we replace it
stats['medabsdevmed'] = statsAnnulus['medabsdevmed']
if stats['medabsdevmed'] > 0:
snr = (stats['max']-stats['median'])/(stats['medabsdevmed']/.6745)
casalogPost('snr = (%f-%f)/%f = %f' % (stats['max'],stats['median'],stats['medabsdevmed']/.6745,snr))
else:
casalogPost('Because the MAD is zero, using the sigma instead in the calculation of SNR')
snr = (stats['max']-stats['median'])/stats['sigma']
casalogPost('snr = (%f-%f)/%f = %f' % (stats['max'],stats['median'],stats['sigma'],snr))
if type(snr) == list or type(snr) == np.ndarray:
if len(snr) == 1:
snr = snr[0]
mymax = stats['max'][0]
mymedian = stats['median'][0]
myscaledMAD = stats['medabsdevmed'][0]/.6745
if returnAllStats:
if returnFractionalDifference:
return snr, mymax, mymedian, myscaledMAD, fractionNegative
else:
return snr, mymax, mymedian, myscaledMAD
else:
return snr
################################################################################
# Functions below this point are not used by the Cycle 6 or 7 pipeline or PL2020
################################################################################
[docs]def meanSpectrum(img, nBaselineChannels=16, sigmaCube=3, verbose=False,
nanBufferChannels=2, useAbsoluteValue=False,
baselineMode='edge', percentile=20, continuumThreshold=None,
meanSpectrumFile='', centralArcsec=-1, imageInfo=[], chanInfo=[], mask='',
meanSpectrumMethod='peakOverRms', peakFilterFWHM=15, iteration=0,
applyMaskToMask=False, useIAGetProfile=True,
useThresholdWithMask=False, overwrite=False):
"""
This function is not used by Cycle 6 pipeline, but remains as manual user option.
Computes a threshold and then uses it to compute the average spectrum across a
CASA image cube, via the specified method.
Inputs:
nBaselineChannels: number of channels to use as the baseline in
the 'meanAboveThreshold' methods.
baselineMode: how to select the channels to use as the baseline:
'edge': use an equal number of channels on each edge of the spw
'min': use the percentile channels with the lowest absolute intensity
sigmaCube: multiply this value by the rms to get the threshold above which a pixel
is included in the mean spectrum
nanBufferChannels: when removing or replacing NaNs, do this many extra channels
beyond their actual extent
useAbsoluteValue: passed to avgOverCube
percentile: used with baselineMode='min'
continuumThreshold: if specified, only use pixels above this intensity level
meanSpectrumFile: name of ASCII file to produce to speed up future runs
centralArcsec: default=-1 means whole field, or a floating point value in arcsec
mask: a mask image (e.g. from clean); restrict calculations to pixels with mask=1
chanInfo: the list returned by numberOfChannelsInCube
meanSpectrumMethod: 'peakOverMad', 'peakOverRms', 'meanAboveThreshold',
'meanAboveThresholdOverRms', or 'meanAboveThresholdOverMad'
* 'meanAboveThreshold': uses a selection of baseline channels to compute the
rms to be used as a threshold value (similar to constructing a moment map).
* 'meanAboveThresholdOverRms/Mad': scale spectrum by RMS or MAD
* 'peakOverRms/Mad' computes the ratio of the peak in a spatially-smoothed
version of cube to the RMS or MAD. Smoothing is set by peakFilterFWHM.
peakFilterFWHM: value used by 'peakOverRms' and 'peakOverMad' to presmooth
each plane with a Gaussian kernel of this width (in pixels)
Set to 1 to not do any smoothing.
useIAGetProfile: if True, then for peakOverMad, or meanAboveThreshold with
baselineMode='min', then use ia.getprofile instead of ia.getregion
and the subsequent arithmetic (because it should be faster)
useThresholdWithMask: if False, then just take mean rather than meanAboveThreshold
when a mask has been specified
Returns 8 items:
* avgspectrum (vector)
* avgspectrumAboveThresholdNansRemoved (vector)
* avgspectrumAboveThresholdNansReplaced (vector)
* threshold (scalar)
* edgesUsed: 0=lower, 1=upper, 2=both
* nchan (scalar)
* nanmin (the value used to replace NaNs)
* percentagePixelsNotMasked
"""
if meanSpectrumFile == '':
meanSpectrumFile = img + '.meanSpectrum.' + meanSpectrumMethod
if meanSpectrumMethod == 'auto':
print("Invalid meanSpectrumMethod: cannot be 'auto' at this point.")
return
if (not os.path.exists(img)):
casalogPost("Could not find image = %s" % (img))
return
myia = iatool()
usermaskdata = ''
if (len(mask) > 0):
# This is the user-specified mask (not the minpb mask inside the cube).
myia.open(mask)
maskAxis = findSpectralAxis(myia)
usermaskShape = myia.shape()
if useIAGetProfile:
myshape = myia.shape()
if myshape[maskAxis] > 1:
singlePlaneUserMask = False
else:
singlePlaneUserMask = True
else:
print("Calling myia.getregion(axes=2)")
usermaskdata = myia.getregion(axes=2)
if (verbose): print("shape(usermaskdata) = ", np.array(np.shape(usermaskdata)))
if applyMaskToMask:
usermaskmask = myia.getregion(getmask=True)
idx = np.where(usermaskmask==False)
if len(idx) > 0:
casalogPost('applyMaskToMask has zeroed out %d pixels.' % (len(idx[0])))
usermaskdata[idx] = 0
if (np.shape(usermaskdata)[maskAxis] > 1):
singlePlaneUserMask = False
else:
singlePlaneUserMask = True
if singlePlaneUserMask:
if (meanSpectrumMethod.find('meanAboveThreshold') >= 0):
casalogPost("single plane user masks are not supported by meanSpectrumMethod='meanAboveThreshold', try peakOverMad.")
myia.close()
return
myia.close()
myia.open(img)
axis = findSpectralAxis(myia)
nchan = myia.shape()[axis]
if verbose: print("Found spectral axis = ", axis)
if len(imageInfo) == 0:
imageInfo = getImageInfo(img)
myrg = None
if True:
myrg = rgtool()
if (centralArcsec < 0 or centralArcsec == 'auto' or useIAGetProfile):
if not useIAGetProfile:
centralArcsec = -1
if ((len(mask) > 0 or meanSpectrumMethod != 'peakOverMad') and not useIAGetProfile):
print("Running ia.getregion() useIAGetProfile=", useIAGetProfile)
pixels = myia.getregion()
print("Running ia.getregion(getmask=True)")
maskdata = myia.getregion(getmask=True)
blc = [0,0,0,0]
trc = [myia.shape()[0]-1, myia.shape()[1]-1, 0, 0]
else:
bmaj, bmin, bpa, cdelt1, cdelt2, naxis1, naxis2, freq, imgShape, crval1, crval2, maxBaseline = imageInfo
nchan = chanInfo[0]
x0 = int(round_half_up(naxis1*0.5 - centralArcsec*0.5/np.abs(cdelt1)))
x1 = int(round_half_up(naxis1*0.5 + centralArcsec*0.5/np.abs(cdelt1)))
y0 = int(round_half_up(naxis2*0.5 - centralArcsec*0.5/cdelt2))
y1 = int(round_half_up(naxis2*0.5 + centralArcsec*0.5/cdelt2))
# avoid going off the edge of non-square images
if (x0 < 0): x0 = 0
if (y0 < 0): y0 = 0
if (x0 >= naxis1): x0 = naxis1 - 1
if (y0 >= naxis2): y0 = naxis2 - 1
blc = [x0,y0,0,0]
trc = [x1,y1,0,0]
trc[axis] = nchan
region = myrg.box(blc=blc, trc=trc)
print("Running ia.getregion(region=region)")
pixels = myia.getregion(region=region)
print("Running ia.getregion(region=region, getmask=True)")
maskdata = myia.getregion(region=region,getmask=True)
if (len(mask) > 0):
casalogPost("Taking submask for central area of image: blc=%s, trc=%s" % (str(blc),str(trc)))
usermaskdata = submask(usermaskdata, region)
if verbose:
print("shape of pixels = ", np.array(np.shape(pixels)))
if len(mask) > 0:
if useIAGetProfile:
if not (myia.shape() == usermaskShape).all():
casalogPost("Mismatch in shape between image (%s) and mask (%s)" % (myia.shape(), usermaskShape))
if myrg is not None: myrg.done()
return
else:
# in principle, the 'if' branch could be used for both cases, but it is not yet tested so we keep the old method.
if not (np.array(np.shape(pixels)) == np.array(np.shape(usermaskdata))).all():
casalogPost("Mismatch in shape between image (%s) and mask (%s)" % (np.shape(pixels),np.shape(usermaskdata)))
if myrg is not None: myrg.done()
return
if ((casaVersionCompare('<','5.3.0-22') or not useIAGetProfile) and
(meanSpectrumMethod.find('OverRms') > 0 or meanSpectrumMethod.find('OverMad') > 0)):
# compute myrms or mymad, ignoring masked values and usermasked values
if (meanSpectrumMethod.find('OverMad') < 0):
casalogPost("Computing std on each plane")
else:
casalogPost("Computing mad on each plane")
myvalue = []
for a in range(nchan):
if ((a+1)%100 == 0):
print("Done %d/%d" % (a+1, nchan))
# Extract this one channel
if (axis == 2):
if len(mask) > 0:
mypixels = pixels[:,:,a,0]
mymask = maskdata[:,:,a,0]
if (singlePlaneUserMask):
myusermask = usermaskdata[:,:,0,0]
else:
myusermask = usermaskdata[:,:,a,0]
else:
blc[axis] = a
trc[axis] = a
myregion = myrg.box(blc=blc,trc=trc)
mypixels = myia.getregion(region=myregion)
mymask = myia.getregion(region=myregion,getmask=True)
elif (axis == 3):
if (len(mask) > 0):
mypixels = pixels[:,:,0,a]
mymask = maskdata[:,:,0,a]
if (singlePlaneUserMask):
myusermask = usermaskdata[:,:,0,0]
else:
myusermask = usermaskdata[:,:,0,a]
else:
blc[axis] = a
trc[axis] = a
myregion = myrg.box(blc=blc,trc=trc)
mypixels = myia.getregion(region=myregion)
mymask = myia.getregion(region=myregion,getmask=True)
if (len(mask) > 0):
# user mask is typically a clean mask, so we want to use the region outside the
# clean mask for computing the MAD, but also avoiding the masked edges of the image,
# which are generally masked to False
pixelsForStd = mypixels[np.where((myusermask<1) * (mymask==True))]
else:
# avoid the masked (typically outer) edges of the image using the built-in mask
pixelsForStd = mypixels[np.where(mymask==True)]
if (meanSpectrumMethod.find('OverMad') < 0):
myvalue.append(np.std(pixelsForStd))
else:
myvalue.append(MAD(pixelsForStd))
if (meanSpectrumMethod.find('OverMad') < 0):
myrms = np.array(myvalue)
else:
mymad = np.array(myvalue)
percentagePixelsNotMasked = 100
threshold = 0 # perhaps it should be None, but this would require modification to print statement in writeMeanSpectrum
edgesUsed = 0
if (meanSpectrumMethod.find('peakOver') == 0):
mybox = '%d,%d,%d,%d' % (blc[0],blc[1],trc[0],trc[1])
threshold = 0
edgesUsed = 0
gaussianSigma = peakFilterFWHM/(2*np.sqrt(2*np.log(2))) # 2.355
if casaVersionCompare('>=','5.3.0-22') and useIAGetProfile:
if (gaussianSigma > 1.1/(2*np.sqrt(2*np.log(2)))):
myia.close()
smoothimg = img+'.imsmooth'
if not os.path.exists(smoothimg):
casalogPost('Creating smoothed cube for extracting peak/mad over box=%s'%(mybox))
imsmooth(imagename=img, kernel='gauss', major='%fpix'%(peakFilterFWHM),
minor='%fpix'%(peakFilterFWHM), pa='0deg',
box=mybox, outfile=smoothimg)
else:
casalogPost('Using existing smoothed cube')
smoothia = iatool()
smoothia.open(smoothimg)
avgspectrum = smoothia.getprofile(axis=axis, function='max/xmadm', mask=mask)['values']
# Old method, before ratios were implemented in toolkit:
# mymax = smoothia.getprofile(axis=axis, function='max', mask=mask)['values']
# mymad = smoothia.getprofile(axis=axis, function='xmadm', mask=mask)['values']
# avgspectrum = mymax / mymad
smoothia.close()
else:
avgspectrum = myia.getprofile(axis=axis, function='max/xmadm', mask=mask)['values']
mymax = myia.getprofile(axis=axis, function='max', mask=mask)['values']
mymad = myia.getprofile(axis=axis, function='xmadm', mask=mask)['values']
else:
# compute mymax (an array of channel maxima), then divide by either myrms or mymad array
# which already exist from above.
myvalue = []
casalogPost("Smoothing and computing peak on each plane over box=%s." % mybox)
if (len(mask) > 0):
pixels[np.where(usermaskdata==0)] = np.nan
for a in range(nchan):
if ((a+1)%100 == 0):
print("Done %d/%d" % (a+1, nchan))
if (axis == 2):
if len(mask) > 0:
mypixels = pixels[:,:,a,0]
else:
blc[axis] = a
trc[axis] = a
myregion = myrg.box(blc=blc,trc=trc)
mypixels = myia.getregion(region=myregion)
elif (axis == 3):
if len(mask) > 0:
mypixels = pixels[:,:,0,a]
else:
blc[axis] = a
trc[axis] = a
myregion = myrg.box(blc=blc,trc=trc)
mypixels = myia.getregion(region=myregion)
if (gaussianSigma > 1.1/(2*np.sqrt(2*np.log(2)))):
if (len(mask) > 0):
# taken from stackoverflow.com/questions/18697532/gaussian-filtering-a-image-with-nan-in-python
V = mypixels.copy()
V[mypixels!=mypixels] = 0
VV = gaussian_filter(V,sigma=gaussianSigma)
W = 0*mypixels.copy() + 1 # the lack of a zero here was a long-standing bug found on Oct 12, 2017
W[mypixels!=mypixels] = 0
WW = gaussian_filter(W,sigma=gaussianSigma)
mypixels = VV/WW
myvalue.append(np.nanmax(mypixels))
else:
myvalue.append(np.nanmax(gaussian_filter(mypixels,sigma=gaussianSigma)))
else:
myvalue.append(np.nanmax(mypixels))
print("finished")
mymax = np.array(myvalue)
if (meanSpectrumMethod == 'peakOverRms'):
avgspectrum = mymax/myrms
elif (meanSpectrumMethod == 'peakOverMad'):
avgspectrum = mymax/mymad
nansRemoved = removeNaNs(avgspectrum, verbose=True)
nansReplaced,nanmin = removeNaNs(avgspectrum, replaceWithMin=True,
nanBufferChannels=nanBufferChannels, verbose=True)
elif (meanSpectrumMethod.find('meanAboveThreshold') == 0):
if (continuumThreshold is not None):
belowThreshold = np.where(pixels < continuumThreshold)
if verbose:
print("shape of belowThreshold = ", np.shape(belowThreshold))
pixels[belowThreshold] = 0.0
naxes = len(myia.shape()) # len(np.shape(pixels))
nchan = myia.shape()[axis] # np.shape(pixels)[axis]
if (baselineMode == 'edge'): # not currently used by pipeline
# Method #1: Use the two edges of the spw to find the line-free rms of the spectrum
nEdgeChannels = nBaselineChannels/2
# lower edge
blc = np.zeros(naxes)
trc = [i-1 for i in list(np.shape(pixels))]
trc[axis] = nEdgeChannels
myrg = rgtool()
region = myrg.box(blc=blc, trc=trc)
print("Running ia.getregion(blc=%s,trc=%s)" % (blc,trc))
lowerEdgePixels = myia.getregion(region=region)
# drop all floating point zeros (which will drop pixels outside the mosaic image mask)
lowerEdgePixels = lowerEdgePixels[np.where(lowerEdgePixels!=0.0)]
stdLowerEdge = MAD(lowerEdgePixels)
medianLowerEdge = nanmedian(lowerEdgePixels)
if verbose: print("MAD of %d channels on lower edge = %f" % (nBaselineChannels, stdLowerEdge))
# upper edge
blc = np.zeros(naxes)
trc = [i-1 for i in list(np.shape(pixels))]
blc[axis] = trc[axis] - nEdgeChannels
region = myrg.box(blc=blc, trc=trc)
print("Running ia.getregion(blc=%s,trc=%s)" % (blc,trc))
upperEdgePixels = myia.getregion(region=region)
# myrg.done()
# drop all floating point zeros
upperEdgePixels = upperEdgePixels[np.where(upperEdgePixels!=0.0)]
stdUpperEdge = MAD(upperEdgePixels)
medianUpperEdge = nanmedian(upperEdgePixels)
casalogPost("meanSpectrum(): edge medians: lower=%.10f, upper=%.10f" % (medianLowerEdge, medianUpperEdge))
if verbose:
print("MAD of %d channels on upper edge = %f" % (nEdgeChannels, stdUpperEdge))
if (stdLowerEdge <= 0.0):
edgesUsed = 1
stdEdge = stdUpperEdge
medianEdge = medianUpperEdge
elif (stdUpperEdge <= 0.0):
edgesUsed = 0
stdEdge = stdLowerEdge
medianEdge = medianLowerEdge
else:
edgesUsed = 2
stdEdge = np.mean([stdLowerEdge,stdUpperEdge])
medianEdge = np.mean([medianLowerEdge,medianUpperEdge])
if (baselineMode != 'edge'): # of the meanAboveThreshold case
# Method #2: pick the N channels with the lowest absolute values (to avoid
# confusion from absorption lines and negative bowls of missing flux)
# However, for the case of significant absorption lines, this only
# works if the continuum has been subtracted, but in
# the pipeline case, it has not been. It should probably be changed
# to first determine if the median is closer to the min or the max,
# and if the latter, then use the maximum absolute values.
if useIAGetProfile:
if mask == '' or useThresholdWithMask:
if mask == '':
maskArgument = ''
else:
maskArgument = "'%s'>0" % (mask)
profile = myia.getprofile(axis=axis, function='min', mask=maskArgument)
absPixels = np.abs(profile['values'])
# Find the lowest pixel values
percentileThreshold = scoreatpercentile(absPixels, percentile)
idx = np.where(absPixels < percentileThreshold)
# Take their statistics
# In the original implementation, the percentile min is computed over all pixels, but
# here I take aggregated over all spatial pixels into a spectral profile, because that
# is what is available from getprofile.
stdMin = MAD(absPixels[idx])
medianMin = np.median(absPixels[idx])
else:
if (centralArcsec < 0):
print("Running ia.getregion")
allPixels = myia.getregion()
else:
allPixels = pixels
# Convert all NaNs to zero
allPixels[np.isnan(allPixels)] = 0
# Drop all floating point zeros and internally-masked pixels from calculation
if (mask == ''):
allPixels = allPixels[np.where((allPixels != 0) * (maskdata==True))]
else:
# avoid identical zeros and clean mask when looking for lowest pixels
allPixels = allPixels[np.where((allPixels != 0) * (maskdata==True) * (usermaskdata<1))]
# Take absolute value
absPixels = np.abs(allPixels)
# Find the lowest pixel values by percentile
percentileThreshold = scoreatpercentile(absPixels, percentile)
idx = np.where(absPixels < percentileThreshold)
# Take their statistics
stdMin = MAD(allPixels[idx])
medianMin = nanmedian(allPixels[idx])
if (baselineMode == 'edge'):
std = stdEdge
median = medianEdge
casalogPost("meanSpectrum(): edge mode: median=%f MAD=%f threshold=%f (edgesUsed=%d)" % (medianEdge, stdEdge, medianEdge+stdEdge*sigmaCube, edgesUsed))
threshold = median + sigmaCube*std
elif (not useIAGetProfile or mask == '' or useThresholdWithMask):
std = stdMin
median = medianMin
edgesUsed = 0
casalogPost("**** meanSpectrum(useIAGetProfile=%s): min mode: median=%f MAD=%f threshold=%f" % (useIAGetProfile, medianMin, stdMin, medianMin+stdMin*sigmaCube))
threshold = median + sigmaCube*std
if useIAGetProfile:
if mask != '':
maskArgument = "'%s'>0" % mask
else:
maskArgument = ''
casalogPost("running ia.getprofile(mean, mask='%s')" % maskArgument)
profile = myia.getprofile(axis=axis, function='mean', mask=maskArgument)
avgspectrum = profile['values']
percentagePixelsNotMasked = sum(profile['npix'])*100. / np.prod(myia.shape())
else:
if (axis == 2 and naxes == 4):
# drop the degenerate axis so that avgOverCube will work with nanmean(axis=0)
pixels = pixels[:,:,:,0]
if (len(mask) > 0):
maskdata = propagateMaskToAllChannels(maskdata, axis)
else:
maskdata = ''
avgspectrum, percentagePixelsNotMasked = avgOverCube(pixels, useAbsoluteValue, mask=maskdata, usermask=usermaskdata)
if meanSpectrumMethod.find('OverRms') > 0:
avgspectrum /= myrms
elif meanSpectrumMethod.find('OverMad') > 0:
avgspectrum /= mymad
if useIAGetProfile:
if mask == '':
casalogPost("running ia.getprofile(mean above threshold: mask='%s>%f')" % (img, threshold))
profile = myia.getprofile(axis=axis, function='mean', mask="'%s'>%f"%(img,threshold))
else:
if useThresholdWithMask:
profile = myia.getprofile(axis=axis, function='mean', mask="'%s'>0 && '%s'>%f"%(mask,img,threshold))
else:
# we just keep the prior result for profile which did not use a threshold
pass
avgspectrumAboveThreshold = profile['values']
percentagePixelsNotMasked = sum(profile['npix'])*100. / np.prod(myia.shape())
else:
casalogPost("Using threshold above which to compute mean spectrum = %f" % (threshold), debug=True)
pixels[np.where(pixels < threshold)] = 0.0
casalogPost("Running avgOverCube")
avgspectrumAboveThreshold, percentagePixelsNotMasked = avgOverCube(pixels, useAbsoluteValue, threshold, mask=maskdata, usermask=usermaskdata)
if meanSpectrumMethod.find('OverRms') > 0:
avgspectrumAboveThreshold /= myrms
elif meanSpectrumMethod.find('OverMad') > 0:
avgspectrumAboveThreshold /= mymad
if useIAGetProfile:
nansRemoved = removeZeros(avgspectrumAboveThreshold)
nansReplaced = nansRemoved
nanmin = 0
else:
if verbose:
print("Running removeNaNs (len(avgspectrumAboveThreshold)=%d)" % (len(avgspectrumAboveThreshold)))
nansRemoved = removeNaNs(avgspectrumAboveThreshold)
nansReplaced,nanmin = removeNaNs(avgspectrumAboveThreshold, replaceWithMin=True,
nanBufferChannels=nanBufferChannels)
myia.close()
if len(chanInfo) == 0:
chanInfo = numberOfChannelsInCube(img, returnChannelWidth=True, returnFreqs=True)
nchan, firstFreq, lastFreq, channelWidth = chanInfo
frequency = np.linspace(firstFreq, lastFreq, nchan)
writeMeanSpectrum(meanSpectrumFile, frequency, avgspectrum, nansReplaced, threshold,
nchan, edgesUsed, nanmin, centralArcsec, mask, iteration)
if (myrg is not None): myrg.done()
return(avgspectrum, nansRemoved, nansReplaced, threshold,
edgesUsed, nchan, nanmin, percentagePixelsNotMasked)
[docs]def getMaxpixpos(img):
"""
This function is called by findContinuum, but only in Cycle 4+5 heuristic.
"""
myia = iatool()
myia.open(img)
maxpixpos = myia.statistics()['maxpos']
myia.close()
return maxpixpos
[docs]def submask(mask, region):
"""
This function is called by meanSpectrum, but only in Cycle 4+5 heuristic.
Takes a spectral mask array, and picks out a subcube defined by a
blc,trc-defined region
region: dictionary containing keys 'blc' and 'trc'
"""
mask = mask[region['blc'][0]:region['trc'][0]+1, region['blc'][1]:region['trc'][1]+1]
return mask
[docs]def avgOverCube(pixels, useAbsoluteValue=False, threshold=None, median=False,
mask='', usermask='', useIAGetProfile=True):
"""
This function was used by Cycle 4 and 5 pipeline from meanSpectrum(),
but will not be used in the Cycle 6 default heuristic of
'mom0mom8jointMask'.
Computes the average spectrum across a multi-dimensional
array read from an image cube, ignoring any NaN values.
Inputs:
pixels: a 3D array (with degenerate Stokes axis dropped)
threshold: value in Jy
median: if True, compute the median instead of the mean
mask: use pixels inside this spatial mask (i.e. with value==1 or True) to compute
the average (the spectral mask is taken as the union of all channels)
This is the mask located inside the image cube specified in findContinuum(img='')
usermask: this array results from the mask specified in findContinuum(mask='') parameter
If threshold is specified, then it only includes pixels
with an intensity above that value.
Returns:
* average spectrum
* percentage of pixels not masked
Note: This function assumes that the spectral axis is the final axis.
If it is not, there is no single setting of the axis parameter that
can make it work.
"""
if (useAbsoluteValue):
pixels = np.abs(pixels)
if (len(mask) > 0):
npixels = np.prod(np.shape(pixels))
before = np.count_nonzero(np.isnan(pixels))
pixels[np.where(mask==False)] = np.nan
after = np.count_nonzero(np.isnan(pixels))
maskfalse = after-before
print("Ignoring %d/%d pixels (%.2f%%) where mask is False" % (maskfalse, npixels, 100.*maskfalse/npixels))
if (len(usermask) > 0):
npixels = np.prod(np.shape(pixels))
before = np.count_nonzero(np.isnan(pixels))
pixels[np.where(usermask==0)] = np.nan
after = np.count_nonzero(np.isnan(pixels))
maskfalse = after-before
print("Ignoring %d/%d pixels (%.2f%%) where usermask is 0" % (maskfalse, npixels, 100.*maskfalse/npixels))
if (len(mask) > 0 or len(usermask) > 0):
nonnan = np.prod(np.shape(pixels)) - np.count_nonzero(np.isnan(pixels))
percentageUsed = 100.*nonnan/npixels
print("Using %d/%d pixels (%.2f%%)" % (nonnan, npixels, percentageUsed))
else:
percentageUsed = 100
nchan = np.shape(pixels)[-1]
# Check if each channel has an intensity contribution from at least one spatial pixel.
for i in range(nchan):
if (len(np.shape(pixels)) == 4):
channel = pixels[:,:,0,i].flatten()
else:
channel = pixels[:,:,i].flatten()
validChan = len(np.where(channel >= threshold)[0])
if (validChan < 1):
casalogPost("ch%4d: none of the %d pixels meet the threshold in this channel" % (i,len(channel)))
# Compute the mean (or median) spectrum by averaging over one spatial dimension followed by the next,
# replacing the pixels array with an array that is smaller by one dimension after the first averaging step.
for i in range(len(np.shape(pixels))-1):
if (median):
pixels = nanmedian(pixels, axis=0)
else:
if (threshold is not None):
idx = np.where(pixels < threshold)
if len(idx[0]) > 0:
pixels[idx] = np.nan
pixels = nanmean(pixels, axis=0)
return(pixels, percentageUsed)
[docs]def propagateMaskToAllChannels(mask, axis=3):
"""
This function is called by meanSpectrum.
Takes a spectral mask array, and propagates every masked spatial pixel to
all spectral channels of the array.
Returns: a 2D mask (of the same spatial dimension as the input,
but with only 1 channel)
"""
casalogPost("Propagating image mask to all channels")
startTime = timeUtilities.time()
newmask = np.sum(mask,axis=axis)
newmask[np.where(newmask>0)] = 1
casalogPost(" elapsed time = %.1f sec" % (timeUtilities.time()-startTime))
return(newmask)
[docs]def widthOfMaskArcsec(mask, maskInfo):
"""
++++ This function is not used by pipeline when meanSpectrumMethod='mom0mom8jointMask' or if mask=''
Finds width of the smallest central square that circumscribes all masked
pixels. Returns the value in arcsec.
"""
print("Opening mask: ", mask)
myia = iatool()
myia.open(mask)
pixels = myia.getregion(axes=[2,3])
# ia.getregion() yields np.shape(pixels) = (1,138119016)
# ia.getregion(axes=[0,1]) yields np.shape(pixels) = (1,1,1,1918)
# ia.getregion(axes=[2,3]) yields np.shape(pixels) = (1960,1960,1,1)
myia.close()
idx = np.where(pixels==True)
leftRadius = np.shape(pixels)[0]/2 - np.min(idx[0])
rightRadius = np.max(idx[0]) - np.shape(pixels)[0]/2
width = 2*np.max(np.abs([leftRadius,rightRadius]))
topRadius = np.max(idx[1]) - np.shape(pixels)[1]/2
bottomRadius = np.shape(pixels)[1]/2 - np.min(idx[1])
height = 2*np.max(np.abs([topRadius,bottomRadius]))
cdelt1 = maskInfo[3]
width = np.abs(cdelt1)*(np.max([width,height])+1)
return width
[docs]def checkForTriangularWavePattern(img, triangleFraction=0.83, pad=20):
"""
+++++++ This function is not used for meanSpectrumMethod == 'mom0mom8jointMask'.
Fit and remove linear slopes to each half of the spectrum, then comparse
the MAD of the residual to the MAD of the original spectrum
pad: fraction of total channels to ignore on each edge (e.g. 20: 1/20th)
triangleFraction: MAD of dual-linfit residual must be less than this fraction*originalMAD
Returns:
* Boolean: whether triangular pattern meets the threshold
* value: either False (if slope test failed) or a float (the ratio of the MADs)
"""
casalogPost('Checking for triangular wave pattern in the noise')
chanlist, mad = computeMadSpectrum(img)
nchan = len(chanlist)
n0 = nchan/2 - nchan/pad
n1 = nchan/2 + nchan/pad
slope = 0
intercept = np.mean(mad[nchan/pad:-nchan/pad])
slope0, intercept0 = linfit(chanlist[nchan/pad:n0], mad[nchan/pad:n0], MAD(mad[nchan/pad:n0]))
slope1, intercept1 = linfit(chanlist[n1:-nchan/pad], mad[n1:-nchan/pad], MAD(mad[n1:-nchan/pad]))
casalogPost("checkForTriangularWavePattern: slope0=%+g, slope1=%+g, %s" % (slope0,slope1,os.path.basename(img)))
slopeTest = slope0 > 0 and slope1 < 0
slopeDiff = abs(abs(slope1)-abs(slope0))
slopeSum = (abs(slope1)+abs(slope0))
similarSlopeThreshold = 0.70 # 0.40
print("slopeSum = ", slopeSum)
similarSlopes = slopeDiff/slopeSum < similarSlopeThreshold
# if the slope is sufficiently high, then it is probably a real feature, not a noise feature
slopeSignPattern = slope0 > 0 and slope1 < 0
largeSlopes = abs(slope0)>1e-5 or abs(slope1)>1e-5
differentSlopeThreshold = 5
differentSlopes = (abs(slope0/slope1) > differentSlopeThreshold) or (abs(slope0/slope1) < 1.0/differentSlopeThreshold)
slopeTest = (slopeSignPattern and similarSlopes and not largeSlopes) or (differentSlopes and not largeSlopes)
if slopeTest:
residual = mad - (chanlist*slope + intercept)
madOfResidualSingleLine = MAD(residual)
residual0 = mad[:nchan/2] - (chanlist[:nchan/2]*slope0 + intercept0)
residual1 = mad[nchan/2:] - (chanlist[nchan/2:]*slope1 + intercept1)
madOfResidual = MAD(list(residual0) + list(residual1))
madRatio = madOfResidual/madOfResidualSingleLine
casalogPost("checkForTriangularWavePattern: MAD_of_residual=%e, thresholdFraction=%.2f, ratio_of_MADs=%.3f, slopeDiff/slopeSum=%.2f, slope0/slope1=%.2f, signPattern=%s similarSlopes=%s largeSlopes=%s differentSlopes=%s" % (madOfResidual, triangleFraction, madRatio, slopeDiff/slopeSum,slope0/slope1,slopeSignPattern,similarSlopes,largeSlopes,differentSlopes))
if (madRatio < triangleFraction):
return True, madRatio
else:
return False, madRatio
else:
casalogPost("checkForTriangularWavePattern: slopeDiff/slopeSum=%.2f signPattern=%s similarSlopes=%s largeSlopes=%s %s" % (slopeDiff/slopeSum,slopeSignPattern,similarSlopes,largeSlopes,os.path.basename(img)))
return False, slopeTest
[docs]def computeMadSpectrum(img, box=''):
"""
+++++++ This function is not used for meanSpectrumMethod == 'mom0mom8jointMask'.
Only used by checkForTriangularWavePattern.
Uses the imstat task to compute an array containing the MAD spectrum of a cube.
"""
axis = findSpectralAxis(img)
myia = iatool()
myia.open(img)
myshape = myia.shape()
myia.close()
axes = list(range(len(myshape)))
axes.remove(axis)
nchan = myshape[axis]
chanlist = np.array(range(nchan))
casalogPost("Computing MAD spectrum with imstat.")
print("imstat", end=' ')
mydict = imstat(img, axes=axes, box=box, listit=imstatListit, verbose=imstatVerbose)
return(chanlist, mydict['medabsdevmed'])
[docs]def isSingleContinuum(vis, spw='', source='', intent='OBSERVE_TARGET',
verbose=False, mymsmd=None):
"""
+++++++ This function is not used by pipeline, rather it is an expected input parameter.
Checks whether an spw was defined as single continuum in the OT by
looking at the transition name in the SOURCE table of a measurement set.
vis: a single measurement set, a comma-delimited list, or a python list
(only the first one will be used)
Optional parameters:
spw: can be integer ID or string integer ID; if not specified, then it
will use the first science spw
source: passed to transition(); if not specified, it will use the first one
intent: if source is blank then use first one with matching intent and spw
mymsmd: an existing instance of the msmd tool
"""
if vis=='': return False
if type(vis) == list or type(vis) == np.ndarray:
vis = vis[0]
else:
vis = vis.split(',')[0]
if not os.path.exists(vis): return False
needToClose = False
if spw=='':
if mymsmd is None:
needToClose = True
mymsmd = msmdtool()
mymsmd.open(vis)
spw = getScienceSpws(vis, returnString=False, mymsmd=mymsmd)[0]
info = transition(vis, spw, source, intent, verbose, mymsmd)
if needToClose:
mymsmd.close()
if len(info) > 0:
if info[0].find('Single_Continuum') >= 0:
casalogPost("Identified spectral setup as Single_Continuum from transition name.")
return True
return False
[docs]def transition(vis, spw, source='', intent='OBSERVE_TARGET',
verbose=True, mymsmd=None):
"""
+++++++ This function is not used by pipeline, because it is only used by isSingleContinuum.
Returns the list of transitions for specified spw (and source).
vis: measurement set
spw: can be integer ID or string integer ID
Optional parameters:
source: can be integer ID or string name; if not specified, use the first one
intent: if source is blank then use first one with matching intent and spw
"""
if (not os.path.exists(vis)):
print("Could not find measurement set")
return
needToClose = False
if mymsmd is None:
needToClose = True
mymsmd = msmdtool()
mymsmd.open(vis)
spw = int(spw)
if (spw >= mymsmd.nspw()):
print("spw not in the dataset")
if needToClose:
mymsmd.close()
return
mytb = tbtool()
mytb.open(vis+'/SOURCE')
spws = mytb.getcol('SPECTRAL_WINDOW_ID')
sourceIDs = mytb.getcol('SOURCE_ID')
names = mytb.getcol('NAME')
spw = int(spw)
if (type(source) == str):
if (source.isdigit()):
source = int(source)
elif (source == ''):
# pick source
fields1 = mymsmd.fieldsforintent(intent+'*')
fields2 = mymsmd.fieldsforspw(spw)
fields = np.intersect1d(fields1,fields2)
source = mymsmd.namesforfields(fields[0])[0]
if verbose:
print("For spw %d, picked source: " % (spw), source)
if (type(source) == str or type(source) == np.string_):
sourcerows = np.where(names==source)[0]
if (len(sourcerows) == 0):
# look for characters ()/ and replace with underscore
names = np.array(sanitizeNames(names))
sourcerows = np.where(source==names)[0]
else:
sourcerows = np.where(sourceIDs==source)[0]
spwrows = np.where(spws==spw)[0]
row = np.intersect1d(spwrows, sourcerows)
if (len(row) > 0):
if (mytb.iscelldefined('TRANSITION',row[0])):
transitions = mytb.getcell('TRANSITION',row[0])
else:
transitions = []
else:
transitions = []
if (len(transitions) == 0):
print("No value found for this source/spw (row=%s)." % row)
mytb.close()
if needToClose:
mymsmd.close()
return(transitions)
[docs]def getScienceSpws(vis, intent='OBSERVE_TARGET#ON_SOURCE', returnString=True,
tdm=True, fdm=True, mymsmd=None, sqld=False):
"""
+++++++ This function is not used by pipeline, because it is only used by isSingleContinuum.
Return a list of the each spw with the specified intent. For ALMA data,
it ignores channel-averaged and SQLD spws.
returnString: if True, it returns: '1,2,3'
if False, it returns: [1,2,3]
"""
needToClose = False
if (mymsmd is None or mymsmd == ''):
mymsmd = msmdtool()
mymsmd.open(vis)
needToClose = True
if (intent not in mymsmd.intents()):
if intent.split('#')[0] in [i.split('#')[0] for i in mymsmd.intents()]:
# VLA uses OBSERVE_TARGET#UNSPECIFIED, so try that before giving up
intent = intent.split('#')[0]+'*'
else:
casalogPost("Intent %s not in dataset (nor is %s*)." % (intent,intent.split('#')[0]))
spws = mymsmd.spwsforintent(intent)
observatory = getObservatoryName(vis)
if (observatory.find('ALMA') >= 0 or observatory.find('OSF') >= 0):
almaspws = mymsmd.almaspws(tdm=tdm,fdm=fdm,sqld=sqld)
if (len(spws) == 0 or len(almaspws) == 0):
scienceSpws = []
else:
scienceSpws = np.intersect1d(spws,almaspws)
else:
scienceSpws = spws
if needToClose:
mymsmd.close()
if (returnString):
return(','.join(str(i) for i in scienceSpws))
else:
return(list(scienceSpws))
[docs]def getObservationStart(vis, obsid=-1, verbose=False):
"""
Reads the start time of the observation from the OBSERVATION table (using tb tool)
and reports it in MJD seconds.
obsid: if -1, return the start time of the earliest obsID
-Todd Hunter
"""
if (os.path.exists(vis) == False):
print("vis does not exist = %s" % (vis))
return
if (os.path.exists(vis+'/table.dat') == False):
print("No table.dat. This does not appear to be an ms.")
print("Use au.getObservationStartDateFromASDM().")
return
mytb = tbtool()
try:
mytb.open(vis+'/OBSERVATION')
except:
print("ERROR: failed to open OBSERVATION table on file "+vis)
return(3)
time_range = mytb.getcol('TIME_RANGE')
mytb.close()
if verbose: print("time_range: ", str(time_range))
# the first index is whether it is starttime(0) or stoptime(1)
time_range = time_range[0]
if verbose: print("time_range[0]: ", str(time_range))
if (obsid >= len(time_range)):
print("Invalid obsid")
return
if obsid >= 0:
time_range = time_range[obsid]
elif (type(time_range) == np.ndarray):
time_range = np.min(time_range)
return(time_range)
[docs]def getObservatoryName(vis):
"""
+++++++ This function is not used by pipeline, because it is only used by getScienceSpws.
Returns the observatory name in the specified ms.
"""
antTable = vis+'/OBSERVATION'
try:
mytb = tbtool()
mytb.open(antTable)
myName = mytb.getcell('TELESCOPE_NAME')
mytb.close()
except:
casalogPost("Could not open OBSERVATION table to get the telescope name: %s" % (antTable))
myName = ''
return(myName)
[docs]def makeUvcontsub(files='*.dat', fitorder=1, useFrequency=False):
"""
+++++++ This function is not used anywhere else in this file.
Takes a list of output files from findContinuum and builds an spw selection
for uvcontsub, then prints the resulting commands to the screen.
files: a list of files, either a comma-delimited string, a python list, or a wildcard string
fitorder: passed through to the uvcontsub
useFrequency: if True, then produce selection string in frequency; otherwise use channels
Note: frequencies in the findContinuum .dat file are topo, which is what uvcontsub wants.
"""
if files.find('*') >= 0:
resultFiles = sorted(glob.glob(files))
uids = []
for resultFile in resultFiles:
uid = resultFile.split('.')[0]
if len(np.unique(uids)) > 1:
print("There are results for more than one OUS in this directory. Be more specific.")
return
elif type(files) == str:
resultFiles = sorted(files.split(','))
else:
resultFiles = sorted(files)
freqRanges = {}
spws = []
for rf in resultFiles:
spw = rf.split('spw')[1].split('.')[0]
spws.append(spw)
uid = rf.split('.')[0]
field = rf[len(uid)+6:].split('_')[1]
f = open(rf,'r')
lines = f.readlines()
f.close()
for line in lines:
tokens = line.split()
if line == lines[0]:
channelRanges = tokens[0]
if len(tokens) == 2:
uid = tokens[0]
if uid not in freqRanges:
freqRanges[uid] = {}
if field not in freqRanges[uid]:
freqRanges[uid][field] = ''
else:
freqRanges[uid][field] += ','
if useFrequency:
freqRanges[uid][field] += '%s:%s' % (spw,tokens[1])
else:
freqRanges[uid][field] += '%s:%s' % (spw,channelRanges)
spws = ','.join(np.unique(spws))
if freqRanges == {} and useFrequency:
print("There are no frequency ranges in the *.dat files. You need to run findContinuum with the 'vis' parameter specified.")
return
for uid in freqRanges:
for field in freqRanges[uid]:
print("uvcontsub('%s', field='%s', fitorder=%d, spw='%s', fitspw='%s')\n" % (uid, field, fitorder, spws, freqRanges[uid][field]))
[docs]def getcube(i, filename='cubelist.txt'):
"""
++++ This function is not used by pipeline.
Translates a PDF page number to a cube name, for regression purposes,
by reading the specified file.
filename: a file containing 2-column lines like: '1 mycube.residual'
"""
f = open(filename, 'r')
lines = f.readlines()
for line in lines:
token = line.split()
if len(token) == 2:
if i == int(token[0]):
cube = token[1]
f.close()
casalogPost('Translated cube %d into %s' % (i, cube))
return cube
[docs]def readViewerOutputFile(lines, debug=False):
"""
++++++ This function is not used by the pipeline.
Reads an ASCII spectrum file produced by the CASA viewer or by tt.ispec.
Returns: 4 items: 2 arrays and 2 strings:
* array for x-axis, array for y-axis
* x-axis units, intensity units
"""
x = []; y = []
pixel = ''
xunits = 'unknown'
intensityUnits = 'unknown'
for line in lines:
if (line[0] == '#'):
if (line.find('pixel') > 0):
pixel = line.split('[[')[1].split(']]')[0]
elif (line.find('xLabel') > 0):
xunits = line.split()[-1]
if (debug):
print("Read xunits = ", xunits)
elif (line.find('yLabel') > 0):
tokens = line.split()
if (len(tokens) == 2):
intensityUnits = tokens[1]
else:
intensityUnits = tokens[1] + ' (' + tokens[2] + ')'
continue
tokens = line.split()
if (len(tokens) < 2):
continue
x.append(float(tokens[0]))
y.append(float(tokens[1]))
return(np.array(x), np.array(y), xunits, intensityUnits)
[docs]def readMeanSpectrumFITSFile(meanSpectrumFile, unit=0, nanBufferChannels=1):
"""
++++++ This function is not used by the pipeline.
Reads a spectrum from a FITS table, such as one output by spectralcube.
Returns: 8 items
* average spectrum
* average spectrum with the NaNs replaced with the minimum value
* threshold used (currently hardcoded to 0.0)
* edges used (currently hardcoded to 2)
* number of channels
* the minimum value
* first frequency
* last frequency
"""
f = pyfits.open(meanSpectrumFile)
tbheader = f[unit].header
tbdata = f[unit].data
nchan = len(tbdata)
crpix = tbheader['CRPIX1']
crval = tbheader['CRVAL1']
cdelt = tbheader['CDELT1']
units = tbheader['CUNIT1']
if (units.lower() != ('hz')):
print("Warning: frequency units are not Hz = ", units.lower())
return
firstFreq = crval - (crpix-1)*cdelt
lastFreq = firstFreq + cdelt*(nchan-1)
avgspectrum = tbdata
edgesUsed = 2
threshold = 0
avgSpectrumNansReplaced,nanmin = removeNaNs(tbdata, replaceWithMin=True,
nanBufferChannels=nanBufferChannels)
return(avgspectrum, avgSpectrumNansReplaced, threshold,
edgesUsed, nchan, nanmin, firstFreq, lastFreq)
[docs]def readPreviousMeanSpectrum(meanSpectrumFile, verbose=False, invert=False):
"""
++++++ This function is not used by the pipeline.
Read a previous calculated mean spectrum and its parameters,
or an ASCII file created by the CASA viewer (or tt.ispec).
Note: only the intensity column(s) are returned, not the
channel/frequency columns.
This function will not typically be used by the pipeline, only manual users.
Returns: 11 things:
avgspectrum, avgSpectrumNansReplaced, threshold,
edgesUsed, nchan, nanmin, firstFreq, lastFreq, centralArcsec,
mask, percentagePixelsNotMasked
"""
f = open(meanSpectrumFile, 'r')
lines = f.readlines()
f.close()
if (len(lines) < 3):
return None
i = 0
# Detect file type:
if (lines[0].find('title: Spectral profile') > 0):
# CASA viewer/au.ispec output file
print("Reading CASA viewer output file")
freq, intensity, freqUnits, intensityUnits = readViewerOutputFile(lines)
if (freqUnits.lower().find('ghz')>=0):
freq *= 1e9
elif (freqUnits.lower().find('mhz')>=0):
freq *= 1e6
elif (freqUnits.lower().find('[hz')<0):
print("Spectral axis of viewer output file must be in Hz, MHz or GHz, not %s." % freqUnits)
return
threshold = 0
edgesUsed = 2
return(intensity, intensity, threshold,
edgesUsed, len(intensity), np.nanmin(intensity), freq[0], freq[-1], None, None, None)
centralArcsec = ''
mask = ''
percentagePixelsNotMasked = -1
while (lines[i][0] == '#'):
if (lines[i].find('centralArcsec=') > 0):
if (lines[i].find('=auto') > 0):
centralArcsec = 'auto'
elif (lines[i].find('=mom0mom8jointMask') > 0):
centralArcsec = 'mom0mom8jointMask'
else:
centralArcsec = float(lines[i].split('centralArcsec=')[1].split()[0])
token = lines[i].split()
if (len(token) > 6):
mask = token[6]
if (len(token) > 7):
percentagePixelsNotMasked = int(token[7])
i += 1
token = lines[i].split()
if len(token) == 4:
a,b,c,d = token
threshold = float(a)
edgesUsed = int(b)
nchan = int(c)
nanmin = float(d)
else:
threshold = 0
edgesUsed = 0
nchan = 0
nanmin = 0
avgspectrum = []
avgSpectrumNansReplaced = []
freqs = []
channels = []
for line in lines[i+1:]:
if (line[0] == '#'): continue
tokens = line.split()
if (len(tokens) == 2):
# continue to support the old 2-column format
freq,a = line.split()
b = a
nchan += 1
freq = float(freq)
if freq < 1e6:
freq *= 1e6 # convert MHz to Hz
freqs.append(freq)
elif len(tokens) > 2:
chan,freq,a,b = line.split()
channels.append(int(chan))
freqs.append(float(freq))
else:
print("len(tokens) = ", len(tokens))
if invert:
a = -float(a)
b = -float(b)
avgspectrum.append(float(a))
avgSpectrumNansReplaced.append(float(b))
avgspectrum = np.array(avgspectrum)
avgSpectrumNansReplaced = np.array(avgSpectrumNansReplaced)
if invert:
avgspectrum += abs(np.min(avgspectrum))
avgSpectrumNansReplaced += abs(np.min(avgSpectrumNansReplaced))
if (len(freqs) > 0):
firstFreq = freqs[0]
lastFreq = freqs[-1]
else:
firstFreq = 0
lastFreq = 0
casalogPost("Read previous mean spectrum with %d channels, (%d freqs: %f-%f)" % (len(avgspectrum),len(freqs),firstFreq,lastFreq),verbose)
return(avgspectrum, avgSpectrumNansReplaced, threshold,
edgesUsed, nchan, nanmin, firstFreq, lastFreq, centralArcsec,
mask, percentagePixelsNotMasked)
[docs]def removeNaNs(a, replaceWithMin=False, verbose=False, nanBufferChannels=0,
replaceWithZero=False):
"""
+++++++ This function is not used for meanSpectrumMethod == 'mom0mom8jointMask' by pipeline,
But it is called by readMeanSpectrumFITSFile available to manual users.
Remove or replace the nan values from an array.
replaceWithMin: if True, then replace NaNs with np.nanmin of array
if False, then simply remove the NaNs
replaceWithZero: if True, then replace NaNs with np.nanmin of array
if False, then simply remove the NaNs
nanBufferChannels: only active if replaceWithMin=True
Returns:
* new array
* if replaceWithMin=True, then also return the value used to replace NaNs
"""
a = np.array(a)
if (len(a) < 1): return(a)
startLength = len(a)
if (replaceWithMin or replaceWithZero):
idx = np.isnan(a)
print("Found %d nan channels: %s" % (len(np.where(idx==True)[0]), np.where(idx==True)[0]))
idx2 = np.isinf(a)
print("Found %d inf channels" % (len(np.where(idx2==True)[0])))
a_nanmin = np.nanmin(a)
if (nanBufferChannels > 0):
idxlist = splitListIntoHomogeneousLists(idx)
idx = []
for i,mylist in enumerate(idxlist):
if (mylist[0]):
idx += mylist
else:
newSubString = nanBufferChannels*[True]
if (i < len(idxlist)-1):
newSubString += mylist[nanBufferChannels:-nanBufferChannels] + nanBufferChannels*[True]
else:
newSubString += mylist[-nanBufferChannels:]
# If the channel block was less than 2*nanBufferChannels wide, then only insert up to its width
idx += newSubString[:len(mylist)]
idx = np.array(idx)
if (verbose):
print("Replaced %d NaNs" % (len(idx)))
print("Replaced %d infs" % (len(idx2)))
if (replaceWithMin):
a[idx] = a_nanmin
a[idx2] = a_nanmin
return(a, a_nanmin)
elif (replaceWithZero):
a[idx] = 0
a[idx2] = 0
return(a, 0)
else:
a = a[np.where(np.isnan(a) == False)]
if (verbose):
print("Removed %d NaNs" % (startLength-len(a)))
startLength = len(a)
a = a[np.where(np.isinf(a) == False)]
if (verbose):
print("Removed %d infs" % (startLength-len(a)))
return(a)
[docs]def splitListIntoHomogeneousLists(mylist):
"""
This function is called only by removeNaNs, and hence is not used in Cycle 6-onward pipeline.
Converts [1,1,1,2,2,3] into [[1,1,1],[2,2],[3]], etc.
-Todd Hunter
"""
mylists = []
newlist = [mylist[0]]
for i in range(1,len(mylist)):
if (mylist[i-1] != mylist[i]):
mylists.append(newlist)
newlist = [mylist[i]]
else:
newlist.append(mylist[i])
mylists.append(newlist)
return(mylists)
[docs]def removeZeros(a):
"""
+++++ This function is not used for meanSpectrumMethod='mom0mom8jointMask'.
Takes an array, and replaces all appearances of 0.0 with the minimum value
of all channels not equal to 0.0. This is used to reduce the bias caused
by the edge channels being 0.0 in the profiles returned by ia.getprofile.
If the absolute value of the minimum is greater than the absolute value of
the maximum, then set the channels to the maximum value.
"""
idx = np.where(a == 0.0)
idx2 = np.where(a != 0.0)
a_nanmin = np.nanmin(a[idx2])
a_nanmax = np.nanmax(a[idx2])
if np.abs(a_nanmin) < np.abs(a_nanmax):
# emission spectrum
a[idx] = a_nanmin
else:
# absorption spectrum
a[idx] = a_nanmax
print("Replaced %d zeros with %f" % (len(idx[0]), a_nanmin))
return a
[docs]def plotStatisticalSpectrumFromMask(cube, jointMask='', pbcube=None,
statistic='mean', normalizeByMAD=False,
png='', jointMaskForNormalize=''):
"""
Simple wrapper to call computeStatisticalSpectrumFromMask and plot it.
jointMask: either a mask image, or an expression with < or > on an mask
image
"""
if not os.path.exists(cube):
print("Could not find cube")
return
statistics = statistic.split(',')
jointMasks = jointMask.split(',')
for jointMask in jointMasks:
if jointMask != '' and jointMask.find('>') < 0 and jointMask.find('>') < 0:
if not os.path.exists(jointMask):
print("Could not find jointmask")
return
plt.clf()
colors = ['r','k','b']
for i,statistic in enumerate(statistics):
if len(jointMasks) == 1:
jointMask = jointMasks[0]
else:
jointMask = jointMasks[i]
if jointMaskForNormalize == '':
myJointMaskForNormalize = jointMask
else:
myJointMaskForNormalize = jointMaskForNormalize
channels, frequency, intensity, normalized = computeStatisticalSpectrumFromMask(cube, jointMask, pbcube, statistic=statistic, normalizeByMAD=normalizeByMAD, jointMaskForNormalize=myJointMaskForNormalize)
plt.plot(channels, intensity, '-', color=colors[i])
plt.xlabel('Channel')
plt.ylabel(statistic)
print( "peak over MAD = ", np.max(intensity)/MAD(intensity))
plt.title(','.join([os.path.basename(cube), os.path.basename(jointMask)]))
if png == '':
png = cube + '.' + jointMask + '.' + statistic + '.png'
plt.savefig(png)
print("Wrote ", png)
plt.draw()
[docs]def pruneMask(mymask, psfcube=None, minbeamfrac=0.3, prunesize=6.0, nchan=1, overwrite=True):
"""
available in CASA >= 5.4.0-X (see CAS-11335)
Removes any regions smaller than prunesize contiguous pixels.
psfcube: if specified, then use minbeamfrac * pixelsPerBeam, otherwise use prunesize
prunesize: value used if psfcube is None; use 6 because pipeline size mitigation could
produce images that have only 3 pixels across the beam, meaning only ~7-8 pts/beam
nchan: number of channels to process, will always be 1 in findContinuum's use case
overwrite: if True, replaces mymask with pruned mask, putting original in mymask.prepruned
if False, is simply creates: mymask.pruned, but only if it pruned anything
If all regions are pruned, it checks if it is ACA7m (maxBaseline < 60m) and if so, tries
again with minbeamfrac*0.5.
"""
npruned = 0
if not os.path.exists(mymask):
print("Could not find mask image")
return npruned
casalogPost('Pruning the joint mask')
if psfcube is not None:
myinfo = getImageInfo(psfcube)
# np.log(2) standard factor for solid angle was not in Cycles0..7
# pixelsPerBeam = myinfo[0]*myinfo[1]*np.pi/(4.*np.log(2))/np.abs(myinfo[3]*myinfo[4])
pixelsPerBeam = myinfo[0]*myinfo[1]*np.pi/4./np.abs(myinfo[3]*myinfo[4])
maxBaseline = myinfo[11] # in meters
prunesize = np.max([4,int(pixelsPerBeam*minbeamfrac)])
casalogPost('Beam = %.3fx%.3f, pixels per beam = %.2f, using prunesize = %.2f' % (myinfo[0],myinfo[1],pixelsPerBeam, prunesize))
try:
maskhandler = synthesismaskhandler()
except:
casalogPost("This casa does not contain the tool synthesismaskhandler(), so the joint mask cannot be pruned.")
return npruned
chanflag = np.zeros(nchan, dtype=bool)
mymask = mymask.rstrip('/')
try:
mydict = maskhandler.pruneregions(mymask, prunesize, chanflag)
except:
casalogPost("pruneregions failed, skipping the prune step")
return npruned
npruned = mydict['N_reg_pruned'][0]
newmask = mymask + '.pruned'
if npruned > 0:
casalogPost('Pruned %d regions.' % (npruned))
remainingPixels = countPixelsAboveZero(newmask)
casalogPost('Remaining pixels = %s.' % (str(remainingPixels)))
if remainingPixels < 1 and psfcube is not None:
# Everything got pruned away, so check if ACA and reduce minbeamfrac by factor of 2
casalogPost('All regions were pruned.')
if maxBaseline < 60:
newprunesize = np.max([4,int(pixelsPerBeam*minbeamfrac*0.5)])
casalogPost('New prunesize = %d.' % (npruned))
if prunesize != newprunesize:
casalogPost('All regions pruned and this looks like a 7m image (maxBaseline ~ %.0fm), so setting new prunesize=%d' % (maxBaseline,newprunesize))
shutil.rmtree(newmask)
mydict = maskhandler.pruneregions(mymask, newprunesize, chanflag)
npruned = mydict['N_reg_pruned'][0]
maskhandler.done()
if overwrite:
if npruned > 0:
removeIfNecessary(mymask+'.prepruned')
os.rename(mymask, mymask+'.prepruned')
os.rename(newmask, mymask)
casalogPost("Updated joint mask after pruning %d region(s)" % (npruned))
else:
# Remove newly-created file because it is identical to original
shutil.rmtree(newmask)
else:
print("Pruned %d regions" % (npruned))
return npruned