This is the User Notebook to run validation tests at NERSC You only need to input your localdir (at NERSC) to save outputs.
Each cell should have one test
To run a test you need to input
Output: path to png image to plot
In [1]:
from desi_image_validation import *
from IPython.display import Image
from astropy.io import fits
import scipy.signal
import matplotlib.pyplot as plt
#%matplotlib inline
In [46]:
#help(scipy.signal);
In [2]:
localdir = '/global/homes/m/manera/DESI/validation-outputs/'
In [2]:
verbose=False
sample=mysample('DECaLS','DR7','g',localdir,verbose)
# TEST Information
print '----- check values sample ----'
print 'band = ', sample.band
print 'localdir = ', sample.localdir
print 'survey = ', sample.survey
print 'DR = ', sample.DR
print 'path ccd = ',sample.ccds
print '----------'
print 'zp0 = ', sample.zp0
print 'extinction coef = ', sample.extc
print 'extinction raw = ', sample.be
print 'magnitude limit = ', sample.recm
print 'photoz req = ', sample.phreq
print 'catalogue = ', sample.catalog
print 'verbose = ', sample.verbose
print 'fraction of exposures = ', sample.FracExp
print '------------------------'
print ' '
In [6]:
import numpy
def smooth(x,window_len=11,window='hanning'):
"""smooth the data using a window with requested size.
This method is based on the convolution of a scaled window with the signal.
The signal is prepared by introducing reflected copies of the signal
(with the window size) in both ends so that transient parts are minimized
in the begining and end part of the output signal.
input:
x: the input signal
window_len: the dimension of the smoothing window; should be an odd integer
window: the type of window from 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'
flat window will produce a moving average smoothing.
output:
the smoothed signal
example:
t=linspace(-2,2,0.1)
x=sin(t)+randn(len(t))*0.1
y=smooth(x)
see also:
numpy.hanning, numpy.hamming, numpy.bartlett, numpy.blackman, numpy.convolve
scipy.signal.lfilter
TODO: the window parameter could be the window itself if an array instead of a string
NOTE: length(output) != length(input), to correct this: return y[(window_len/2-1):-(window_len/2)] instead of just y.
"""
if x.ndim != 1:
raise ValueError, "smooth only accepts 1 dimension arrays."
if x.size < window_len:
raise ValueError, "Input vector needs to be bigger than window size."
if window_len<3:
return x
if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
raise ValueError, "Window is on of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'"
s=numpy.r_[x[window_len-1:0:-1],x,x[-2:-window_len-1:-1]]
#print(len(s))
if window == 'flat': #moving average
w=numpy.ones(window_len,'d')
else:
w=eval('numpy.'+window+'(window_len)')
y=numpy.convolve(w/w.sum(),s,mode='valid')
return y
from numpy import *
from pylab import *
def smooth_demo():
t=linspace(-4,4,100)
x=sin(t)
xn=x+randn(len(t))*0.1
y=smooth(x)
ws=31
subplot(211)
plot(ones(ws))
windows=['flat', 'hanning', 'hamming', 'bartlett', 'blackman']
hold(True)
for w in windows[1:]:
eval('plot('+w+'(ws) )')
axis([0,30,0,1.1])
legend(windows)
title("The smoothing windows")
subplot(212)
plot(x)
plot(xn)
for w in windows:
plot(smooth(xn,10,w))
l=['original signal', 'signal with noise']
l.extend(windows)
legend(l)
title("Smoothing a noisy signal")
show()
if __name__=='__main__':
smooth_demo()
In [9]:
verbose=False
sample=mysample('BASS','DR6','g',localdir,verbose)
print sample.ccds
hdul=fits.open(sample.ccds)
myheader=hdul[1].header
mydata=hdul[1].data
hdul.close()
galdepth=mydata['galdepth']
mjdobs=mydata['mjd_obs']
mjddepth=galdepth[mjdobs.argsort()] # depth sorted by modified julian date
depthsmooth=smooth(mjddepth,window_len=401,window='flat') #remove window in output y[200,-200]
plt.scatter(mjdobs,depthsmooth[200:-200],s=0.05,c='green')
plt.xlabel("MJD observed")
plt.ylabel("Average depth")
plt.title("g-band hanning mean depth \n (400 exposures window)")
plt.show()
In [10]:
verbose=False
sample=mysample('BASS','DR6','r',localdir,verbose)
print sample.ccds
hdul=fits.open(sample.ccds)
myheader=hdul[1].header
mydata=hdul[1].data
hdul.close()
galdepth=mydata['galdepth']
mjdobs=mydata['mjd_obs']
mjddepth=galdepth[mjdobs.argsort()] # depth sorted by modified julian date
depthsmooth=smooth(mjddepth,window_len=401,window='flat') #remove window in output y[200,-200]
plt.scatter(mjdobs,depthsmooth[200:-200],s=0.05,c='red')
plt.xlabel("MJD observed")
plt.ylabel("Average depth")
plt.title("r-band hanning mean depth \n (400 exposures window)")
plt.show()
In [11]:
verbose=False
sample=mysample('MZLS','DR6','z',localdir,verbose)
print sample.ccds
hdul=fits.open(sample.ccds)
myheader=hdul[1].header
mydata=hdul[1].data
hdul.close()
galdepth=mydata['galdepth']
mjdobs=mydata['mjd_obs']
mjddepth=galdepth[mjdobs.argsort()] # depth sorted by modified julian date
depthsmooth=smooth(mjddepth,window_len=401,window='flat') #remove window in output y[200,-200]
plt.scatter(mjdobs,depthsmooth[200:-200],s=0.05,c='blue')
plt.xlabel("MJD observed")
plt.ylabel("Average depth")
plt.title("z-band hanning mean depth \n (400 exposures window)")
plt.show()
In [ ]:
In [19]:
verbose=False
sample=mysample('BASS','DR6','g',localdir,verbose)
print sample.ccds
hdul=fits.open(sample.ccds)
myheader=hdul[1].header
mydata=hdul[1].data
hdul.close()
galdepth=mydata['galdepth']
mjdobs=mydata['mjd_obs']
mjddepth=galdepth[mjdobs.argsort()] # depth sorted by modified julian date
depthsmooth=smooth(mjddepth,window_len=2001,window='hanning') #remove window in output y[200,-200]
plt.scatter(mjdobs,depthsmooth[1000:-1000],s=0.05,c='green')
plt.xlabel("MJD observed")
plt.ylabel("Average depth")
plt.title("g-band hanning mean depth \n (2000 exposures window)")
plt.show()
In [20]:
verbose=False
sample=mysample('BASS','DR6','r',localdir,verbose)
print sample.ccds
hdul=fits.open(sample.ccds)
myheader=hdul[1].header
mydata=hdul[1].data
hdul.close()
galdepth=mydata['galdepth']
mjdobs=mydata['mjd_obs']
mjddepth=galdepth[mjdobs.argsort()] # depth sorted by modified julian date
depthsmooth=smooth(mjddepth,window_len=2001,window='hanning') #remove window in output y[200,-200]
plt.scatter(mjdobs,depthsmooth[1000:-1000],s=0.05,c='red')
plt.xlabel("MJD observed")
plt.ylabel("Average depth")
plt.title("r-band hanning mean depth \n (2000 exposures window)")
plt.show()
In [21]:
verbose=False
sample=mysample('MZLS','DR6','z',localdir,verbose)
print sample.ccds
hdul=fits.open(sample.ccds)
myheader=hdul[1].header
mydata=hdul[1].data
hdul.close()
galdepth=mydata['galdepth']
mjdobs=mydata['mjd_obs']
mjddepth=galdepth[mjdobs.argsort()] # depth sorted by modified julian date
depthsmooth=smooth(mjddepth,window_len=2001,window='hanning') #remove window in output y[200,-200]
plt.scatter(mjdobs,depthsmooth[1000:-1000],s=0.05,c='blue')
plt.xlabel("MJD observed")
plt.ylabel("Average depth")
plt.title("z-band hanning mean depth \n (2000 exposures window)")
plt.show()
In [9]:
verbose=False
sample=mysample('BASS','DR6','g',localdir,verbose)
fname=val3p4c_depthfromIvar(sample,Nexpmin=3)
fig = Image(filename=(fname))
fig
Out[9]:
In [11]:
verbose=False
sample=mysample('BASS','DR6','r',localdir,verbose)
fname=val3p4c_depthfromIvar(sample,Nexpmin=3)
fig = Image(filename=(fname))
fig
Out[11]:
In [10]:
verbose=False
sample=mysample('MZLS','DR6','z',localdir,verbose)
fname=val3p4c_depthfromIvar(sample,Nexpmin=3)
fig = Image(filename=(fname))
fig
Out[10]:
In [12]:
verbose=False
sample=mysample('BASS','DR6','g',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[12]:
In [13]:
verbose=False
sample=mysample('BASS','DR6','r',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[13]:
In [7]:
verbose=False
sample=mysample('MZLS','DR6','z',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
#projection in some pointings to healpix pixels is not correct
#This is only for 8 pointins and close to dec=0, usually excluded.
#Edit careful line in quickipManera.py to check the effect
#The efect is actually a covering band around dec=0, whicn need not be there
#Keep the results without these 8 pointings is a very good approximation for what we want
Out[7]:
In [4]:
verbose=False
sample=mysample('DECaLS','DR5','g',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[4]:
In [5]:
verbose=False
sample=mysample('DECaLS','DR5','r',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[5]:
In [6]:
verbose=False
sample=mysample('DECaLS','DR5','z',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[6]:
In [5]:
verbose=False
sample=mysample('DECaLS','DR5','g',localdir,verbose)
fname=val3p4c_depthfromIvar(sample,Nexpmin=3)
fig = Image(filename=(fname))
fig
Out[5]:
In [4]:
verbose=False
sample=mysample('DECaLS','DR5','r',localdir,verbose)
fname=val3p4c_depthfromIvar(sample,Nexpmin=3)
fig = Image(filename=(fname))
fig
Out[4]:
In [3]:
verbose=False
sample=mysample('DECaLS','DR5','z',localdir,verbose)
fname=val3p4c_depthfromIvar(sample,Nexpmin=3)
fig = Image(filename=(fname))
fig
Out[3]:
In [ ]:
verbose=False
sample=mysample('DEShyb','DR5','g',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[ ]:
In [3]:
verbose=False
sample=mysample('NGCproxy','DR5','g',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[3]:
In [4]:
verbose=False
sample=mysample('DEShyb','DR5','r',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[4]:
In [5]:
verbose=False
sample=mysample('NGCproxy','DR5','r',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[5]:
In [6]:
verbose=False
sample=mysample('DEShyb','DR5','z',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[6]:
In [7]:
verbose=False
sample=mysample('NGCproxy','DR5','z',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[7]:
In [6]:
verbose=False
sample=mysample('NGCproxy','DR5','r',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[6]:
In [7]:
verbose=False
sample=mysample('DEShyb','DR5','z',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[7]:
In [8]:
verbose=False
sample=mysample('NGCproxy','DR5','z',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[8]:
In [4]:
verbose=False
sample=mysample('BASS','DR4','g',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[4]:
In [5]:
verbose=False
sample=mysample('BASS','DR4','r',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[5]:
In [6]:
verbose=False
sample=mysample('MZLS','DR4','z',localdir,verbose)
fname=val3p4c_depthfromIvar(sample)
fig = Image(filename=(fname))
fig
Out[6]:
In [3]:
verbose=True
sample=mysample('BASS','DR6','r',localdir,verbose)
fname=val3p4b_maghist_pred(sample,Nexpmin=3)
fig = Image(filename=(fname))
fig
Out[3]:
In [4]:
verbose=True
sample=mysample('BASS','DR6','g',localdir,verbose)
fname=val3p4b_maghist_pred(sample,Nexpmin=3)
fig = Image(filename=(fname))
fig
Out[4]:
In [5]:
verbose=True
sample=mysample('MZLS','DR6','z',localdir,verbose)
fname=val3p4b_maghist_pred(sample,Nexpmin=3)
fig = Image(filename=(fname))
fig
Out[5]:
In [6]:
verbose=True
sample=mysample('BASS','DR6','r',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[6]:
In [7]:
verbose=True
sample=mysample('BASS','DR6','g',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[7]:
In [8]:
verbose=True
sample=mysample('MZLS','DR6','z',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[8]:
In [6]:
verbose=True
sample=mysample('DECaLS','DR5','r',localdir,verbose)
fname=val3p4b_maghist_pred(sample,Nexpmin=3)
fig = Image(filename=(fname))
fig
Out[6]:
In [7]:
verbose=True
sample=mysample('DECaLS','DR5','g',localdir,verbose)
fname=val3p4b_maghist_pred(sample,Nexpmin=3)
fig = Image(filename=(fname))
fig
Out[7]:
In [8]:
verbose=True
sample=mysample('DECaLS','DR5','z',localdir,verbose)
fname=val3p4b_maghist_pred(sample,Nexpmin=3)
fig = Image(filename=(fname))
fig
Out[8]:
In [13]:
verbose=True
sample=mysample('DECaLS','DR5','g',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[13]:
In [11]:
verbose=True
sample=mysample('DECaLS','DR5','r',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[11]:
In [12]:
verbose=True
sample=mysample('DECaLS','DR5','z',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[12]:
In [5]:
verbose=True
sample=mysample('DEShyb','DR5','g',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[5]:
In [6]:
verbose=True
sample=mysample('NGCproxy','DR5','g',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[6]:
In [7]:
verbose=True
sample=mysample('DECaLS','DR5','g',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[7]:
In [8]:
verbose=True
sample=mysample('DEShyb','DR5','r',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[8]:
In [9]:
verbose=True
sample=mysample('NGCproxy','DR5','r',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[9]:
In [11]:
verbose=True
sample=mysample('DECaLS','DR5','r',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[11]:
In [12]:
verbose=True
sample=mysample('DEShyb','DR5','z',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[12]:
In [13]:
verbose=True
sample=mysample('NGCproxy','DR5','z',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[13]:
In [14]:
verbose=True
sample=mysample('DECaLS','DR5','z',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[14]:
In [3]:
verbose=True
sample=mysample('BASS','DR4','g',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[3]:
In [4]:
verbose=True
sample=mysample('BASS','DR4','r',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[4]:
In [5]:
verbose=True
sample=mysample('MZLS','DR4','z',localdir,verbose)
fname=val3p4b_maghist_pred(sample)
fig = Image(filename=(fname))
fig
Out[5]:
z-band image quality will be smaller than 1.3 arcsec FWHM in at least one pass.
val3p4c_seeing - runs everything
val3p4c_seeingplots - runs the plots only if the previous have been run
In [3]:
verbose=True
sample= mysample('BASS','DR4','g',localdir,verbose)
#fname1,fname2 = val3p4c_seeing(sample)
fname1, fname2 = val3p4c_seeing(sample)
print 'For pass >=3'
fig = Image(filename=(fname1))
fig
Out[3]:
In [4]:
fig = Image(filename=(fname2))
fig
Out[4]:
In [7]:
verbose=False
sample = mysample('BASS','DR4','g',localdir,verbose)
fname = v5p1e_photometricReqPlot(sample)
fig = Image(filename=(fname))
fig
Out[7]:
In [6]:
verbose=False
sample = mysample('BASS','DR4','r',localdir,verbose)
fname = v5p1e_photometricReqPlot(sample)
fig = Image(filename=(fname))
fig
Out[6]:
In [3]:
verbose=False
sample = mysample('MZLS','DR4','z',localdir,verbose)
fname = v5p1e_photometricReqPlot(sample)
fig = Image(filename=(fname))
fig
Out[3]:
In [4]:
verbose=False
sample1 = mysample('DECaLS','DR3','g',localdir,verbose)
sample2 = mysample('BASS','DR4','g',localdir,verbose)
fname = v3p5_Areas(sample1,sample2)
fig = Image(filename=(fname))
fig
Out[4]:
In [5]:
verbose=False
sample1 = mysample('DECaLS','DR3','r',localdir,verbose)
sample2 = mysample('BASS','DR4','r',localdir,verbose)
fname = v3p5_Areas(sample1,sample2)
fig = Image(filename=(fname))
fig
Out[5]:
In [3]:
verbose=False
sample1 = mysample('DECaLS','DR3','z',localdir,verbose)
sample2 = mysample('MZLS','DR4','z',localdir,verbose)
fname = v3p5_Areas(sample1,sample2)
fig = Image(filename=(fname))
fig
Out[3]:
In [1]:
import numpy as np
import healpy as hp
import astropy.io.fits as pyfits
from multiprocessing import Pool
import matplotlib as mpl
## mpl.use('Agg')
import matplotlib.pyplot as plt
from quicksipManera import *
In [2]:
inputdir='/project/projectdirs/cosmo/data/legacysurvey/dr3/'
localdir = '/global/homes/m/manera/DESI/validation-outputs/'
fname = inputdir+'ccds-annotated-decals.fits.gz'
In [ ]:
# Obtain Header and DATA
tbdataDR3 = pyfits.open(fname)[1].data
headerDR3=pyfits.open(fname)[1].header
print len(tbdataDR3)
headerDR3
In [3]:
# Compare extinction values
gdepth=tbdataDR3['galdepth'][indsDR3]
ebv=tbdataDR3['EBV'][indsDR3]
exti=tbdataDR3['decam_extinction'][indsDR3][:,1]
print ebv
print exti #compare exti with ebv*3.303
In [ ]:
# Select particular indices
indsDR4 = np.where((tbdataDR3['filter'] == 'g') & (tbdataDR3['photometric'] == True) & (tbdataDR3['blacklist_ok'] == True))
print np.size(indsDR3)
In [ ]:
# Inspect one item
ic=69720
print tbdataDR3['galdepth'][ic]
print tbdataDR3['height'][ic]
print tbdataDR3['CRPIX1'][ic]
print tbdataDR3['CD1_2'][ic]
print tbdataDR3['CRVAL1'][ic], tbdataDR3['CRVAL2'][ic]
print tbdataDR3['RA0'][ic], tbdataDR3['RA1'][ic],tbdataDR3['RA2'][ic], tbdataDR3['RA3'][ic]
print tbdataDR3['DEC0'][ic], tbdataDR3['DEC1'][ic],tbdataDR3['DEC2'][ic], tbdataDR3['DEC3'][ic]
print tbdataDR3['RA'][ic], tbdataDR3['DEC'][ic]
print tbdataDR3['decam_extinction'][ic]
In [12]:
def nanomaggiesToMag(nm):
return -2.5 * (log(nm,10.) - 9.)
def Magtonanomaggies(m):
return 10.**(-m/2.5+9.)
In [13]:
# Way 1
ext = 0.13
mag = 22.5
detsig1 = Magtonanomaggies(mag)/5. #total noise
signalext = 1./10.**(-ext/2.5)
nmagNew=detsig1*signalext
print nmagNew
In [14]:
# Way 2
ext = 0.13
mag = 22.5
mag2 = mag -ext
detsig1 = Magtonanomaggies(mag2)/5. #total noise
nmagNew=detsig1
print nmagNew