In [1]:
# This changes the current directory to the base saga directory - make sure to run this first!
# This is necessary to be able to import the py files and use the right directories,
# while keeping all the notebooks in their own directory.
import os
import sys
import numpy as np

if 'saga_base_dir' not in locals():
    saga_base_dir = os.path.abspath('..')
if saga_base_dir not in sys.path:
    os.chdir(saga_base_dir)

In [2]:
from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.table import Table
from astropy.io import fits

In [3]:
%matplotlib inline
from matplotlib import rcParams, style
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

rcParams['image.cmap'] = 'viridis'
rcParams['image.origin'] = 'lower'
rcParams['axes.prop_cycle'] = style.library['seaborn-deep']['axes.prop_cycle']
rcParams['figure.figsize'] = (14, 8)
rcParams['axes.titlesize'] =  rcParams['axes.labelsize'] = 16
rcParams['xtick.labelsize'] =  rcParams['ytick.labelsize'] = 14

# this final line just shows the first color that plt.plot uses. 
# plt.bar and plt.scatter don't use this so we'll use it explicitly below.
next(iter(rcParams['axes.prop_cycle']))['color']


Out[3]:
u'#4C72B0'

Load the data table and have a quick look at its format


In [4]:
dirtytab = Table.read('SAGADropbox/data/saga_spectra_dirty.fits.gz')

In [5]:
print len(dirtytab)
dirtytab[:5].show_in_notebook()


97671
Out[5]:
<Table masked=True length=5>
OBJIDRADECPHOTPTYPEPHOT_SGFLAGSSATURATEDBAD_COUNTS_ERRORBINNED1ugrizu_errg_errr_erri_errz_errMODELMAGERR_UMODELMAGERR_GMODELMAGERR_RMODELMAGERR_IMODELMAGERR_ZEXTINCTION_UEXTINCTION_GEXTINCTION_REXTINCTION_IEXTINCTION_ZDERED_UDERED_GDERED_RDERED_IDERED_ZPETRORAD_UPETRORAD_GPETRORAD_RPETRORAD_IPETRORAD_ZPETRORADERR_UPETRORADERR_GPETRORADERR_RPETRORADERR_IPETRORADERR_ZDEVRAD_UDEVRADERR_UDEVRAD_GDEVRADERR_GDEVRAD_RDEVRADERR_RDEVRAD_IDEVRADERR_IDEVRAD_ZDEVRADERR_ZDEVAB_UDEVAB_GDEVAB_RDEVAB_IDEVAB_ZCMODELMAG_UCMODELMAGERR_UCMODELMAG_GCMODELMAGERR_GCMODELMAG_RCMODELMAGERR_RCMODELMAG_ICMODELMAGERR_ICMODELMAG_ZCMODELMAGERR_ZPSFMAG_UPSFMAGERR_UPSFMAG_GPSFMAGERR_GPSFMAG_RPSFMAGERR_RPSFMAG_IPSFMAGERR_IPSFMAG_ZPSFMAGERR_ZFIBERMAG_UFIBERMAGERR_UFIBERMAG_GFIBERMAGERR_GFIBERMAG_RFIBERMAGERR_RFIBERMAG_IFIBERMAGERR_IFIBERMAG_ZFIBERMAGERR_ZFRACDEV_UFRACDEV_GFRACDEV_RFRACDEV_IFRACDEV_ZQ_UU_UQ_GU_GQ_RU_RQ_IU_IQ_ZU_ZEXPAB_UEXPRAD_UEXPPHI_UEXPAB_GEXPRAD_GEXPPHI_GEXPAB_REXPRAD_REXPPHI_REXPAB_IEXPRAD_IEXPPHI_IEXPAB_ZEXPRAD_ZEXPPHI_ZFIBER2MAG_RFIBER2MAGERR_REXPMAG_REXPMAGERR_RPETROR50_RPETROR90_RPETROMAG_RSB_EXP_RSB_PETRO_RJJERRHHERRKKERRSPEC_ZSPEC_Z_ERRSPEC_Z_WARNPHOTOZPHOTOZ_ERRW1W1ERRW2W2ERRHOST_RAHOST_DECHOST_DISTHOST_VHOSTHOST_MKHOST_NSAIDHOST_FLAGHOST_SAGA_NAMERHOST_ARCMRHOST_KPCOBJ_NSAIDSATSPROBABILITY_CLASS1RESCALED_PROBABILITY_CLASS1REMOVETELNAMEMASKNAMEZQUALITYSPEC_CLASSSPECOBJIDSPEC_REPEAT
1237655499747360800224.594490222-1.091016197173GALAXY1301856397430876262144026843545614.185512.315511.289910.773310.33330.005534670.005644160.001646980.001599650.001765650.005534670.005644160.001646980.001599650.001765650.2985370.2196610.1593160.1208050.085652114.205912.319111.561610.963710.535155.321471.53167.237464.520648.74093.585348.4484211.8677.739164.7128129.66830.34583329.67780.75785929.67430.23614729.67360.36581229.66920.4499290.6520030.7937260.7414140.5966540.64249814.56040.0055492712.53740.0056441511.92560.0016921711.2320.001627810.67280.0017682418.43030.11437916.75650.038413916.04420.021969415.21050.032178814.6040.062456818.48440.11111616.84870.032926215.84910.010092615.38420.025236314.94970.05366051.00.9291860.4881320.8332470.988473-0.3276280.047358-0.5609560.0478639-0.4840590.0358114-0.505223-0.00365747-0.465421-0.006061390.74434217.659729.96370.81112117.504264.9270.77956111.668176.07190.73713211.680477.4690.68827911.631878.064916.66320.014111612.16490.0016830528.045889.035811.397719.49538110620.63253507812.4510.11311.670.13111.0850.1040.00641111-1.00-9999.0-9999.0-1.0-1.0-1.0-1.0224.594-1.0910228.40381922.0-24.573882229914572900.01486810427760.12284519146914572930.8862785422490.9946832579193NSAned41.23765549975e+18NSA
1237655499747360810224.592809309-1.106758375236STAR352877549978520026843545619.994318.669318.142617.952717.85510.04200860.008460460.00721080.008452550.02158220.04200860.008460460.00721080.008452550.02158220.2986460.2197410.1593740.1208490.085683219.695618.449617.983217.831917.76941.658871.561161.576421.457971.352440.09009060.02017850.02118030.02538340.0487850.132330.1282573.14149e-050.002375880.005122930.04353120.04837910.585260.002186930.1956740.5501220.8037260.708520.2452850.40964519.96460.055176618.66980.008460518.14260.007210817.95040.0089943817.85590.021582819.98580.044082118.66480.022942318.14390.019597717.95660.015099117.83260.023445920.30070.049227319.03420.0089144518.49510.007464318.32040.0094370818.24050.02766391.00.01.00.00.00.07012160.0530215-0.0325870.0100525-0.0159202-0.0233685-0.0437342-0.0368547-0.0384183-0.03784720.6402670.196673115.9890.5178931.72086e-05108.5380.5217030.00158619133.8820.4775640.0531211103.2420.219330.00562169103.58218.97370.0086585118.14310.007210810.7665021.7452518.16846.1402780549319.5864032139999.09999.09999.09999.09999.09999.0-2.94789e-051.03843e-050-1.0-1.0-1.0-1.0-1.0-1.0224.594-1.0910228.40381922.0-24.573882229914572900.9470001281917.82442794493-1-11.15361361361e-090.02SDSSSDSS41.23765549975e+18SDSS
1237655499747361648224.609500242-1.119447842823GALAXY689879124480026843545622.664322.897519.537518.434417.56512.402451.315170.09083610.05790180.09653362.402451.315170.09083610.05790180.09653360.2996550.2204840.1599130.1212570.085972922.364722.67719.377618.313117.47910.1037392.970582.970131.552761.447690.0546913-1000.0-1000.01.068231.098180.7080552.433082.306674.2824512.10125.673090.1024870.0843820.1569970.1948220.050.050.5106680.1346390.098229822.8160.5365623.59930.38117619.53750.090836120.15360.040070519.13070.060277123.27910.55030923.76550.37844521.89510.10480420.16560.040027219.14940.057894223.15210.51383623.83880.40787822.08930.134820.5190.05622319.54910.0844120.00.01.01.01.00.837103-0.345898-0.352387-0.1825410.07439390.00903828-0.106151-0.0737305-0.0801154-0.01704660.051.20527-1.561220.9992210.34540279.94070.8103482.96512-1.563050.2999330.159794107.080.1125370.227741115.98522.64590.14940520.43590.09972893.312475.3164920.492524.791578186825.08870796689999.09999.09999.09999.09999.09999.0-0.0001161385.00134e-0500.6571610.035225-1.0-1.0-1.0-1.0224.594-1.0910228.40381922.0-24.573882229914572901.9426572599716.0508754783-126.2079872647e-130.0-1SDSSSDSS41.23765549975e+18SDSS
1237655499747360964224.573298544-1.13364361536STAR689879129600026843545622.855620.076418.556517.583917.04250.4056840.01991780.008921290.006935150.01226760.4056840.01991780.008921290.006935150.01226760.2990150.2200120.1595710.1209980.085789222.556619.856418.396917.462916.9567166.491.438091.520141.472731.50997-1000.00.0538210.02563220.01966910.04267060.050075224.80024.53165e-050.006991490.05670230.07206690.0499150.04656030.00103030.06482390.05274280.5891410.8226380.9161040.48059122.85880.40256520.08550.019899518.55650.0089212917.58240.0069354717.05210.012262322.73210.35172820.06160.028106718.56780.020166417.59520.014472617.04760.017051723.32820.58666820.46320.024044418.91070.0098460517.95260.007224517.42440.01445931.01.00.00.01.0-0.62284-0.81507-0.0155725-0.00787439-0.0241636-0.0174835-0.0395406-0.0307027-0.0796210.007070010.05171920.0016545971.60020.5818993.86793e-05102.7720.9966360.061196666.29070.9589950.0888293101.8760.1399320.00130554106.55619.39120.011252318.55650.008921290.7600611.7323418.584214.485557170919.983926906615.7450.07115.1490.07814.9270.16-0.0002029151.14885e-050-1.0-1.0-1.0-1.0-1.0-1.0224.594-1.0910228.40381922.0-24.573882229914572902.8429880022823.4897038086-1-12.91974617381e-120.0-1SDSSSDSS41.23765549975e+18SDSS
1237655693551927544224.554562478-1.060181032096STAR352533600015520026843545619.313519.241119.037719.005219.07380.02598420.01081560.0110710.01390710.04339710.02598420.01081560.0110710.01390710.04339710.2982020.2194140.1591380.1206690.08555619.015319.021718.878518.884518.98831.2940910.67311.186811.239491.27830.05051344.63490.02378770.03131650.2046760.00025490.03574620.1028860.01325630.03059732.49264e-050.03275880.1058560.0001264340.02804570.741220.9998010.7450880.9981490.66114919.3170.030557819.23010.010808119.03770.012950819.00140.013904819.08140.045479919.2920.026171419.26330.018959719.04680.014318219.01480.017646519.06440.043161219.63220.030699319.56820.012285419.36030.013962419.33210.01917319.40760.0676990.00.01.00.00.00.00795507-0.00712278-0.04262550.0293627-0.0578933-0.0160306-0.102017-0.0217958-0.1150970.04117020.2012140.00792197129.1190.9997910.1146372.57860.7986130.036188372.24810.983940.0375343103.0510.6588910.000321663108.11419.84510.015661719.03770.01295040.5919111.241619.094213.826030435819.95092326339999.09999.09999.09999.09999.09999.01.370780.0002932620-1.0-1.0-1.0-1.0-1.0-1.0224.594-1.0910228.40381922.0-24.573882229914572903.0034851458524.8157837737-1-11.10945027795e-090.0-1SDSSSDSS41.23765569355e+18SDSS

Some quality/assumption checks

Lets check what ZQUALITY values are present


In [6]:
np.unique(dirtytab['ZQUALITY'])


Out[6]:
<MaskedColumn name='ZQUALITY' dtype='int64' length=6>
-1
2
3
4
5
--

In [7]:
np.sum(dirtytab['ZQUALITY'].mask)


Out[7]:
1612

~1000 of the of the ZQUALITY entries are masked (that's what the "--" means in the np.unique call). Lets check their value...


In [8]:
zqmsked = dirtytab['ZQUALITY'][dirtytab['ZQUALITY'].mask]
zqmsked.mask = False
np.unique(zqmsked)


Out[8]:
<MaskedColumn name='ZQUALITY' dtype='int64' length=1>
1

Hmm, all of the ZQUALITY values that are 1 are masked. That's probably not intended, so we'll un-mask them


In [9]:
dirtytab['ZQUALITY'].mask = False
np.unique(dirtytab['ZQUALITY'])


Out[9]:
<MaskedColumn name='ZQUALITY' dtype='int64' length=6>
-1
1
2
3
4
5

In [10]:
for zq in np.unique(dirtytab['ZQUALITY']):
    print('ZQUALITY={} is {:%}'.format(zq, np.sum(zq==dirtytab['ZQUALITY'])/len(dirtytab)))


ZQUALITY=-1 is 4.125073%
ZQUALITY=1 is 1.650439%
ZQUALITY=2 is 0.157672%
ZQUALITY=3 is 2.115265%
ZQUALITY=4 is 91.355674%
ZQUALITY=5 is 0.595878%

Lets just have a quick look at the CMD


In [11]:
plt.scatter(dirtytab['g'] - dirtytab['r'], dirtytab['r'], c='k', s=1, lw=0, alpha=.25)
plt.xlim(-1,4)
plt.ylim(24,15)


Out[11]:
(24, 15)

Now do the actual business of making the completeness plots


In [12]:
okspec_msk = dirtytab['ZQUALITY']>2

gmr = dirtytab['g'] - dirtytab['r']
gmr_okspec = gmr[okspec_msk]
gmr_badspec = gmr[~okspec_msk]

In [13]:
# not that these are the bin *edges*, so even though there are 11 elements, we end up with 10 bins
completeness_bins = np.linspace(0,2.5, 11)
completeness_bin_centers = (completeness_bins[1:] + completeness_bins[:-1])/2

n_okspec_inbin = np.histogram(gmr_okspec, completeness_bins)[0]
n_badspec_inbin = np.histogram(gmr_badspec, completeness_bins)[0]
n_total_inbin = n_okspec_inbin + n_badspec_inbin

In [14]:
# note that bar takes the *left* edge of a bin, which is why we use completeness_bins[:-1]
plt.bar(completeness_bins[:-1], n_okspec_inbin/n_total_inbin, width=np.diff(completeness_bins), color='#4C72B0')

plt.xlabel('$g-r$')
plt.ylabel('completeness fraction')
plt.tight_layout()



In [15]:
plt.scatter(completeness_bin_centers, n_okspec_inbin/n_total_inbin, color='#4C72B0')

plt.ylim(0,1)
plt.xlabel('$g-r$')
plt.ylabel('completeness fraction')
plt.tight_layout()