In [1]:
# Initial setup...
import numpy as np
import pandas as pd
import math
import matplotlib.pyplot as plt
import easyaccess as ea
%matplotlib inline
In [2]:
# What zeropoints are available in the DESDM database?...
ea_conn=ea.connect('desoper')
query = """
select distinct z.source, z.version, z.tag
from prod.zeropoint@desoper z
order by z.source, z.version, z.tag
"""
df_zps_provenance = ea_conn.query_to_pandas(query)
ea_conn.close()
# Print out results of the query...
df_zps_provenance
Out[2]:
In [3]:
# Grab all expCalib Y4N_FIRSTCUT zeropoints...
ea_conn=ea.connect('desoper')
query = """
select z.*
from prod.zeropoint@desoper z
where z.source='expCalib' and z.version='v1.0' and z.tag='Y4N_FIRSTCUT'
order by z.expnum, z.ccdnum
"""
df_zps_expCalib = ea_conn.query_to_pandas(query)
ea_conn.close()
# Print out the first 10 results of the query...
df_zps_expCalib.head(10)
Out[3]:
In [4]:
# Grab all FGCM Y4A1_FINALCUT zeropoints...
ea_conn=ea.connect('desoper')
query = """
select z.*
from prod.zeropoint@desoper z
where z.source='FGCM' and z.version='y4a1_v1.5' and z.tag='Y4A1_FINALCUT'
order by z.expnum, z.ccdnum
"""
df_zps_fgcm = ea_conn.query_to_pandas(query)
ea_conn.close()
# Print out the first 10 results of the query...
df_zps_fgcm.head(10)
Out[4]:
In [5]:
# Match the FGCM and expCalib dataframes by EXPNUM and CCDNUM:
df_zps_merged = pd.merge(df_zps_fgcm, df_zps_expCalib, on=['EXPNUM','CCDNUM'], how='inner')
In [6]:
# Look at the first ten matches...
df_zps_merged.head(10)
Out[6]:
In [7]:
# Create a column for the matched dataframe showing the difference between the
# FGCM Y4A1_FINALCUT zeropoint (MAG_ZERO_x) and the expCalib Y4A1_FIRSTCUT zeropoint (MAG_ZERO_y)
# for each EXPNUM,CCDNUM match...
df_zps_merged.loc[:,'DELTA_MAG_ZERO'] = df_zps_merged.loc[:,'MAG_ZERO_x'] - (df_zps_merged.loc[:,'MAG_ZERO_y'])
In [8]:
# Look at the first 10 DELTA_MAG_ZEROs...
# Don't forget: the "_x" values are the FGCM Y4A1_FINALCUT values, and
# the "_y" values are the expCalib Y4N_FIRSTCUT values.
# (BAND_x is the same as BAND_y, so we only list out here BAND_x.)
df_zps_merged[['EXPNUM', 'CCDNUM','BAND_x','MAG_ZERO_x', 'MAG_ZERO_y','DELTA_MAG_ZERO']].head(10)
Out[8]:
In [9]:
# Plot the histogram of DELTA_MAG_ZERO (= MAG_ZERO_x-MAG_ZERO_y = MAG_ZERO_fgcm - MAG_ZERO_expcalib)
# for all EXPNUM,CCDNUM matches in all bands:
df_zps_merged.hist('DELTA_MAG_ZERO')
# Note: there are signal values at c. -10000 to exclude:
Out[9]:
In [10]:
# Plot the histogram of DELTA_MAG_ZERO (= MAG_ZERO_x-MAG_ZERO_y = MAG_ZERO_fgcm - MAG_ZERO_expcalib)
# for all EXPNUM,CCDNUM matches for EACH band:
bandList = ['g','r','i','z','Y']
for band in bandList:
mask = ( (df_zps_merged.BAND_y==band) & (df_zps_merged.DELTA_MAG_ZERO > -1.) & (df_zps_merged.DELTA_MAG_ZERO > -1.) )
median = df_zps_merged[mask].DELTA_MAG_ZERO.median()
mean = df_zps_merged[mask].DELTA_MAG_ZERO.mean()
stddev = df_zps_merged[mask].DELTA_MAG_ZERO.std()
print """%1s %6.3f %6.3f %6.3f""" % (band, median, mean, stddev)
df_zps_merged[df_zps_merged.BAND_y==band].hist('DELTA_MAG_ZERO', bins=41, range=[-0.25,0.25])
Grab files from Alex's directory...
In [12]:
fname_y3a2_zps = '/home/s1/kadrlica/projects/bliss/proc/v5/y3a2_zeropoints.csv'
df_y3a2_zps = pd.read_csv(fname_y3a2_zps)
df_y3a2_zps.head(10)
Out[12]:
In [13]:
fname_bliss_zps = '/home/s1/kadrlica/projects/bliss/proc/v5/bliss_zeropoints.csv'
df_bliss_zps = pd.read_csv(fname_bliss_zps)
df_bliss_zps.head(10)
Out[13]:
In [14]:
df_bliss_zps.rename(columns={'expnum': 'EXPNUM',
'ccdnum': 'CCDNUM',
'band': 'BAND',
'mag_zero': 'MAG_ZERO',
'sigma_mag_zero': 'SIGMA_MAG_ZERO',
'flag': 'FLAG'
}, inplace=True)
df_bliss_zps.head(10)
Out[14]:
In [15]:
# Match the Y3A2 and BLISS dataframes by EXPNUM and CCDNUM:
df_y3a2_bliss_zps_merged = pd.merge(df_y3a2_zps, df_bliss_zps, on=['EXPNUM','CCDNUM'], how='inner')
df_y3a2_bliss_zps_merged.head(10)
Out[15]:
In [16]:
# Create a column for the matched dataframe showing the difference between the
# FGCM Y3A2_FINALCUT zeropoint (MAG_ZERO_x) and the expCalib BLISS zeropoint (MAG_ZERO_y)
# for each EXPNUM,CCDNUM match...
df_y3a2_bliss_zps_merged.loc[:,'DELTA_MAG_ZERO'] = df_y3a2_bliss_zps_merged.loc[:,'MAG_ZERO_x'] + (df_y3a2_bliss_zps_merged.loc[:,'MAG_ZERO_y'])
df_y3a2_bliss_zps_merged.head(10)
Out[16]:
In [17]:
# Look at the first 10 DELTA_MAG_ZEROs...
# Don't forget: the "_x" values are the Y3A2 values, and
# the "_y" values are the BLISS values.
# (BAND_x is the same as BAND_y, so we only list out here BAND_x.)
df_y3a2_bliss_zps_merged[['EXPNUM', 'CCDNUM','BAND_x','MAG_ZERO_x', 'MAG_ZERO_y','DELTA_MAG_ZERO']].head(10)
Out[17]:
In [18]:
# Plot the histogram of DELTA_MAG_ZERO (= MAG_ZERO_x-MAG_ZERO_y = MAG_ZERO_fgcm_y3a2 - MAG_ZERO_bliss)
# for all EXPNUM,CCDNUM matches in all bands:
df_y3a2_bliss_zps_merged.hist('DELTA_MAG_ZERO')
# Note: there are signal values at c. -10000 to exclude:
Out[18]:
In [19]:
# Plot the histogram of DELTA_MAG_ZERO (= MAG_ZERO_x-MAG_ZERO_y = MAG_ZERO_fgcm_y3a2 - MAG_ZERO_expcalib_bliss)
# for all EXPNUM,CCDNUM matches for EACH band:
bandList = ['g','r','i','z','Y']
print 'band median mean stddev'
for band in bandList:
mask = ( (df_y3a2_bliss_zps_merged.BAND_y==band) & (df_y3a2_bliss_zps_merged.DELTA_MAG_ZERO > -1.) & (df_y3a2_bliss_zps_merged.DELTA_MAG_ZERO < 1.) )
median = df_y3a2_bliss_zps_merged[mask].DELTA_MAG_ZERO.median()
mean = df_y3a2_bliss_zps_merged[mask].DELTA_MAG_ZERO.mean()
stddev = df_y3a2_bliss_zps_merged[mask].DELTA_MAG_ZERO.std()
print """%1s %6.3f %6.3f %6.3f""" % (band, median, mean, stddev)
df_y3a2_bliss_zps_merged[df_y3a2_bliss_zps_merged.BAND_y==band].hist('DELTA_MAG_ZERO', bins=41, range=[-0.25,0.25])
In [20]:
# Match the expCalib Y4A1_FIRSTCUT and expCalib BLISS dataframes by EXPNUM and CCDNUM:
df_expCalib_y4a1_bliss_zps_merged = pd.merge(df_zps_expCalib, df_bliss_zps, on=['EXPNUM','CCDNUM'], how='inner')
df_expCalib_y4a1_bliss_zps_merged.head(10)
Out[20]:
In [21]:
# Create a column for the matched dataframe showing the difference between the
# expCalib Y4A1_FIRSTCUT zeropoint (MAG_ZERO_x) and the expCalib BLISS zeropoint (MAG_ZERO_y)
# for each EXPNUM,CCDNUM match...
df_expCalib_y4a1_bliss_zps_merged.loc[:,'DELTA_MAG_ZERO'] = df_expCalib_y4a1_bliss_zps_merged.loc[:,'MAG_ZERO_x'] + (df_expCalib_y4a1_bliss_zps_merged.loc[:,'MAG_ZERO_y'])
df_expCalib_y4a1_bliss_zps_merged.head(10)
Out[21]:
In [22]:
# Look at the first 10 DELTA_MAG_ZEROs...
# Don't forget: the "_x" values are the expCalib Y4A1_FIRSTCUT values, and
# the "_y" values are the expCalib BLISS values.
# (BAND_x is the same as BAND_y, so we only list out here BAND_x.)
df_expCalib_y4a1_bliss_zps_merged[['EXPNUM', 'CCDNUM','BAND_x','MAG_ZERO_x', 'MAG_ZERO_y','DELTA_MAG_ZERO']].head(10)
Out[22]:
In [23]:
# Plot the histogram of DELTA_MAG_ZERO (= MAG_ZERO_x-MAG_ZERO_y = MAG_ZERO_expCalib_y4a1 - MAG_ZERO_expcalib_bliss)
# for all EXPNUM,CCDNUM matches in all bands:
df_expCalib_y4a1_bliss_zps_merged.hist('DELTA_MAG_ZERO')
# Note: there are signal values at c. -10000 to exclude:
Out[23]:
In [24]:
# Plot the histogram of DELTA_MAG_ZERO (= MAG_ZERO_x-MAG_ZERO_y = (= MAG_ZERO_x-MAG_ZERO_y = MAG_ZERO_expCalib_y4a1 - MAG_ZERO_expcalib_bliss)
# for all EXPNUM,CCDNUM matches for EACH band:
bandList = ['g','r','i','z','Y']
print 'band median mean stddev'
for band in bandList:
mask = ( (df_expCalib_y4a1_bliss_zps_merged.BAND_y==band) & (df_expCalib_y4a1_bliss_zps_merged.DELTA_MAG_ZERO > -1.) & (df_expCalib_y4a1_bliss_zps_merged.DELTA_MAG_ZERO < 1.) )
median = df_expCalib_y4a1_bliss_zps_merged[mask].DELTA_MAG_ZERO.median()
mean = df_expCalib_y4a1_bliss_zps_merged[mask].DELTA_MAG_ZERO.mean()
stddev = df_expCalib_y4a1_bliss_zps_merged[mask].DELTA_MAG_ZERO.std()
print """%1s %6.3f %6.3f %6.3f""" % (band, median, mean, stddev)
df_expCalib_y4a1_bliss_zps_merged[df_expCalib_y4a1_bliss_zps_merged.BAND_y==band].hist('DELTA_MAG_ZERO', bins=41, range=[-0.25,0.25])
In [27]:
# Same as above sets of plots, but with smaller range on the x-axis
# AND tighter restriction on outliers:
# Plot the histogram of DELTA_MAG_ZERO (= MAG_ZERO_x-MAG_ZERO_y = (= MAG_ZERO_x-MAG_ZERO_y = MAG_ZERO_expCalib_y4a1 - MAG_ZERO_expcalib_bliss)
# for all EXPNUM,CCDNUM matches for EACH band:
bandList = ['g','r','i','z','Y']
print 'band median mean stddev'
for band in bandList:
mask = ( (df_expCalib_y4a1_bliss_zps_merged.BAND_y==band) & (df_expCalib_y4a1_bliss_zps_merged.DELTA_MAG_ZERO > -0.1) & (df_expCalib_y4a1_bliss_zps_merged.DELTA_MAG_ZERO < 0.1) )
median = df_expCalib_y4a1_bliss_zps_merged[mask].DELTA_MAG_ZERO.median()
mean = df_expCalib_y4a1_bliss_zps_merged[mask].DELTA_MAG_ZERO.mean()
stddev = df_expCalib_y4a1_bliss_zps_merged[mask].DELTA_MAG_ZERO.std()
print """%1s %6.3f %6.3f %6.3f""" % (band, median, mean, stddev)
df_expCalib_y4a1_bliss_zps_merged[df_expCalib_y4a1_bliss_zps_merged.BAND_y==band].hist('DELTA_MAG_ZERO', bins=51, range=[-0.025,0.025])
In [ ]: