In [3]:
import numpy as np
from astropy.table import Table
from astropy.io.ascii import InconsistentTableError
from astropy.io.fits import getheader
import matplotlib.pyplot as plt
from scipy import stats
import os
from glob import glob
from scipy.optimize import least_squares
from tqdm import tqdm_notebook

In [4]:
def model2(theta, mag, y, zp=22.5, sigdet=5, k=1):
    a, b, maglim = theta 
    teff = np.exp(a + b * (maglim - 21.)) 

    # Compute flux/limit. 
    F = 10**(-0.4 * (mag - zp)) 
    Flim = 10**(-0.4 * (maglim - zp)) 

    # Compute noise. 
    Fnoise = (Flim / sigdet)**2 * k * teff - Flim
    magerr = 2.5 / np.log(10.) * np.sqrt((1. + Fnoise / F) / (F * k * teff))

    return np.sum(np.abs(y - magerr))
    #return magerr - y

In [5]:
datapath = f'{os.environ["HOME"]}/Projects/planckClusters/data/proc2'
files = glob(f'{datapath}/PSZ**/*K_cal.cat')

In [6]:
files


Out[6]:
['/home/boada/Projects/planckClusters/data/proc2/PSZ1_G148.20+23.49/PSZ1_G148.20+23.49K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G142.17+37.28/PSZ1_G142.17+37.28K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G137.58+53.88/PSZ2_G137.58+53.88K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G286.25+62.68/PSZ1_G286.25+62.68K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G031.91+67.94/PSZ1_G031.91+67.94K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G244.48+34.06/PSZ1_G244.48+34.06K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G096.44-10.40/PSZ1_G096.44-10.40K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G071.67-42.76/PSZ2_G071.67-42.76K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G105.91-38.39/PSZ1_G105.91-38.39K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G108.90-52.04/PSZ1_G108.90-52.04K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G138.60-10.85/PSZ1_G138.60-10.85K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G189.79-37.25/PSZ2_G189.79-37.25K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G055.83-41.64/PSZ1_G055.83-41.64K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G093.04-32.38/PSZ2_G093.04-32.38K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G102.86-31.07/PSZ1_G102.86-31.07K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G146.00-49.42/PSZ1_G146.00-49.42K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G185.45-32.01/PSZ2_G185.45-32.01K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G206.64-21.17/PSZ1_G206.64-21.17K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G092.46-35.22/PSZ2_G092.46-35.22K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G127.35-10.69/PSZ2_G127.35-10.69K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G162.30-26.92/PSZ1_G162.30-26.92K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G127.55+20.84/PSZ1_G127.55+20.84K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G100.45+16.79/PSZ2_G100.45+16.79K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G107.41-09.57/PSZ2_G107.41-09.57K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G166.27-24.71/PSZ2_G166.27-24.71K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G166.27-25.02/PSZ2_G166.27-25.02K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G153.56+36.82/PSZ2_G153.56+36.82K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G044.83+10.02/PSZ2_G044.83+10.02K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G137.24+53.93/PSZ2_G137.24+53.93K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G310.81+83.91/PSZ2_G310.81+83.91K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G173.76+22.92/PSZ2_G173.76+22.92K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G210.71+63.08/PSZ2_G210.71+63.08K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G136.31+54.67/PSZ2_G136.31+54.67K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G320.94+83.69/PSZ2_G320.94+83.69K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G048.47+34.86/PSZ2_G048.47+34.86K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G163.22-26.48/PSZ2_G163.22-26.48K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G145.25+50.84/PSZ2_G145.25+50.84K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G125.55+32.72/PSZ2_G125.55+32.72K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G328.96+71.97/PSZ2_G328.96+71.97K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G146.88+17.13/PSZ2_G146.88+17.13K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G120.76+44.14/PSZ2_G120.76+44.14K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G305.76+44.79/PSZ2_G305.76+44.79K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G022.03+17.75/PSZ2_G022.03+17.75K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G104.15-38.85/PSZ2_G104.15-38.85K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G096.43-20.89/PSZ2_G096.43-20.89K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G112.07-39.86/PSZ2_G112.07-39.86K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G192.40-67.89/PSZ2_G192.40-67.89K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G092.11-33.73/PSZ2_G092.11-33.73K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G165.39+09.22/PSZ2_G165.39+09.22K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G086.35-13.94/PSZ2_G086.35-13.94K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G131.27-25.82/PSZ2_G131.27-25.82K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G144.84-35.16/PSZ2_G144.84-35.16K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G166.56-17.69/PSZ2_G166.56-17.69K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G183.92+16.36/PSZ2_G183.92+16.36K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G093.41-16.26/PSZ2_G093.41-16.26K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G128.15-24.17/PSZ2_G128.15-24.17K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G139.72-17.13/PSZ2_G139.72-17.13K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G230.28-28.57/PSZ2_G230.28-28.57K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G216.25+10.10/PSZ2_G216.25+10.10K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G032.12-14.96/PSZ2_G032.12-14.96K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G228.35-66.31/PSZ2_G228.35-66.31K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G098.38+77.22/PSZ2_G098.38+77.22K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G237.68+57.83/PSZ2_G237.68+57.83K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G112.54+59.53/PSZ2_G112.54+59.53K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G270.88+37.23/PSZ2_G270.88+37.23K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G254.52+62.52/PSZ2_G254.52+62.52K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G179.33-22.22/PSZ2_G179.33-22.22K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G191.57+58.88/PSZ2_G191.57+58.88K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G153.56+36.23/PSZ1_G153.56+36.23K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G093.71-30.90/PSZ2_G093.71-30.90K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G153.41+36.58/PSZ1_G153.41+36.58K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G206.45+13.89/PSZ1_G206.45+13.89K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G249.01+73.75/PSZ1_G249.01+73.75K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G318.46+83.79/PSZ2_G318.46+83.79K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G118.06+31.10/PSZ1_G118.06+31.10K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G185.93-31.21/PSZ1_G185.93-31.21K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ1_G142.38+22.82/PSZ1_G142.38+22.82K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G135.94-68.22/PSZ2_G135.94-68.22K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G203.32+08.91/PSZ2_G203.32+08.91K_cal.cat',
 '/home/boada/Projects/planckClusters/data/proc2/PSZ2_G032.31+66.07/PSZ2_G032.31+66.07K_cal.cat']

In [7]:
filters = ['g', 'r', 'i', 'z', 'K']
limits = {}
for filt in tqdm_notebook(filters, desc='filters'):
    maglim = []
    for f in tqdm_notebook(files, desc='files', leave=False):
        new_f = f'{f[:-9]}{filt}_cal.cat'        
        if os.path.isfile(new_f):
            f = new_f
        else:
            maglim.append(-1)
            continue
        cat = Table.read(f, format='ascii')
        # filter out non-detections
        cat = cat[cat['MAG_AUTO'] < 30]
        # filter out the stars
        cat = cat[cat['CLASS_STAR']< 0.8]
        if len(cat) < 100:
            maglim.append(-1)
        else:
            fits_file = f'{f[:-8]}.fits'
            hdr = getheader(fits_file)
            zp = hdr['magzero']
            result = least_squares(model2, (4, 1, 20), loss='cauchy', f_scale=np.std(cat['MAGERR_AUTO']),
                           args=(cat['MAG_AUTO'], cat['MAGERR_AUTO']), kwargs={'zp':zp, 'sigdet':3}, 
                           bounds=([3,0.5, 15], [5,2, 30]))
            maglim.append(result['x'][2])
    limits[filt] = maglim




In [13]:
import pandas as pd

In [14]:
df = pd.DataFrame(limits)

In [15]:
df.describe()


Out[15]:
g r i z K
count 80.000000 80.000000 80.000000 80.000000 80.000000
mean 16.249540 16.059091 16.026271 15.470404 21.318638
std 11.380659 11.255119 11.226700 11.181487 0.681492
min -1.000000 -1.000000 -1.000000 -1.000000 18.801006
25% -1.000000 -1.000000 -1.000000 -1.000000 20.814546
50% 23.446769 23.187573 23.237844 22.704416 21.491795
75% 23.919518 23.709339 23.555344 23.126283 21.823434
max 24.669797 24.345082 24.175711 24.127846 22.167039

In [16]:
df.replace(-1, np.nan).describe()


Out[16]:
g r i z K
count 56.000000 56.000000 56.000000 55.000000 80.000000
mean 23.642200 23.370131 23.323245 22.956951 21.318638
std 0.743882 0.738106 0.569614 0.481309 0.681492
min 20.533843 20.048669 21.216852 21.450051 18.801006
25% 23.377925 23.114832 23.107482 22.668354 20.814546
50% 23.761087 23.465553 23.366265 22.992246 21.491795
75% 24.051688 23.797146 23.699036 23.216523 21.823434
max 24.669797 24.345082 24.175711 24.127846 22.167039

In [17]:
fields = [f'{f.rsplit("/")[-1][:-9]}' for f in files]

In [18]:
df['fields'] = fields

In [19]:
df


Out[19]:
g r i z K fields
0 23.552037 23.238198 23.253262 23.064814 21.928263 PSZ1_G148.20+23.49
1 23.462874 23.121495 23.233203 23.215255 21.954919 PSZ1_G142.17+37.28
2 23.733361 23.391225 23.486331 22.975911 21.681539 PSZ2_G137.58+53.88
3 23.333869 23.336319 23.427244 22.946449 22.118406 PSZ1_G286.25+62.68
4 23.603999 23.494681 23.460437 22.948528 22.065140 PSZ1_G031.91+67.94
5 -1.000000 -1.000000 -1.000000 -1.000000 21.324506 PSZ1_G244.48+34.06
6 23.307893 23.014027 23.315319 23.391770 21.996544 PSZ1_G096.44-10.40
7 24.032375 23.783068 23.677466 23.034849 22.066333 PSZ2_G071.67-42.76
8 24.088340 23.667774 23.517554 22.971916 22.090588 PSZ1_G105.91-38.39
9 23.907309 23.857055 23.826679 -1.000000 21.482872 PSZ1_G108.90-52.04
10 24.206176 23.777907 23.764127 23.160876 22.155448 PSZ1_G138.60-10.85
11 23.327755 22.005806 21.216852 21.955558 21.952371 PSZ2_G189.79-37.25
12 23.815991 23.408164 23.359358 22.834768 21.637327 PSZ1_G055.83-41.64
13 23.683143 23.228242 22.806845 22.331975 22.008086 PSZ2_G093.04-32.38
14 24.136263 23.713542 23.679194 23.153346 22.101006 PSZ1_G102.86-31.07
15 23.901936 23.676050 23.545750 23.045908 22.069240 PSZ1_G146.00-49.42
16 -1.000000 -1.000000 -1.000000 -1.000000 20.706868 PSZ2_G185.45-32.01
17 -1.000000 -1.000000 -1.000000 -1.000000 20.592505 PSZ1_G206.64-21.17
18 22.401153 21.709094 22.501569 22.416918 22.026540 PSZ2_G092.46-35.22
19 23.824819 23.436425 23.327462 23.305568 22.161771 PSZ2_G127.35-10.69
20 23.495649 23.333892 23.260735 23.217791 22.167039 PSZ1_G162.30-26.92
21 23.285940 22.993146 22.891321 23.051818 19.423085 PSZ1_G127.55+20.84
22 23.498881 23.281216 23.296631 23.138347 21.392570 PSZ2_G100.45+16.79
23 24.508927 24.255537 23.284348 24.127846 20.554182 PSZ2_G107.41-09.57
24 23.997023 23.675831 23.841795 23.608635 21.441228 PSZ2_G166.27-24.71
25 23.056980 22.793571 23.283847 23.313126 20.302829 PSZ2_G166.27-25.02
26 24.669797 23.890716 23.516643 22.674915 21.887894 PSZ2_G153.56+36.82
27 23.303081 23.088222 22.928222 22.860862 18.801006 PSZ2_G044.83+10.02
28 24.496035 24.001895 23.967519 22.526523 21.695187 PSZ2_G137.24+53.93
29 21.486490 22.285564 21.658246 23.168663 21.701451 PSZ2_G310.81+83.91
... ... ... ... ... ... ...
50 -1.000000 -1.000000 -1.000000 -1.000000 20.805966 PSZ2_G131.27-25.82
51 -1.000000 -1.000000 -1.000000 -1.000000 20.728862 PSZ2_G144.84-35.16
52 -1.000000 -1.000000 -1.000000 -1.000000 20.788421 PSZ2_G166.56-17.69
53 -1.000000 -1.000000 -1.000000 -1.000000 20.814900 PSZ2_G183.92+16.36
54 -1.000000 -1.000000 -1.000000 -1.000000 20.602994 PSZ2_G093.41-16.26
55 -1.000000 -1.000000 -1.000000 -1.000000 20.813484 PSZ2_G128.15-24.17
56 -1.000000 -1.000000 -1.000000 -1.000000 20.897196 PSZ2_G139.72-17.13
57 -1.000000 -1.000000 -1.000000 -1.000000 20.518969 PSZ2_G230.28-28.57
58 -1.000000 -1.000000 -1.000000 -1.000000 20.622390 PSZ2_G216.25+10.10
59 23.577113 23.424609 23.721942 23.187969 21.130331 PSZ2_G032.12-14.96
60 -1.000000 -1.000000 -1.000000 -1.000000 20.362136 PSZ2_G228.35-66.31
61 24.403918 24.105055 24.030546 23.280531 21.328997 PSZ2_G098.38+77.22
62 -1.000000 -1.000000 -1.000000 -1.000000 20.956816 PSZ2_G237.68+57.83
63 -1.000000 -1.000000 -1.000000 -1.000000 20.540980 PSZ2_G112.54+59.53
64 -1.000000 -1.000000 -1.000000 -1.000000 19.531491 PSZ2_G270.88+37.23
65 -1.000000 -1.000000 -1.000000 -1.000000 19.974693 PSZ2_G254.52+62.52
66 -1.000000 -1.000000 -1.000000 -1.000000 20.997525 PSZ2_G179.33-22.22
67 -1.000000 -1.000000 -1.000000 -1.000000 20.806465 PSZ2_G191.57+58.88
68 23.392610 23.292547 23.242485 22.776855 21.648347 PSZ1_G153.56+36.23
69 -1.000000 -1.000000 -1.000000 -1.000000 20.780143 PSZ2_G093.71-30.90
70 23.859063 23.514430 23.373171 22.495223 21.814751 PSZ1_G153.41+36.58
71 23.305324 22.877618 22.303899 22.178162 21.533171 PSZ1_G206.45+13.89
72 22.920207 22.871203 22.985992 22.116866 21.598057 PSZ1_G249.01+73.75
73 20.533843 20.048669 22.450358 22.661200 21.521523 PSZ2_G318.46+83.79
74 22.250233 22.335039 22.930601 22.793787 21.572631 PSZ1_G118.06+31.10
75 23.430664 23.094840 23.092001 22.585222 21.605050 PSZ1_G185.93-31.21
76 23.108123 22.836460 22.902435 22.733917 21.849481 PSZ1_G142.38+22.82
77 -1.000000 -1.000000 -1.000000 -1.000000 20.914606 PSZ2_G135.94-68.22
78 -1.000000 -1.000000 -1.000000 -1.000000 20.962021 PSZ2_G203.32+08.91
79 -1.000000 -1.000000 -1.000000 -1.000000 20.586275 PSZ2_G032.31+66.07

80 rows × 6 columns


In [20]:
df = df.replace(-1, np.nan)

In [22]:
df.to_csv('./psz_depths.csv', index=False)

In [ ]: