In [1]:
# This changes the current directory to the base saga directory - make sure to run this first!
# This is necessary to be able to import the py files and use the right directories,
# while keeping all the notebooks in their own directory.
import os
import sys

if 'saga_base_dir' not in locals():
    saga_base_dir = os.path.abspath('..')
if saga_base_dir not in sys.path:
    os.chdir(saga_base_dir)

In [2]:
from __future__ import division, print_function

import numpy as np

from astropy import units as u
from astropy import table
from astropy.coordinates import SkyCoord

In [3]:
%matplotlib inline

from matplotlib import pyplot as plt

plt.rcParams['image.cmap'] = 'viridis'
plt.rcParams['figure.figsize'] = [10.0, 6.0]

Hosts

Hosts are identified in the HSC overlap check notebook. For DR1 they are (in "300 kpc" circles):

  • Alice: 75%
  • Othello: 66%
  • Dune: 100%

In [4]:
import hosts

In [5]:
hostobjs = hosts.get_saga_hosts_from_google()
hosts.use_base_catalogs(hostobjs)
hschosts = tuple([h for h in hostobjs if h.name in ('Alice', 'Othello', 'Dune')])
assert len(hschosts) == 3
hschosts


Using cached version of google hosts list from file "hosts_dl.pkl2"
Out[5]:
(<hosts.NSAHost object w/ name 'Alice' AKA: ['NGC4030', 'NSA140594']>,
 <hosts.NSAHost object w/ name 'Dune' AKA: ['NGC5750', 'NSA165536']>,
 <hosts.NSAHost object w/ name 'Othello' AKA: ['NGC5792', 'NSA145729']>)

In [6]:
for h in hschosts:
    h.hscfn = os.path.join('catalogs', 'hsc_pdr1_{}.csv.gz'.format(h.name))

Generate queries

These queries are meant for the HSC casjobs at https://hsc-release.mtk.nao.ac.jp/datasearch/ (which requires login)


In [7]:
hsc_qry_templ="""
SELECT {cols}
FROM {table}
WHERE coneSearch(coord, {hra}, {hdec}, {radius})
"""[1:-1]

bands = 'gri'
cols = 'object_id,ra,dec'.split(',')
for band in bands:
    magcol_to_add = []
    magcol_to_add.append('{}cmodel_mag'.format(band))
    magcol_to_add.append('{}mag_psf'.format(band))
    magcol_to_add.append('{}mag_kron'.format(band))
    magcol_to_add.append('{}mag_aperture10'.format(band))
    magcol_to_add.append('{}mag_aperture20'.format(band))
    magcol_to_add.append('{}mag_aperture30'.format(band))
    for magcol in magcol_to_add:
        cols.append(magcol)
        cols.append(magcol+'_err')
    
    cols.append('{}flux_kron_radius'.format(band))
    cols.append('a_{}'.format(band))
cols = ','.join(cols)

In [8]:
table_to_query='pdr1_wide.forced'

for h in hschosts:
    print('Query for host', h.name, 'which should be saved to', h.hscfn)
    qry = hsc_qry_templ.format(table=table_to_query, cols=cols,
                               hra=h.ra, hdec=h.dec, 
                               radius=h.environsarcmin*u.arcmin.to(u.arcsec))
    print(qry, '\n')


Query for host Alice which should be saved to catalogs/hsc_pdr1_Alice.csv.gz
SELECT object_id,ra,dec,gcmodel_mag,gcmodel_mag_err,gmag_psf,gmag_psf_err,gmag_kron,gmag_kron_err,gmag_aperture10,gmag_aperture10_err,gmag_aperture20,gmag_aperture20_err,gmag_aperture30,gmag_aperture30_err,gflux_kron_radius,a_g,rcmodel_mag,rcmodel_mag_err,rmag_psf,rmag_psf_err,rmag_kron,rmag_kron_err,rmag_aperture10,rmag_aperture10_err,rmag_aperture20,rmag_aperture20_err,rmag_aperture30,rmag_aperture30_err,rflux_kron_radius,a_r,icmodel_mag,icmodel_mag_err,imag_psf,imag_psf_err,imag_kron,imag_kron_err,imag_aperture10,imag_aperture10_err,imag_aperture20,imag_aperture20_err,imag_aperture30,imag_aperture30_err,iflux_kron_radius,a_i
FROM pdr1_wide.forced
WHERE coneSearch(coord, 180.09842901, -1.10007802185, 2691.60292683) 

Query for host Dune which should be saved to catalogs/hsc_pdr1_Dune.csv.gz
SELECT object_id,ra,dec,gcmodel_mag,gcmodel_mag_err,gmag_psf,gmag_psf_err,gmag_kron,gmag_kron_err,gmag_aperture10,gmag_aperture10_err,gmag_aperture20,gmag_aperture20_err,gmag_aperture30,gmag_aperture30_err,gflux_kron_radius,a_g,rcmodel_mag,rcmodel_mag_err,rmag_psf,rmag_psf_err,rmag_kron,rmag_kron_err,rmag_aperture10,rmag_aperture10_err,rmag_aperture20,rmag_aperture20_err,rmag_aperture30,rmag_aperture30_err,rflux_kron_radius,a_r,icmodel_mag,icmodel_mag_err,imag_psf,imag_psf_err,imag_kron,imag_kron_err,imag_aperture10,imag_aperture10_err,imag_aperture20,imag_aperture20_err,imag_aperture30,imag_aperture30_err,iflux_kron_radius,a_i
FROM pdr1_wide.forced
WHERE coneSearch(coord, 221.546363418, -0.222941918742, 2463.36543572) 

Query for host Othello which should be saved to catalogs/hsc_pdr1_Othello.csv.gz
SELECT object_id,ra,dec,gcmodel_mag,gcmodel_mag_err,gmag_psf,gmag_psf_err,gmag_kron,gmag_kron_err,gmag_aperture10,gmag_aperture10_err,gmag_aperture20,gmag_aperture20_err,gmag_aperture30,gmag_aperture30_err,gflux_kron_radius,a_g,rcmodel_mag,rcmodel_mag_err,rmag_psf,rmag_psf_err,rmag_kron,rmag_kron_err,rmag_aperture10,rmag_aperture10_err,rmag_aperture20,rmag_aperture20_err,rmag_aperture30,rmag_aperture30_err,rflux_kron_radius,a_r,icmodel_mag,icmodel_mag_err,imag_psf,imag_psf_err,imag_kron,imag_kron_err,imag_aperture10,imag_aperture10_err,imag_aperture20,imag_aperture20_err,imag_aperture30,imag_aperture30_err,iflux_kron_radius,a_i
FROM pdr1_wide.forced
WHERE coneSearch(coord, 224.594490222, -1.09101619717, 2201.0200465) 

Alice Catalog Experiments


In [9]:
def compute_ap_sb(mags, aperturerad):
    A = 2.5*np.log10(np.pi*(aperturerad.to(u.arcsec).value)**2)
    return np.array(mags + A) * u.mag * u.arcsec**-2

In [10]:
alice = [h for h in hostobjs if h.name=='Alice']
assert len(alice)==1
alice = alice[0]

In [11]:
alice.hsc_cat = table.Table.read(alice.hscfn, format='ascii.csv')
alice.hsc_cat['# object_id'].name = 'object_id'
alice.hsc_cat


Out[11]:
<Table length=1017983>
object_idradecgcmodel_maggcmodel_mag_errgmag_psfgmag_psf_errgmag_krongmag_kron_errgmag_aperture10gmag_aperture10_errgmag_aperture20gmag_aperture20_errgmag_aperture30gmag_aperture30_errgflux_kron_radiusa_grcmodel_magrcmodel_mag_errrmag_psfrmag_psf_errrmag_kronrmag_kron_errrmag_aperture10rmag_aperture10_errrmag_aperture20rmag_aperture20_errrmag_aperture30rmag_aperture30_errrflux_kron_radiusa_ricmodel_magicmodel_mag_errimag_psfimag_psf_errimag_kronimag_kron_errimag_aperture10imag_aperture10_errimag_aperture20imag_aperture20_errimag_aperture30imag_aperture30_erriflux_kron_radiusa_i
int64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64float64
41108545034256385180.08726146-1.40139313589nannannannannannan25.06471440.0391006023nannannannannan0.0901541635nannannannannannan24.72783090.0412764698nannannannannan0.0638859197nannannannannannan24.48708150.0427338928nannannannannan0.0455929674
41108545034256386180.067552591-1.3998198391424.36638070.025417005624.36723330.02510465524.4434910.036865901224.82105640.028030777424.45973970.039433274424.41064830.0562784970.3626508410.089358076523.05382540.0093420818423.05518530.0093056503723.09896090.01554667323.35994340.010574281223.16405110.017145084223.13559720.02500359340.3626508410.063321784122.56298640.0065970616422.56384660.006563130322.58212280.011587762322.85021780.0079151503722.63874050.012542441522.5861130.01768580640.3626508410.0451903641
41108545034256387180.066260836-1.40140046234nannannannannannan26.86730960.18593888nannannannannan0.0893126726nannannannannannan26.50794220.19236137nannannannannan0.0632896125nannannannannannan26.12128070.146682844nannannannannan0.0451674052
41108545034256388180.062107353-1.40140188906nannannannannannan27.71192740.405397326nannannannannan0.089106448nannannannannannan25.90737530.113375291nannannannannan0.0631434768nannannannannannan25.51061060.0877475813nannannannannan0.0450631119
41108545034256389180.049693498-1.4012661411nannannannannannan25.57078930.05887224924.84534840.0602041632nannannan0.0889777243nannannannannannan25.09952350.05297385924.29762650.0514290482nannannan0.0630522594nannannannannannan24.31767270.027647165623.45084950.025119897nannannan0.0449980125
41108545034256390180.043253237-1.40140827291nannannannannannan27.93706130.490639031nannannannannan0.0893190503nannannannannannan26.99664120.283427835nannannannannan0.063294135nannannannannannan26.3389130.164360657nannannannannan0.0451706313
41108545034256391180.039286383-1.40140959674nannannannannannan27.26921080.256713986nannannannannan0.0895407796nannannannannannan27.00355150.282018602nannannannannan0.0634512529nannannannannannan26.27026370.158590898nannannannannan0.0452827625
41108545034256392180.036252837-1.4010373546925.48588750.0881770775nannannannan26.08276370.091524332825.64278790.12314271925.40587430.147951245nan0.089703619525.28874780.0944700837nannannannan25.92677310.1012708425.28386690.11300477425.03073690.135821059nan0.063566647525.26168440.0899407715nannannannan25.74609950.096921801625.35582540.1379971525.12042810.167994276nan0.0453651138
41108545034256393180.032518882-1.39856580545nannan18.69673160.00061761029118.01276970.00063439755519.13195420.00066627265218.27781490.00046395385318.14071270.0004489981221.771185640.0900523365nannan19.26805690.0012005706117.68729210.00077937333819.45633320.0012813195418.17610550.00079831149217.90807720.000691063761.771185640.0638137609nannan21.08266450.0041242595818.18688390.0012497348921.00288770.0035786419219.73479460.0020010622718.86343960.00129805541.771185640.045541469
41108545034256395180.030885964-1.40141237799nannannannannannan27.55519490.354659319nannannannannan0.0898294449nannannannannannan26.72605130.227794886nannannannannan0.063655816nannannannannannan26.83938030.274041593nannannannannan0.0454287492
.......................................................................................................................................
41114064067268755180.03974572-0.37217325556426.00831410.15135496926.45518880.164185837nannan26.88342480.18258266225.96314240.15791834925.82138820.209383711nan0.09106349226.02226830.16274899226.3101330.166398749nannan26.63730810.19205166426.20149990.25804656725.75366970.256119847nan0.064530298126.14251330.21718038626.61704440.233819544nannan26.59382820.22680567225.90047840.24376544425.44765660.242387816nan0.0460528322
41114064067268756180.039428185-0.37212442266826.30806920.18806019426.61364360.19142931725.20132060.22002433227.02358250.21206335726.4039040.24016790125.8427830.2152255621.047824380.091064713926.00705720.17887236226.46043590.18531125824.66837310.18099179926.71803090.20119154526.06952670.22775255125.66766740.2403769491.047824380.064531162427.90097431.1989949927.99863240.84638589625.59597970.5205187229.12567522.362166429.34044085.7939443626.48679920.6296616791.047824380.0460534506
41114064067268757180.006913316-0.36437650227326.63392830.20314486326.8046150.210146025nannan27.06718830.19894143926.71039770.28923779726.47198870.350569844nan0.09105233126.49144940.18640777526.47888180.183775574nannan26.82172780.21214738526.33965870.27546536926.41990470.447708458nan0.064522385626.24348450.22101916426.4552040.225421041nannan26.5352230.22929234826.51959420.45565700526.48712350.659958184nan0.0460471883
41114064067268758180.006532053-0.3644210159825.84725950.14556714926.43973540.153797552nannan26.86976810.16909754326.1381760.17268584726.0715790.243198052nan0.091055050526.35648350.23875351326.96587180.288362652nannan26.94385720.23781478426.43027310.29948604126.21511650.371008396nan0.064524315325.89008520.24555769626.57424740.25383693nannan27.05444530.37268635626.04084210.29615348626.73272320.834563434nan0.0460485667
41114064067268759180.078068335-0.35843365913126.91279410.24720440827.07325940.249939859nannan27.36831470.24020975827.60708620.59706091927.93723491.21957576nan0.08940276526.51176070.20144146726.73537250.206875399nannan26.79762840.19217468826.89527510.42525687827.3055630.94192493nan0.063353456627.21696850.46438065227.58259390.529587626nannan27.30291180.38920605227.59834861.0327236727.22238541.10889304nan0.0452129692
41114064067268760180.077860237-0.35831580873228.13248440.66593849728.28418540.761809409nannan28.82454680.91478353735.1377487616.701111nannan1.254883530.089399591126.6789970.19795751626.69683650.20129270926.54805761.0522857927.01418690.23515912927.69180870.89121383427.30579570.944113911.254883530.063351206526.82197760.26186433426.81605530.260344177nannan27.13594060.33032906126.76582910.476670682nannan1.254883530.0452113599
41114064067268796180.052689031-0.45611174904126.59201430.1774267726.60405540.17986440727.0385590.36749598427.00050160.19444219827.08884810.41966837627.06206320.6154282690.3498240410.095604635828.00298120.76704710728.09916310.83785641230.16592418.9193973528.00528140.662848234nannannannan0.3498240410.067748285827.2925110.49224513827.28177450.48781627428.9599594.0080518728.66840931.6677755128.10136031.9816988727.80120472.255933760.3498240410.0483493879
41114064067268797180.052773969-0.45578261901726.70664220.19596999926.71144290.197278544nannan27.14756970.22111457627.03366090.39643502227.71232411.114003661.277844310.095591008727.95706940.75896483728.09405330.861151755nannan27.50581170.43174260929.53705415.56766462nannan1.277844310.067738629927.28013990.48863738827.30019760.498457044nannan27.2795830.46171268826.87667850.64008462428.03733632.807795051.277844310.0483424962
41114064067268798179.928921948-0.38997524474826.12635230.1333869726.3459930.13441193125.97491460.19991733126.70099070.13826325526.28872110.19080594226.12523650.2484646440.5280883310.091457709726.36009790.21690055726.69054790.23109705726.37864110.41890257626.96971130.25590375126.17930030.247709227.46395871.218102810.5280883310.064809650226.22003170.22937424526.53734210.23616780326.4043560.54817169926.77378460.27295315326.56402780.45085436126.7834530.8299821620.5280883310.0462521985
41114064067268799179.929300362-0.38965701411826.2589340.1575178826.52337840.163253739nannan26.86345860.16468246326.47924420.2328032726.35726740.312323451nan0.091443322626.78192710.31597730527.10417560.331964582nannan27.25601960.32569077627.00306130.51917082127.99768641.95849049nan0.064799457827.9876691.2227630628.15327071.01373911nannan29.10620312.2616767928.30661582.19503188nannannan0.046244923

In [12]:
alice.hsc_cat['coord'] = SkyCoord.guess_from_table(alice.hsc_cat, unit=u.deg)

In [14]:
alice.hsc_cat['r_sb10'] = compute_ap_sb(alice.hsc_cat['rmag_aperture10'], 0.5*u.arcsec)
alice.hsc_cat['r_sb20'] = compute_ap_sb(alice.hsc_cat['rmag_aperture20'], 1.0*u.arcsec)
alice.hsc_cat['r_sb_kron'] = compute_ap_sb(alice.hsc_cat['rmag_kron'], alice.hsc_cat['rflux_kron_radius']*u.arcsec)

In [15]:
fig, (ax1, ax2) = plt.subplots(2,1,figsize=(10, 8))

Ar = alice.hsc_cat['a_r']
ax1.scatter(alice.hsc_cat['r_sb_kron']-Ar, alice.hsc_cat['rflux_kron_radius'], alpha=.05, s=1, lw=0)
ax2.scatter(alice.hsc_cat['r_sb_kron']-Ar, alice.hsc_cat['rcmodel_mag']-Ar, alpha=.05, s=1, lw=0)

for ax in (ax1, ax2):
    ax.set_xlim(20, 30)
    ax.set_xlabel('r_sb_kron')
ax1.set_ylim(0, 5)
ax2.set_ylim(30, 20)

ax1.set_ylabel('rflux_kron_radius')
ax2.set_ylabel('rcmodel_mag')


Out[15]:
<matplotlib.text.Text at 0x110634a10>

Write out


In [22]:
coo = alice.hsc_cat['coord']
del alice.hsc_cat['coord']
alice.hsc_cat.write('catalogs/Alice_HSC_reprocessed.fits', format='fits')
!gzip catalogs/Alice_HSC_reprocessed.fits
!ls -lh catalogs/Alice*
coo = alice.hsc_cat['coord'] = coo


-rw-r--r--@ 1 erik  staff    35M Mar 26  2015 catalogs/Alice in Wonderland_sdss.dat
-rw-r--r--@ 1 erik  staff   2.6M Mar 26  2015 catalogs/Alice in Wonderland_usnob.dat
-rw-r--r--@ 1 erik  staff    35M Mar 26  2015 catalogs/AliceInWonderland_sdss.dat
-rw-r--r--  1 erik  staff   300M Aug 16 16:26 catalogs/Alice_HSC_reprocessed.fits.gz
-rw-r--r--  1 erik  staff   327M Aug 16 16:05 catalogs/Alice_decals_dr3.fits

Matches with SDSS


In [24]:
sdsscat = alice.get_sdss_catalog()
idx, d2d, _ = sdsscat['coord'].match_to_catalog_sky(alice.hsc_cat['coord'])

In [25]:
plt.hist(d2d.arcsec, range=(0, 10), log=True, bins=100, histtype='step');



In [26]:
matched = d2d < 1*u.arcsec
plt.scatter(sdsscat['r'][matched], alice.hsc_cat['rcmodel_mag'][idx[matched]], alpha=.2, lw=0, s=2)
plt.plot([17, 24], [17, 24], c='k')
plt.xlim(17, 24)
plt.ylim(17, 24)
plt.xlabel('SDSS r')
plt.ylabel('HSC r')


Out[26]:
<matplotlib.text.Text at 0x26ab3be90>

Matches with DECaLS


In [27]:
deccat = table.Table.read('catalogs/Alice_decals_dr3.fits')


WARNING: UnitsWarning: '1/deg^2' did not parse as fits unit: Numeric factor not supported by FITS [astropy.units.core]
WARNING: UnitsWarning: 'nanomaggy' did not parse as fits unit: At col 0, Unit 'nanomaggy' not supported by the FITS standard.  [astropy.units.core]
WARNING: UnitsWarning: '1/nanomaggy^2' did not parse as fits unit: Numeric factor not supported by FITS [astropy.units.core]
WARNING: UnitsWarning: '1/arcsec^2' did not parse as fits unit: Numeric factor not supported by FITS [astropy.units.core]

In [28]:
deccat['coord'] = SkyCoord(deccat['ra'], deccat['dec'], unit=u.deg)
idx, d2d, _ = deccat['coord'].match_to_catalog_sky(alice.hsc_cat['coord'])

In [29]:
plt.hist(d2d.arcsec, range=(0, 10), log=True, bins=100, histtype='step');



In [30]:
matched = d2d < 1*u.arcsec
plt.scatter(deccat['mag_r'][matched], alice.hsc_cat['rcmodel_mag'][idx[matched]], alpha=.2, lw=0, s=2)
plt.plot([17, 24], [17, 24], c='k')
plt.xlim(17, 24)
plt.ylim(17, 24)
plt.xlabel('DECaLS r')
plt.ylabel('HSC r')


Out[30]:
<matplotlib.text.Text at 0x25217d210>