Run the x-match (from 'booq') over real data

Now, as a very final step -- for the time being --, we cross-match some real data.

Two catalogs in optical are to be used, from CS82 and SDSS, both covering a small region at the very central region of the celestial equator: ~1 sq-degree of sky containing 200K+ sources each.


In [1]:
import booq

In [2]:
from booq.io import fits
reload(fits)

cs82 = fits.read('data/cs82.fits')
sdss = fits.read('data/sdss.fits')

In [3]:
cs82.info


  file: data/cs82.fits
  extension: 1
  type: BINARY_TBL
  extname: /home/chbrandt/Desktop/Catalogs/x-match/cs82_x_sdss/cs82/cs82.fits#1
  rows: 289916
  column info:
    OBJID               i8  
    RA                  f8  
    DEC                 f8  
    INDX_LAS            i4  
    INDX_SDSS           i4  
    INDX_SPALL_V5_5_0
                        i4  
    INDX_SPECOBJ_DR10
                        i4  
    Z_SPECOBJ_DR10      f4  
    UKIDSS_SOURCEID     i8  
    SDSS_FLAGS          i8  
    MASKFLAG            i4  
    YPPERRBITS          i4  
    J_1PPERRBITS        i4  
    HPPERRBITS          i4  
    KPPERRBITS          i4  
    DSG_PSF_R           f4  
    DSG_PSF_I           f4  
    DSG_PSF_Z           f4  
    JK                  f4  
    MODELMAG_U          f4  
    MODELMAG_G          f4  
    MODELMAG_R          f4  
    MODELMAG_I          f4  
    MODELMAG_Z          f4  
    FIBERMAG_I          f4  
    FIBER2MAG_I         f4  
    CMODELMAG_R         f4  
    CMODELMAG_I         f4  
    CMODELMAG_Z         f4  
    ZBOSS               f4  
    ZOTHER              f4  
    ZOTHER_SOURCE       i2  
    UMAG                f4  
    GMAG                f4  
    RMAG                f4  
    IMAG                f4  
    ZMAG                f4  
    YMAG                f4  
    JMAG                f4  
    HMAG                f4  
    KMAG                f4  
    UERR                f4  
    GERR                f4  
    RERR                f4  
    IERR                f4  
    ZERR                f4  
    YERR                f4  
    JERR                f4  
    HERR                f4  
    KERR                f4  
    UFNU                f4  
    GFNU                f4  
    RFNU                f4  
    IFNU                f4  
    ZFNU                f4  
    YFNU                f4  
    JFNU                f4  
    HFNU                f4  
    KFNU                f4  
    UFNUERR             f4  
    GFNUERR             f4  
    RFNUERR             f4  
    IFNUERR             f4  
    ZFNUERR             f4  
    YFNUERR             f4  
    JFNUERR             f4  
    HFNUERR             f4  
    KFNUERR             f4  
    YPETROMAG           f4  
    JPETROMAG           f4  
    HPETROMAG           f4  
    KPETROMAG           f4  
    YPETROMAGERR        f4  
    JPETROMAGERR        f4  
    HPETROMAGERR        f4  
    KPETROMAGERR        f4  
    YHALLMAG            f4  
    JHALLMAG            f4  
    HHALLMAG            f4  
    KHALLMAG            f4  
    YHALLMAGERR         f4  
    JHALLMAGERR         f4  
    HHALLMAGERR         f4  
    KHALLMAGERR         f4  
    YHALLTOT            f4  
    JHALLTOT            f4  
    HHALLTOT            f4  
    KHALLTOT            f4  
    YISOMAG             f4  
    JISOMAG             f4  
    HISOMAG             f4  
    KISOMAG             f4  

In [4]:
sdss.info


  file: data/sdss.fits
  extension: 1
  type: BINARY_TBL
  extname: /home/chbrandt/Desktop/Catalogs/photometry/sdss/Stripe82_photo_ch...
  rows: 225908
  column info:
    ObjID               i8  
    ra                  f8  
    dec                 f8  
    u                   f4  
    g                   f4  
    r                   f4  
    i                   f4  
    z                   f4  
    err_u               f4  
    err_g               f4  
    err_r               f4  
    err_i               f4  
    err_z               f4  
    dered_u             f4  
    dered_g             f4  
    dered_r             f4  
    dered_i             f4  
    dered_z             f4  
    c_u                 f4  
    c_g                 f4  
    c_r                 f4  
    c_i                 f4  
    c_z                 f4  
    nChild              i2  
    probPSF             f4  
    insideMask          u1  
    flags               i8  

In [5]:
colsA = {'ra':'RA','dec':'DEC','id':'OBJID'}
catA = cs82.load(columns=['OBJID','RA','DEC'])

In [6]:
colsB = {'id':'ObjID'}
catB = sdss.load(columns=['ObjID','ra','dec'])

In [7]:
from booq.table import aTable
A = aTable(catA.data).to_pandas()

In [8]:
from booq.table import aTable
B = aTable(catB.data).to_pandas()

In [9]:
from booq.pipelines import xmatch_nn
# X-match the catalogs
matched_catalog_parallel = xmatch_nn.xmatch(A, B, columns_A=colsA, columns_B=colsB, parallel_run=True)


Running in parallel
Running in parallel
/home/chbrandt/fido/repos/booq/booq/pipelines/xmatch_nn.py:121: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  B_matched['A_idx'] = df_matched_idx.index

In [10]:
from booq.pipelines import xmatch_nn
# X-match the catalogs
matched_catalog_serial = xmatch_nn.xmatch(A,B,columns_A=colsA, columns_B=colsB)

In [11]:
matched_catalog_parallel.equals(matched_catalog_serial)


Out[11]:
True

In [12]:
matched_catalog_parallel.describe()


Out[12]:
A B AB
OBJID RA DEC ObjID ra dec dist
count 2.899160e+05 289916.000000 289916.000000 2.258970e+05 225897.000000 225897.000000 2.258970e+05
mean 8.647475e+18 177.321630 0.016463 8.647475e+18 176.927110 0.012460 7.569240e-06
std 2.144854e+11 179.439324 0.569329 2.145081e+11 179.433142 0.568618 3.597543e-03
min 8.647475e+18 0.000131 -1.001772 8.647475e+18 0.000131 -1.001772 0.000000e+00
25% 8.647475e+18 0.541974 -0.473527 8.647475e+18 0.541925 -0.477736 0.000000e+00
50% 8.647475e+18 1.054288 0.025700 8.647475e+18 1.052216 0.021727 1.698607e-12
75% 8.647475e+18 359.451641 0.507979 8.647475e+18 359.451261 0.502914 5.155773e-11
max 8.647475e+18 359.999991 0.998209 8.647475e+18 360.000104 0.998209 1.709863e+00