In [1]:

    

import holoviews as hv
hv.notebook_extension('bokeh')


    

In [2]:

    

import dask
import dask.dataframe as dd
import pandas as pd
from distributed import Client
client = Client(scheduler_file='/scratch/tmorton/dask/scheduler.json')


    

In [3]:

    

from explorer.dataset import QADataset
from explorer.functors import (Mag, MagDiff, CustomFunctor, DeconvolvedMoments, Column,
                            SdssTraceSize, PsfSdssTraceSizeDiff, HsmTraceSize, Seeing,
                            PsfHsmTraceSizeDiff, CompositeFunctor, StarGalaxyLabeller)

# mag_diff = MagDiff('modelfit_CModel', 'base_PsfFlux')
psf_mag = Mag('base_PsfFlux')
gauss_mag = Mag('base_GaussianFlux')
# seeing = CustomFunctor('0.168*2.35*sqrt(0.5*(ext_shapeHSM_HsmPsfMoments_xx**2 + ext_shapeHSM_HsmPsfMoments_yy**2))')
seeing = Seeing()
label = StarGalaxyLabeller()
funcs = CompositeFunctor({'psf':psf_mag, 'gauss':gauss_mag, 'seeing':seeing, 'label':label})


    

In [4]:

    

# import sys
# sys.path.append('..')

from explorer.catalog import ParquetCatalog, MatchedCatalog, MultiMatchedCatalog
from explorer.utils import get_visits
import glob, re

def parse_visit(filename):
    m = re.search('visit-(\d+)', filename)
    return int(m.group(1))

def get_test_catalog(n_visits=3):
    coadd_cat = ParquetCatalog(['test_data/HSC-G/tract-8766/forced.parq'])
    filenames = glob.glob('test_data/HSC-G/tract-8766/visit*/catalog.parq')
    filenames.sort()
    visit_cats = [ParquetCatalog([f], name=parse_visit(f)) for f in filenames[2:2+n_visits]]
    return MultiMatchedCatalog(coadd_cat, visit_cats, match_registry='test_registry.h5')


    

In [5]:

    

cat = get_test_catalog()


    

In [6]:

    

cat.match()


    

In [7]:

    

data = QADataset(cat, {'psf':psf_mag, 'gauss':gauss_mag, 'seeing':seeing}, flags=['calib_psfUsed', 'qaBad_flag'], client=client)


    

In [8]:

    

%time data.df.head()


    

    




---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<timed eval> in <module>()

~/repositories/qa_explorer/explorer/dataset.py in df(self)
     96     def df(self):
     97         if self._df is None:
---> 98             self._make_df()
     99         return self._df
    100 

~/repositories/qa_explorer/explorer/dataset.py in _make_df(self, **kwargs)
    119 
    120     def _make_df(self, **kwargs):
--> 121         f = CompositeFunctor(self.allfuncs)
    122         if self.is_multi_matched:
    123             kwargs.update(how='all')

~/repositories/qa_explorer/explorer/dataset.py in allfuncs(self)
     83         # Set coordinates and x value
     84         allfuncs.update({'ra':RAColumn(), 'dec': DecColumn(), 
---> 85                          'x':self.xFunc, self.id_name : Column(self.id_name)})
     86 
     87         # Include flags

~/repositories/qa_explorer/explorer/dataset.py in id_name(self)
    114             name = 'patchId'
    115         else:
--> 116             raise ValueError('No id name available (looked for ccdId, patchId)?')
    117 
    118         return name

ValueError: No id name available (looked for ccdId, patchId)?

In [9]:

    

data.ds


    

    
Out[9]:





:Dataset   [ra,dec,x,label,calib_psfUsed,qaBad_flag]   (psf,gauss,seeing,match_distance)

In [10]:

    

%opts Points [width=600, height=400, tools=['hover'], color_index='y', colorbar=True] (cmap='coolwarm', size=4) 
from holoviews.operation import decimate
from explorer.plots import FilterStream
filter_stream = FilterStream()
dmap = data.visit_explore(filter_stream=filter_stream)
dmap


    

    
Out[10]:





  

    

      
      



    




      
    
  
  

    

      
      
      

        
          visit:
        
        

          

            
          
          

        
      
	  
        
        
      
        

          y:
          
          
        
        
        
        
      
      

        
          x_max:
        
        

          

            
          
          

        
      
	  
        
        
      
        

          label:
          
          
        
        
        
        
        
    

In [11]:

    

from explorer.plots import scattersky, multi_scattersky, FilterStream

multi_scattersky(data.ds.groupby('label'), filter_stream=filter_stream)


    

    
Out[11]:





  

    

      
      



    




      
    
  
  

    

      
      
        

          label:
          
          
        
        
        
        
        
    

In [ ]:

    




    

In [ ]:

    




    

In [ ]:

    




    

In [2]:

    

# For getting real data
from explorer.catalog import ParquetCatalog, MultiMatchedCatalog
from explorer.utils import get_visits

def get_coadd(rerun, filt, tracts, unforced=False):
    table = 'unforced' if unforced else 'forced'
    files = ['{}/plots/{}/tract-{}/{}.parq'.format(rerun, filt, t, table) for t in tracts]
    return ParquetCatalog(files)
    
def get_visit(rerun, filt, tract, visit):
    file = '{}/plots/{}/tract-{}/visit-{}/catalog.parq'.format(rerun, filt, tract, visit)
    return ParquetCatalog([file], name=float('{}.{}'.format(tract,visit)))

def get_matched_catalog(rerun, filt, tracts, field):
    coadd_cat = get_coadd(rerun, filt, tracts)
    visits = {t:get_visits(field, t, filt) for t in tracts}
    visit_cats = [get_visit(rerun, filt, t, v) for t in tracts for v in visits[t]]
    return MultiMatchedCatalog(coadd_cat, visit_cats, match_registry='match_registry.h5')


    

In [6]:

    

rerun = '/scratch/tmorton/hscRerun/DM-12043/SSP_DEEP_XMM_LSS'
field = 'DEEP'
tracts = [8766]#, 8767]
filts = ['HSC-G']#, 'HSC-R']#, 'HSC-I', 'HSC-Z', 'HSC-Y']
cats = {f:get_matched_catalog(rerun, f, tracts, 'DEEP') for f in filts}


    

In [7]:

    

cat = cats['HSC-G']


    

In [8]:

    

cat.match()


    

In [9]:

    

from explorer.plots import multi_scattersky
from explorer.dataset import QADataset


    

In [10]:

    

data = {f:QADataset(cats[f], {'psf':psf_mag, 'gauss':gauss_mag, 'seeing':seeing}, flags=['calib_psfUsed', 'qaBad_flag'], client=client) for f in filts}


    

In [11]:

    

%time data['HSC-G'].df.head()


    

    




CPU times: user 1min 11s, sys: 13.4 s, total: 1min 24s
Wall time: 1min 42s






    
Out[11]:







  

    

      

      
calib_psfUsed
      
...
      
match_distance
    
    

      

      
coadd
      
8766.15202
      
8766.15204
      
8766.15206
      
8766.15208
      
8766.1522
      
8766.1523
      
8766.15232
      
8766.15242
      
8766.15244
      
...
      
8766.15244
      
8766.4236
      
8766.42362
      
8766.42366
      
8766.42368
      
8766.42372
      
8766.42374
      
8766.42378
      
8766.4238
      
coadd
    
    

      
id
      

      

      

      

      

      

      

      

      

      

      

      

      

      

      

      

      

      

      

      

      

    
  
  

    

      
38553275716337665
      
False
      
NaN
      
NaN
      
NaN
      
False
      
NaN
      
NaN
      
NaN
      
NaN
      
NaN
      
...
      
NaN
      
NaN
      
0.310316
      
NaN
      
0.216019
      
NaN
      
0.189845
      
NaN
      
0.155790
      
0.0
    
    

      
38553275716337666
      
False
      
NaN
      
NaN
      
NaN
      
NaN
      
NaN
      
NaN
      
NaN
      
NaN
      
NaN
      
...
      
NaN
      
NaN
      
NaN
      
NaN
      
0.121946
      
NaN
      
NaN
      
NaN
      
NaN
      
0.0
    
    

      
38553275716337667
      
False
      
NaN
      
NaN
      
NaN
      
False
      
False
      
NaN
      
NaN
      
NaN
      
False
      
...
      
0.128770
      
NaN
      
0.094909
      
NaN
      
0.128060
      
NaN
      
0.187418
      
NaN
      
0.070961
      
0.0
    
    

      
38553275716337668
      
False
      
NaN
      
NaN
      
NaN
      
False
      
NaN
      
NaN
      
False
      
NaN
      
False
      
...
      
0.092361
      
NaN
      
0.057760
      
NaN
      
0.012509
      
NaN
      
0.030445
      
NaN
      
0.013816
      
0.0
    
    

      
38553275716337669
      
False
      
NaN
      
NaN
      
False
      
False
      
NaN
      
NaN
      
False
      
NaN
      
False
      
...
      
0.143439
      
NaN
      
0.201162
      
NaN
      
0.175009
      
NaN
      
0.313168
      
NaN
      
0.231344
      
0.0
    
  

5 rows × 180 columns

In [12]:

    

data['HSC-G'].ds


    

    
Out[12]:





:Dataset   [ra,dec,x,label,calib_psfUsed,qaBad_flag]   (psf,gauss,seeing,match_distance)

In [13]:

    

%opts Points [width=800, height=600, tools=['hover'], color_index='y', colorbar=True] (cmap='coolwarm', size=4) {+framewise}
from holoviews.operation import decimate
from explorer.plots import FilterStream
filter_stream = FilterStream()
dmap = data['HSC-G'].visit_explore(filter_stream=filter_stream)
dmap


    

    
Out[13]:





  

    

      
      



    




      
    
  
  

    

      
      
      

        
          visit:
        
        

          

            
          
          

        
      
	  
        
        
      
        

          y:
          
          
        
        
        
        
      
      

        
          x_max:
        
        

          

            
          
          

        
      
	  
        
        
      
        

          label:
          
          
        
        
        
        
        
    

In [14]:

    

from explorer.plots import FilterStream, multi_scattersky

multi_scattersky(data['HSC-G'].ds.groupby('label'), filter_stream=filter_stream)


    

    
Out[14]:





  

    

      
      



    




      
    
  
  

    

      
      
        

          label:
          
          
        
        
        
        
        
    

In [16]:

    

%%opts RGB [width=500, height=400]

from explorer.plots import skyplot, skyplot_layout

select_dict = dict(x=(None, 23), label='star')
width=None; height=None
# width=30; height=30

skyplot_layout([skyplot.instance(vdim=vdim, width=width, height=height)(data[f].ds.select(**select_dict)).relabel('{}: {}'.format(f, vdim)) for f in filts for vdim in ['psf', 'gauss']]).cols(2)


    

    
Out[16]:







    

In [34]:

    

d = QADataset(data['HSC-R'].catalog, seeing)


    

In [35]:

    

d.ds


    

    
Out[35]:





:Dataset   [ra,dec,x,label]   (y0,match_distance)

In [ ]:

    

# coverage plot,


    

In [ ]:

    

skyplot()


    

In [26]:

    

from explorer.plots import FlagSetter
import parambokeh

flag_setter = FlagSetter(filter_stream=filter_stream, flags=data['HSC-R'].flags, bad_flags=data['HSC-R'].flags)
parambokeh.Widgets(flag_setter, callback=flag_setter.event, push=False, on_init=True)


    

In [ ]:

    




    

In [ ]:

    




    

In [ ]:

    




    

In [17]:

    

import re
[c for c in data['HSC-G'].catalog.visit_cats[0].columns if re.search('_xx', c) and c in data['HSC-G'].catalog.coadd_cat.columns]


    

    
Out[17]:





['base_SdssShape_xx',
 'base_SdssShape_psf_xx',
 'ext_shapeHSM_HsmPsfMoments_xx',
 'ext_shapeHSM_HsmSourceMoments_xx',
 'slot_Shape_xx']

In [21]:

    

seeing = CustomFunctor('2.35*sqrt(0.5*(ext_shapeHSM_HsmPsfMoments_xx**2 + ext_shapeHSM_HsmPsfMoments_yy**2))')


    

In [20]:

    

seeing.columns


    

    
Out[20]:





['ext_shapeHSM_HsmPsfMoments_xx', 'ext_shapeHSM_HsmPsfMoments_yy']

In [22]:

    

seeing(data['HSC-R'].catalog.coadd_cat)


    

    
Out[22]:





id
38553275716337665    4.989168
38553275716337666    4.990679
38553275716337667    4.580956
38553275716337668    4.574470
38553275716337669    4.578104
38553275716337670    4.401695
38553275716337671    4.403704
38553275716337672    4.593020
38553275716337673    4.199799
38553275716337674    4.602491
38553275716337675    4.603834
38553275716337676    4.656324
38553275716337677    4.002828
38553275716337678    4.282458
38553275716337679    4.587107
38553275716337680    4.322673
38553275716337681    4.283019
38553275716337682    4.591780
38553275716337683    4.388900
38553275716337684    4.603769
38553275716337685    7.179503
38553275716337686    4.573209
38553275716337687    4.588405
38553275716337688    4.400726
38553275716337689    4.580279
38553275716337690    4.603076
38553275716337691    4.606169
38553275716337692    4.595001
38553275716337693    4.389907
38553275716337694    4.504580
                       ...   
38554263558824450    3.038279
38554263558824451    3.038078
38554263558824452    3.045172
38554263558824453    3.040367
38554263558824454    4.372748
38554263558824455    4.368621
38554263558824456    3.042742
38554263558824457    3.041283
38554263558824458    3.034191
38554263558824459    3.030185
38554263558824460    3.034687
38554263558824461    8.281750
38554263558824462    8.283020
38554263558824463    8.287346
38554263558824464    8.285115
38554263558824465    8.549859
38554263558824466    8.559462
38554263558824467    8.463285
38554263558824468    8.469613
38554263558824469    7.941396
38554263558824470    7.931095
38554263558824471    7.920735
38554263558824472    5.248030
38554263558824473    5.256892
38554263558824474    5.245734
38554263558824475    5.255620
38554263558824476    5.318168
38554263558824477    5.321619
38554263558824478    5.162006
38554263558824479    5.158676
Length: 1081434, dtype: float64

In [ ]:

    




    

In [79]:

    

from lsst.daf.persistence import Butler
butler = Butler(rerun)


    

In [83]:

    

dataId = dict(visit=15202, filter='HSC-G', tract=8766, ccd=0)
exp = butler.get('calexp', dataId=dataId)


    

In [100]:

    

import lsst.afw.coord as afwCoord
import lsst.afw.geom as afwGeom
import lsst.afw.image as afwImage

def sub_image(exposure, ra, dec, width=100, height=100):
    
    pos = afwCoord.IcrsCoord(ra*afwGeom.degrees, dec*afwGeom.degrees) 
    wcs = exposure.getWcs()
    pixel = afwGeom.Point2I(wcs.skyToPixel(pos))
    box = afwGeom.Box2I()  # empty
    box.include(pixel)  # one pixel
    box.grow(afwGeom.Extent2I(width, height))
    sub = exposure[box, afwImage.PARENT]
    return sub


    

In [1]:

    

import lsst.afw.display


    

In [2]:

    

lsst.afw.display.setDefaultBackend("ginga")


    

    




---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
/ssd/lsstsw/stack3_20171021/stack/miniconda3-4.3.21-10a4fa6/Linux64/afw/14.0-22-gc76603d1b+1/python/lsst/afw/display/interface.py in setDefaultBackend(backend)
    248         try:
--> 249             _makeDisplayImpl(None, backend)
    250         except Exception as e:

/ssd/lsstsw/stack3_20171021/stack/miniconda3-4.3.21-10a4fa6/Linux64/afw/14.0-22-gc76603d1b+1/python/lsst/afw/display/interface.py in _makeDisplayImpl(display, backend, *args, **kwargs)
     97             # re-raise the final exception
---> 98             raise exc
     99         else:

/ssd/lsstsw/stack3_20171021/stack/miniconda3-4.3.21-10a4fa6/Linux64/afw/14.0-22-gc76603d1b+1/python/lsst/afw/display/interface.py in _makeDisplayImpl(display, backend, *args, **kwargs)
     86         try:
---> 87             _disp = importlib.import_module(dt, **impargs)
     88             break

~/.conda/envs/qa/lib/python3.6/importlib/__init__.py in import_module(name, package)
    125             level += 1
--> 126     return _bootstrap._gcd_import(name[level:], package, level)
    127 

~/.conda/envs/qa/lib/python3.6/importlib/_bootstrap.py in _gcd_import(name, package, level)

~/.conda/envs/qa/lib/python3.6/importlib/_bootstrap.py in _find_and_load(name, import_)

~/.conda/envs/qa/lib/python3.6/importlib/_bootstrap.py in _find_and_load_unlocked(name, import_)

ModuleNotFoundError: No module named 'lsst.afw.display.ginga'

During handling of the above exception, another exception occurred:

RuntimeError                              Traceback (most recent call last)
<ipython-input-2-08869a78ce3d> in <module>()
----> 1 lsst.afw.display.setDefaultBackend("ginga")

/ssd/lsstsw/stack3_20171021/stack/miniconda3-4.3.21-10a4fa6/Linux64/afw/14.0-22-gc76603d1b+1/python/lsst/afw/display/interface.py in setDefaultBackend(backend)
    690 
    691 def setDefaultBackend(backend):
--> 692     Display.setDefaultBackend(backend)
    693 
    694 

/ssd/lsstsw/stack3_20171021/stack/miniconda3-4.3.21-10a4fa6/Linux64/afw/14.0-22-gc76603d1b+1/python/lsst/afw/display/interface.py in setDefaultBackend(backend)
    250         except Exception as e:
    251             raise RuntimeError(
--> 252                 "Unable to set backend to %s: \"%s\"" % (backend, e))
    253 
    254         Display._defaultBackend = backend

RuntimeError: Unable to set backend to ginga: "No module named 'lsst.afw.display.ginga'"

In [3]:

    

import ginga


    

    




---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-3-e9953e78ccbc> in <module>()
----> 1 import ginga

ModuleNotFoundError: No module named 'ginga'

In [101]:

    

ra, dec = (35.522, -4.502)
sub = sub_image(exp, ra, dec)


    

    




---------------------------------------------------------------------------
LengthError                               Traceback (most recent call last)
<ipython-input-101-7e2b563a5a8e> in <module>()
      1 ra, dec = (35.522, -4.502)
----> 2 sub = sub_image(exp, ra, dec)

<ipython-input-100-76f9725b89e3> in sub_image(exposure, ra, dec, width, height)
     11     box.include(pixel)  # one pixel
     12     box.grow(afwGeom.Extent2I(width, height))
---> 13     sub = exposure[box, afwImage.PARENT]
     14     return sub

/ssd/lsstsw/stack3_20171021/stack/miniconda3-4.3.21-10a4fa6/Linux64/afw/14.0-17-gfc02bf727+1/python/lsst/afw/image/exposure/exposureContinued.py in __getitem__(self, imageSlice)
     56 
     57     def __getitem__(self, imageSlice):
---> 58         return self.Factory(self.getMaskedImage()[imageSlice])
     59 
     60     def __setitem__(self, imageSlice, rhs):

/ssd/lsstsw/stack3_20171021/stack/miniconda3-4.3.21-10a4fa6/Linux64/afw/14.0-17-gfc02bf727+1/python/lsst/afw/image/slicing.py in __getitem__(self, imageSlice)
    114 
    115         _checkOrigin(origin)
--> 116         return self.Factory(self, bbox, origin)
    117     cls.__getitem__ = __getitem__
    118 

/ssd/lsstsw/stack3_20171021/stack/miniconda3-4.3.21-10a4fa6/Linux64/afw/14.0-17-gfc02bf727+1/python/lsst/afw/image/slicing.py in Factory(self, *args, **kwargs)
    102         """Return an object of this type
    103         """
--> 104         return cls(*args, **kwargs)
    105     cls.Factory = Factory
    106 

LengthError: 
  File "src/image/Image.cc", line 80, in static lsst::afw::image::ImageBase<PixelT>::_view_t lsst::afw::image::ImageBase<PixelT>::_makeSubView(const Extent2I&, const Extent2I&, const _view_t&) [with PixelT = float; lsst::afw::image::ImageBase<PixelT>::_view_t = boost::gil::image_view<boost::gil::memory_based_2d_locator<boost::gil::memory_based_step_iterator<boost::gil::pixel<float, boost::gil::layout<boost::mpl::vector1<boost::gil::gray_color_t> > >*> > >; lsst::afw::geom::Extent2I = lsst::afw::geom::Extent<int, 2>]
    Box2I(Point2I(6291,-6184),Extent2I(201,201)) doesn't fit in image 2048x4176 {0}
lsst::pex::exceptions::LengthError: 'Box2I(Point2I(6291,-6184),Extent2I(201,201)) doesn't fit in image 2048x4176'