In [17]:
import healpy as hp
import numpy as np
import vaex as vx
import pylab as plt
%matplotlib inline

In [2]:
tgas = vx.open("/Users/users/breddels/tgas/tgas.hdf5")
gaia = vx.open("/Users/users/breddels/gaia/cdn.gea.esac.esa.int/Gaia/gaia_source/gaia-dr1-unshuffled-b.hdf5")

In [3]:
tgas._stat(selection=[False, "parallax < 0"])


Out[3]:
array([ 2057050.,    30840.])

In [9]:
%%time
c=gaia.healpix_count(binby=gaia.col.phot_g_mean_mag, limits=[[10, 23]])
print("{:,}".format(c.sum()))
print c.shape


1,142,188,884.0
(786432, 128)
CPU times: user 36.7 s, sys: 1min 35s, total: 2min 11s
Wall time: 50.8 s

In [46]:
%%time
gaia.healpix_plot(f="log1p", smooth=0.3,
                  rotation=(-0,0,0),
                  grid=c[:,-20:-1].sum(axis=1), figsize=(14,8), healpix_level=8, colormap="binary_r", what="mean(matched_observations)")
#gaia.healpix_plot(f="log1p", grid=c[:,-43], figsize=(14,8), healpix_level=8, colormap="binary_r", what="mean(matched_observations)")


Sigma is 18.000000 arcmin (0.005236 rad) 
-> fwhm is 42.386761 arcmin
Sigma is 0.000000 arcmin (0.000000 rad) 
-> fwhm is 0.000000 arcmin
CPU times: user 5.1 s, sys: 360 ms, total: 5.46 s
Wall time: 495 ms

In [15]:
hp.mollzoom(c[:,-30], nest=True)

In [32]:
#%%time
#gaia.healpix_plot(f="log1p", healpix_level=10, interactive=True)#, figsize=(10,8), )

In [42]:
%%time
gaia.col.phot_g_mean_flux
gaia.healpix_plot(f="log1p", figsize=(14,8), healpix_level=8, colormap="binary_r", what="mean(matched_observations)", healpix_output="ecliptic")
plt.show()
gaia.healpix_plot(f="log1p", figsize=(14,8), healpix_level=8, colormap="binary_r", what="count(*)", healpix_output="equatorial")
plt.show()
gaia.healpix_plot(f="log1p", figsize=(14,8), healpix_level=8, colormap="binary_r", what="sum(phot_g_mean_flux)", healpix_output="equatorial")


CPU times: user 2min 33s, sys: 35.9 s, total: 3min 9s
Wall time: 20 s

In [38]:
del gaia.variables["healpix"]
gaia.write_meta()


---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-38-4a7495234850> in <module>()
----> 1 del gaia.variables["healpix"]
      2 gaia.write_meta()

/Users/users/breddels/.conda/envs/vaex/lib/python2.7/collections.pyc in __delitem__(self, key, dict_delitem)
     83         # Deleting an existing item uses self.__map to find the link which gets
     84         # removed by updating the links in the predecessor and successor nodes.
---> 85         dict_delitem(self, key)
     86         link_prev, link_next, _ = self.__map.pop(key)
     87         link_prev[1] = link_next                        # update link_prev[NEXT]

KeyError: 'healpix'

In [5]:
%%time
level = 7
NSIDE=2**level
nmax = hp.nside2npix(NSIDE)
#gaia.add_variable("healpix_scaling", 34359738368*4**(12-level))
c=gaia.count(binby="source_id/healpix_scaling", limits=[0., nmax], shape=nmax)#, selection="phot_g_mean_mag < 26")


---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-5-5fa805d04ebd> in <module>()
----> 1 get_ipython().run_cell_magic(u'time', u'', u'level = 7\nNSIDE=2**level\nnmax = hp.nside2npix(NSIDE)\ngaia.add_variable("healpix_scaling", 34359738368*4**(12-level))\nc=gaia.count(binby="source_id/healpix_scaling", limits=[0., nmax], shape=nmax)#, selection="phot_g_mean_mag < 26")')

/Users/users/breddels/.local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in run_cell_magic(self, magic_name, line, cell)
   2118             magic_arg_s = self.var_expand(line, stack_depth)
   2119             with self.builtin_trap:
-> 2120                 result = fn(magic_arg_s, cell)
   2121             return result
   2122 

<decorator-gen-60> in time(self, line, cell, local_ns)

/Users/users/breddels/.local/lib/python2.7/site-packages/IPython/core/magic.pyc in <lambda>(f, *a, **k)
    191     # but it's overkill for just that one bit of state.
    192     def magic_deco(arg):
--> 193         call = lambda f, *a, **k: f(*a, **k)
    194 
    195         if callable(arg):

/Users/users/breddels/.local/lib/python2.7/site-packages/IPython/core/magics/execution.pyc in time(self, line, cell, local_ns)
   1175         else:
   1176             st = clock2()
-> 1177             exec(code, glob, local_ns)
   1178             end = clock2()
   1179             out = None

<timed exec> in <module>()

/Users/users/breddels/src/vaex/vaex/dataset.pyc in count(self, expression, binby, limits, shape, selection, async)
   2356                 stats = [calculate(expression, limits) for expression in expressions]
   2357                 var = finish(*stats)
-> 2358                 return self._async(async, var)
   2359 
   2360         @docsubst

/Users/users/breddels/src/vaex/vaex/dataset.pyc in _async(self, async, task, progressbar)
   2930                         return task
   2931                 else:
-> 2932                         self.executor.execute()
   2933                         return task.get()
   2934 

/Users/users/breddels/src/vaex/vaex/execution.pyc in execute(self)
    151                                                 task._results = []
    152                                                 task.signal_progress.emit(0)
--> 153                                         block_scopes = [dataset._block_scope(0, self.buffer_size) for i in range(self.thread_pool.nthreads)]
    154                                         def process(thread_index, i1, i2):
    155                                                 if not cancelled[0]:

/Users/users/breddels/src/vaex/vaex/dataset.pyc in _block_scope(self, i1, i2)
   4388 
   4389         def _block_scope(self, i1, i2):
-> 4390                 variables = {key:self.evaluate_variable(key) for key in self.variables.keys()}
   4391                 return _BlockScope(self, i1, i2, **variables)
   4392 

/Users/users/breddels/src/vaex/vaex/dataset.pyc in <dictcomp>((key,))
   4388 
   4389         def _block_scope(self, i1, i2):
-> 4390                 variables = {key:self.evaluate_variable(key) for key in self.variables.keys()}
   4391                 return _BlockScope(self, i1, i2, **variables)
   4392 

/Users/users/breddels/src/vaex/vaex/dataset.pyc in evaluate_variable(self, name)
   4348                 if isinstance(self.variables[name], six.string_types):
   4349                         # TODO: this does not allow more than one level deep variable, like a depends on b, b on c, c is a const
-> 4350                         value = eval(self.variables[name], expression_namespace, self.variables)
   4351                         return value
   4352                 else:

<string> in <module>()

NameError: name 'source_id' is not defined

In [ ]:
print c.sum(), nmax
print ds.max("source_id/(34359738368*4*4)")

In [ ]:
c.sum()

In [8]:
gaia.add_virtual_column("healpix", "source_id/34359738368")
#c=gaia.count(binby="healpix", limits=[0., nmax], shape=nmax)#, selection="phot_g_mean_mag < 26")

In [86]:
%%time
def plot_healpix(healpix="healpix", level=12, reduce_level=4):
    plot_level = level-reduce_level
    NSIDE=2**plot_level
    nmax = hp.nside2npix(NSIDE)
    print nmax, np.sqrt(nmax)
    scaling = 4**reduce_level
    c=gaia.count(binby="source_id/34359738368/%s" % scaling, limits=[0., nmax], shape=nmax) #, selection="phot_g_mean_mag < 20")
    #c=gaia.sum(ds.col.phot_g_mean_flux, binby="source_id/34359738368/%s" % scaling, limits=[0., nmax], shape=nmax)
    c = hp.reorder(c, inp="NEST", out="RING")
    #c = hp.smoothing(c, sigma=np.radians(10))
    cf = np.log10(c)
    hp.mollview(cf, rot=(-50,-90,90), nest=False,title="Gaia DR1", coord=['C','G'], cmap="binary_r", hold=True, xsize=2000)#, min=6-1, max=8.7-1)
f=plt.figure(figsize=(14,14))
plot_healpix(reduce_level=6)
#plt.savefig("/Users/users/breddels/Dropbox/gaia/gaia-dr1-flux.pdf")


49152 221.702503369
CPU times: user 23.7 s, sys: 2.53 s, total: 26.2 s
Wall time: 2.35 s

In [21]:
gaia.plot1d(ds.col.phot_g_mean_mag)


Out[21]:
[<matplotlib.lines.Line2D at 0x7f3553047490>]

In [36]:
"{:,}".format(gaia.count(selection=True))


Out[36]:
'312,281,269.0'

In [33]:
gaia.select("phot_g_mean_mag < 20")
#gaia.plot("l", "b", selection=True)
gaia("l", "b").selected().plot(f=np.log1p)


Out[33]:
(<matplotlib.image.AxesImage at 0x7f352876b610>, None)

In [35]:
gaia.select("phot_g_mean_mag < 18")
#gaia.plot("l", "b", selection=True)
gaia("l", "b").selected().plot(f=np.log1p)


Out[35]:
(<matplotlib.image.AxesImage at 0x7f3528461910>, None)

In [133]:
cf = c**0.4#np.log(c+1000)
f=plt.figure(figsize=(14,14))
hp.mollview(cf, nest=True,title="TGAS", coord=['C','G'], cmap="binary_r", hold=True, xsize=1000)
hp.graticule()
plt.savefig("/Users/users/breddels/Dropbox/gaia/gaia-dr1.pdf")


0.0 180.0 -180.0 180.0
The interval between parallels is 30 deg -0.00'.
The interval between meridians is 30 deg -0.00'.

In [54]:
cs = hp.sphtfunc.smoothing(c, sigma=0.01, verbose=True, pol=True)
hp.mollview(cs, nest=True,title="TGAS", coord=['C','G'],)
hp.graticule()


---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-54-7445af657eb9> in <module>()
----> 1 cs = hp.sphtfunc.smoothing(c, sigma=0.01, verbose=True, pol=True)
      2 hp.mollview(cs, nest=True,title="TGAS", coord=['C','G'],)
      3 hp.graticule()

NameError: name 'c' is not defined

In [24]:
gaia = vx.open("/Users/users/breddels/gaia/cdn.gea.esac.esa.int/Gaia/gaia_source/gaia-dr1-unshuffled-b.hdf5")

In [124]:
%%time
plt.figure(figsize=(12,12))
gaia.plot("l", "b", shape=1024*4)
plt.savefig("/Users/users/breddels/Dropbox/gaia/gaia-dr1-flat.pdf")


CPU times: user 54 s, sys: 1min 51s, total: 2min 45s
Wall time: 35.9 s

In [125]:
gaia.plot_bq("l", "b")


/Users/users/breddels/src/vaex/vaex/dataset.py:953: RuntimeWarning: divide by zero encountered in log
  data = f(grid)

In [ ]: