In [1]:
import opsimsummary as oss

In [2]:
import os

In [3]:
example_dir = os.path.join(oss.__path__[0], 'example_data')

In [4]:
dbName = os.path.join(example_dir, 'enigma_1189_micro.db')

In [5]:
hdfName = os.path.join(example_dir, 'enigma_1189_micro.hdf')

In [6]:
from sqlalchemy import create_engine as create_engine

In [7]:
from opsimsummary import OpSimOutput

From the OpSim Output (sqlite database)


In [8]:
opout_hdf = OpSimOutput.fromOpSimHDF(hdfName=hdfName, subset='Combined')


read in proposal
read in proposal
combined {'ddf': 366, u'../conf/survey/galacticplaneprop.conf': 362, u'../conf/survey/northeclipticspur-18c.conf': 365, 'wfd': 364, u'../conf/survey/southcelestialpole-18.conf': 363}
propIDs [366, 364] <type 'list'> <type 'numpy.int64'>
summarydf cols Index([        u'obsHistID',         u'sessionID',            u'propID',
                 u'fieldID',           u'fieldRA',          u'fieldDec',
                  u'filter',           u'expDate',            u'expMJD',
                   u'night',         u'visitTime',      u'visitExpTime',
                 u'finRank',         u'finSeeing',      u'transparency',
                 u'airmass',        u'vSkyBright', u'filtSkyBrightness',
               u'rotSkyPos',               u'lst',          u'altitude',
                 u'azimuth',         u'dist2Moon',        u'solarElong',
                  u'moonRA',           u'moonDec',           u'moonAlt',
                  u'moonAZ',         u'moonPhase',            u'sunAlt',
                   u'sunAz',        u'phaseAngle',          u'rScatter',
              u'mieScatter',         u'moonIllum',        u'moonBright',
              u'darkBright',         u'rawSeeing',              u'wind',
                u'humidity',          u'slewDist',          u'slewTime',
          u'fiveSigmaDepth',        u'ditheredRA',       u'ditheredDec'],
      dtype='object')
squery_str propID == @propIDs
 Num entries  76888

In [7]:
opout = OpSimOutput.fromOpSimDB(dbname=dbName, subset='_all')


 reading from database sqlite:////Users/rbiswas/.local/lib/python2.7/site-packages/opsimsummary/example_data/enigma_1189_micro.db

In [8]:
opout.proposalTable


Out[8]:
propID propConf propName objectID objectHost Session_sessionID
0 362 ../conf/survey/GalacticPlaneProp.conf WL 27692624 enigma 1189
1 363 ../conf/survey/SouthCelestialPole-18.conf WL 27692368 enigma 1189
2 364 ../conf/survey/Universal-18-0824B.conf WLTSS 27692112 enigma 1189
3 365 ../conf/survey/NorthEclipticSpur-18c.conf WLTSS 27692240 enigma 1189
4 366 ../conf/survey/DDcosmology1.conf WLTSS 29065424 enigma 1189

In [9]:
opout.propIDDict


Out[9]:
{u'../conf/survey/galacticplaneprop.conf': 362,
 u'../conf/survey/northeclipticspur-18c.conf': 365,
 u'../conf/survey/southcelestialpole-18.conf': 363,
 'ddf': 366,
 'wfd': 364}

In [10]:
opout.summary.propID.unique()


Out[10]:
array([364, 366])

In [11]:
opout.propIDVals('combined', opout.propIDDict, opout.proposalTable)


Out[11]:
[366, 364]

In [12]:
opout.propIds


Out[12]:
array([362, 363, 364, 365, 366])

In [13]:
opout.subset


Out[13]:
'_all'

In [14]:
opout.summary.head()


Out[14]:
sessionID propID fieldID fieldRA fieldDec filter expDate expMJD night visitTime ... moonBright darkBright rawSeeing wind humidity slewDist slewTime fiveSigmaDepth ditheredRA ditheredDec
obsHistID
171077 1189 364 2123 3.502578 -0.203741 z 18745559 49569.962492 217 34.0 ... 0.0 90.575602 1.395861 0.0 0.0 2.729285 0.000000 21.402429 3.486983 -0.230192
171078 1189 364 2109 3.556282 -0.208795 z 18745598 49569.962940 217 34.0 ... 0.0 88.780090 1.395861 0.0 0.0 0.052807 4.701006 21.420436 3.540671 -0.235246
171079 1189 364 2103 3.609762 -0.213110 z 18745636 49569.963388 217 34.0 ... 0.0 87.241597 1.395861 0.0 0.0 0.052469 4.665852 21.435982 3.594137 -0.239561
171080 1189 364 2082 3.663150 -0.216431 z 18745675 49569.963837 217 34.0 ... 0.0 85.930303 1.395861 0.0 0.0 0.052268 4.820142 21.449321 3.647513 -0.242882
171081 1189 364 1947 3.690123 -0.263957 z 18745714 49569.964286 217 34.0 ... 0.0 84.412797 1.395861 0.0 0.0 0.054263 4.841302 21.464869 3.674303 -0.290408

5 rows × 44 columns


In [15]:
opout.summary[['fieldID', 'fieldRA', 'fieldDec', 'filter', 'expMJD', 'ditheredRA', 'ditheredDec']]


Out[15]:
fieldID fieldRA fieldDec filter expMJD ditheredRA ditheredDec
obsHistID
171077 2123 3.502578 -0.203741 z 49569.962492 3.486983 -0.230192
171078 2109 3.556282 -0.208795 z 49569.962940 3.540671 -0.235246
171079 2103 3.609762 -0.213110 z 49569.963388 3.594137 -0.239561
171080 2082 3.663150 -0.216431 z 49569.963837 3.647513 -0.242882
171081 1947 3.690123 -0.263957 z 49569.964286 3.674303 -0.290408
171082 1822 3.716717 -0.311837 z 49569.964736 3.700671 -0.338288
171083 1697 3.743185 -0.359940 z 49569.965185 3.726867 -0.386391
171084 1709 3.689779 -0.358409 z 49569.965636 3.673471 -0.384860
171085 1731 3.636496 -0.355408 z 49569.966084 3.620206 -0.381859
171086 1740 3.583366 -0.351045 z 49569.966533 3.567102 -0.377496
171087 1878 3.557426 -0.301194 z 49569.966981 3.541434 -0.327645
171088 1852 3.610495 -0.305999 z 49569.967429 3.594479 -0.332450
171089 1832 3.663574 -0.309601 z 49569.967877 3.647540 -0.336052
171090 1967 3.636956 -0.261116 z 49569.968326 3.621148 -0.287567
171091 1987 3.583766 -0.257075 z 49569.968774 3.567975 -0.283526
171092 1997 3.530442 -0.252053 z 49569.969222 3.514672 -0.278504
171093 2007 3.476805 -0.246309 z 49569.969671 3.461058 -0.272760
171094 1887 3.504231 -0.295396 z 49569.970119 3.488268 -0.321847
171095 1759 3.530340 -0.345461 z 49569.970567 3.514109 -0.371912
171096 1783 3.477274 -0.338835 z 49569.971016 3.461081 -0.365286
171097 1897 3.450679 -0.288856 z 49569.971464 3.434747 -0.315307
171098 2025 3.422629 -0.240146 z 49569.971913 3.406906 -0.266597
171099 2141 3.393799 -0.192613 z 49569.972362 3.378239 -0.219064
171100 2127 3.448477 -0.198239 z 49569.972811 3.432900 -0.224690
171101 2249 3.419720 -0.151046 z 49569.973260 3.404272 -0.177497
171102 2237 3.474169 -0.156208 z 49569.973708 3.458709 -0.182659
171103 2229 3.528220 -0.161175 z 49569.974157 3.512747 -0.187626
171104 2219 3.582024 -0.165646 z 49569.974604 3.566540 -0.192097
171105 2207 3.635713 -0.169338 z 49569.975052 3.620219 -0.195789
171106 2197 3.689379 -0.171989 z 49569.975499 3.673878 -0.198440
... ... ... ... ... ... ... ...
272045 1427 0.925184 -0.478900 u 49718.079575 0.897222 -0.468981
272046 1427 0.925184 -0.478900 u 49718.079992 0.897222 -0.468981
272047 1427 0.925184 -0.478900 u 49718.080408 0.897222 -0.468981
272048 1427 0.925184 -0.478900 u 49718.080825 0.897222 -0.468981
272049 1427 0.925184 -0.478900 u 49718.081242 0.897222 -0.468981
272050 1427 0.925184 -0.478900 u 49718.081658 0.897222 -0.468981
272051 1427 0.925184 -0.478900 u 49718.082075 0.897222 -0.468981
272052 1427 0.925184 -0.478900 u 49718.082492 0.897222 -0.468981
272053 1427 0.925184 -0.478900 u 49718.082908 0.897222 -0.468981
272054 1427 0.925184 -0.478900 u 49718.083325 0.897222 -0.468981
272405 2786 2.624318 0.049506 u 49718.248656 2.599472 0.059425
272406 2786 2.624318 0.049506 u 49718.249073 2.599472 0.059425
272407 2786 2.624318 0.049506 u 49718.249490 2.599472 0.059425
272408 2786 2.624318 0.049506 u 49718.249906 2.599472 0.059425
272409 2786 2.624318 0.049506 u 49718.250323 2.599472 0.059425
272410 2786 2.624318 0.049506 u 49718.250740 2.599472 0.059425
272411 2786 2.624318 0.049506 u 49718.251156 2.599472 0.059425
272412 2786 2.624318 0.049506 u 49718.251573 2.599472 0.059425
272413 2786 2.624318 0.049506 u 49718.251990 2.599472 0.059425
272414 2786 2.624318 0.049506 u 49718.252406 2.599472 0.059425
272415 2786 2.624318 0.049506 u 49718.252823 2.599472 0.059425
272416 2786 2.624318 0.049506 u 49718.253240 2.599472 0.059425
272417 2786 2.624318 0.049506 u 49718.253656 2.599472 0.059425
272418 2786 2.624318 0.049506 u 49718.254073 2.599472 0.059425
272419 2786 2.624318 0.049506 u 49718.254490 2.599472 0.059425
272420 2786 2.624318 0.049506 u 49718.254906 2.599472 0.059425
272421 2786 2.624318 0.049506 u 49718.255323 2.599472 0.059425
272422 2786 2.624318 0.049506 u 49718.255740 2.599472 0.059425
272423 2786 2.624318 0.049506 u 49718.256156 2.599472 0.059425
272424 2786 2.624318 0.049506 u 49718.256573 2.599472 0.059425

76888 rows × 7 columns


In [15]:
print(len(opout.summary))


76888

In [16]:
# Write to hdf file

In [17]:
opout.writeOpSimHDF('/tmp/opsim_small.hdf')


/usr/local/software/lib/python2.7/site-packages/pandas/io/pytables.py:260: PerformanceWarning: 
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->unicode,key->axis0] [items->None]

  f(store)
/usr/local/software/lib/python2.7/site-packages/pandas/io/pytables.py:260: PerformanceWarning: 
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->unicode,key->block0_items] [items->None]

  f(store)
/usr/local/software/lib/python2.7/site-packages/pandas/io/pytables.py:260: PerformanceWarning: 
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->unicode,key->block1_items] [items->None]

  f(store)
/usr/local/software/lib/python2.7/site-packages/pandas/core/generic.py:1101: PerformanceWarning: 
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->unicode,key->block2_values] [items->['filter']]

  return pytables.to_hdf(path_or_buf, key, self, **kwargs)
/usr/local/software/lib/python2.7/site-packages/pandas/io/pytables.py:260: PerformanceWarning: 
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->unicode,key->block2_items] [items->None]

  f(store)
/usr/local/software/lib/python2.7/site-packages/pandas/core/generic.py:1101: PerformanceWarning: 
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->unicode,key->block1_values] [items->['propConf', 'propName', 'objectHost']]

  return pytables.to_hdf(path_or_buf, key, self, **kwargs)

In [22]:
opsim_from_hdf = OpSimOutput.fromOpSimHDF('/tmp/opsim_small.hdf', subset='_all')


read in proposal
read in proposal
_all {'ddf': 366, u'../conf/survey/galacticplaneprop.conf': 362, u'../conf/survey/northeclipticspur-18c.conf': 365, 'wfd': 364, u'../conf/survey/southcelestialpole-18.conf': 363}
propIDs [362, 363, 364, 365, 366] <type 'list'> <type 'int'>
summarydf cols Index([        u'obsHistID',         u'sessionID',            u'propID',
                 u'fieldID',           u'fieldRA',          u'fieldDec',
                  u'filter',           u'expDate',            u'expMJD',
                   u'night',         u'visitTime',      u'visitExpTime',
                 u'finRank',         u'finSeeing',      u'transparency',
                 u'airmass',        u'vSkyBright', u'filtSkyBrightness',
               u'rotSkyPos',               u'lst',          u'altitude',
                 u'azimuth',         u'dist2Moon',        u'solarElong',
                  u'moonRA',           u'moonDec',           u'moonAlt',
                  u'moonAZ',         u'moonPhase',            u'sunAlt',
                   u'sunAz',        u'phaseAngle',          u'rScatter',
              u'mieScatter',         u'moonIllum',        u'moonBright',
              u'darkBright',         u'rawSeeing',              u'wind',
                u'humidity',          u'slewDist',          u'slewTime',
          u'fiveSigmaDepth',        u'ditheredRA',       u'ditheredDec'],
      dtype='object')
squery_str propID == @propIDs
 Num entries  76888

In [20]:
from pandas.util.testing import assert_frame_equal

In [23]:
opsim_from_hdf.subset


Out[23]:
'_all'

In [24]:
assert_frame_equal(opsim_from_hdf.summary, opout.summary)

From OpSimDB but combined (This is WFD and DDF combined)


In [16]:
opsim_comb = OpSimOutput.fromOpSimDB(dbname=dbName)


 reading from database sqlite:////Users/rbiswas/.local/lib/python2.7/site-packages/opsimsummary/example_data/enigma_1189_micro.db
SELECT * FROM Summary WHERE PROPID in (366, 364)

In [17]:
opsim_comb.writeOpSimHDF('tmp/hdfFile.hdf')


---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-17-b5b97aded478> in <module>()
----> 1 opsim_comb.writeOpSimHDF('tmp/hdfFile.hdf')

/Users/rbiswas/.local/lib/python2.7/site-packages/opsimsummary/opsim_out.pyc in writeOpSimHDF(self, hdfName)
    188         """
    189         if self.subset != '_all':
--> 190             raise ValueError('Should be Done only for self.subset == _all')
    191         self.summary.to_hdf(hdfName, key='Summary', append=False)
    192         self.proposalTable.to_hdf(hdfName, key='Proposal', append=False)

ValueError: Should be Done only for self.subset == _all

From OpSimHDF


In [18]:
opsim_all = OpSimOutput.fromOpSimDB(dbname=dbName, subset='_all')


 reading from database sqlite:////Users/rbiswas/.local/lib/python2.7/site-packages/opsimsummary/example_data/enigma_1189_micro.db

In [20]:
opsim_all.writeOpSimHDF('test_hdf.hdf')


/usr/local/software/lib/python2.7/site-packages/pandas/io/pytables.py:260: PerformanceWarning: 
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->unicode,key->axis0] [items->None]

  f(store)
/usr/local/software/lib/python2.7/site-packages/pandas/io/pytables.py:260: PerformanceWarning: 
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->unicode,key->block0_items] [items->None]

  f(store)
/usr/local/software/lib/python2.7/site-packages/pandas/io/pytables.py:260: PerformanceWarning: 
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->unicode,key->block1_items] [items->None]

  f(store)
/usr/local/software/lib/python2.7/site-packages/pandas/core/generic.py:1101: PerformanceWarning: 
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->unicode,key->block2_values] [items->['filter']]

  return pytables.to_hdf(path_or_buf, key, self, **kwargs)
/usr/local/software/lib/python2.7/site-packages/pandas/io/pytables.py:260: PerformanceWarning: 
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->unicode,key->block2_items] [items->None]

  f(store)
/usr/local/software/lib/python2.7/site-packages/pandas/core/generic.py:1101: PerformanceWarning: 
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->unicode,key->block1_values] [items->['propConf', 'propName', 'objectHost']]

  return pytables.to_hdf(path_or_buf, key, self, **kwargs)

In [21]:
opsim_fromhdf = OpSimOutput.fromOpSimHDF('test_hdf.hdf', subset='_all')


---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-21-7473a32603de> in <module>()
----> 1 opsim_fromhdf = OpSimOutput.fromOpSimHDF('test_hdf.hdf', subset='_all')

/Users/rbiswas/.local/lib/python2.7/site-packages/opsimsummary/opsim_out.pyc in fromOpSimHDF(cls, hdfName, subset, tableNames, propIDs)
    142         summarydf = pd.read_hdf(hdfName, key='Summary')
    143 
--> 144         if 'obsHistID' not in summarydf.columns:
    145             summarydf.reset_index(inplace=True)
    146             if 'obsHistID' not in summarydf.columns:

/usr/local/software/lib/python2.7/site-packages/pandas/core/generic.pyc in __getattr__(self, name)
   2670             if name in self._info_axis:
   2671                 return self[name]
-> 2672             return object.__getattribute__(self, name)
   2673 
   2674     def __setattr__(self, name, value):

AttributeError: 'DataFrame' object has no attribute 'colums'

In [12]:
from sqlalchemy import create_engine

In [13]:
engine = create_engine('sqlite:///'+dbName)

In [14]:
import pandas as pd

In [15]:
_ = pd.read_sql_query('SELECT * FROM Summary WHERE propID in (364, 366) LIMIT 5', con=engine)

In [21]:
_


Out[21]:
obsHistID sessionID propID fieldID fieldRA fieldDec filter expDate expMJD night ... moonBright darkBright rawSeeing wind humidity slewDist slewTime fiveSigmaDepth ditheredRA ditheredDec
0 171077 1189 364 2123 3.502578 -0.203741 z 18745559 49569.962492 217 ... 0.0 90.575602 1.395861 0.0 0.0 2.729285 0.000000 21.402429 3.486983 -0.230192
1 171078 1189 364 2109 3.556282 -0.208795 z 18745598 49569.962940 217 ... 0.0 88.780090 1.395861 0.0 0.0 0.052807 4.701006 21.420436 3.540671 -0.235246
2 171079 1189 364 2103 3.609762 -0.213110 z 18745636 49569.963388 217 ... 0.0 87.241597 1.395861 0.0 0.0 0.052469 4.665852 21.435982 3.594137 -0.239561
3 171080 1189 364 2082 3.663150 -0.216431 z 18745675 49569.963837 217 ... 0.0 85.930303 1.395861 0.0 0.0 0.052268 4.820142 21.449321 3.647513 -0.242882
4 171081 1189 364 1947 3.690123 -0.263957 z 18745714 49569.964286 217 ... 0.0 84.412797 1.395861 0.0 0.0 0.054263 4.841302 21.464869 3.674303 -0.290408

5 rows × 45 columns


In [22]:
pids = [364, 366]

In [23]:
x = ', '.join(list(str(pid) for pid in pids))

In [24]:
'SELECT * FROM Summary WHERE propID == ({})'.format(x)


Out[24]:
'SELECT * FROM Summary WHERE propID == (364, 366)'

In [25]:
opout_comb = oss.OpSimOutput.fromOpSimDB(dbname=dbName, subset='combined')


 reading from database sqlite:////Users/rbiswas/.local/lib/python2.7/site-packages/opsimsummary/example_data/enigma_1189_micro.db
SELECT * FROM Summary WHERE PROPID in (366, 364)

In [38]:
opout_comb.summary.propID.unique()


Out[38]:
array([364, 366])

In [26]:
import pandas as pd

In [27]:
summarydf = pd.read_hdf('/tmp/opsim_small.hdf', key='Summary')

In [28]:
from  pandas.util.testing import assert_frame_equal

In [29]:
assert_frame_equal(summarydf, opout.summary)

In [30]:
engine = create_engine('sqlite:///' + dbName)

In [31]:
import pandas as pd

In [32]:
pids = tuple([366])

In [33]:
pids = [366, 364]

In [34]:
', '.join('{}'.format(x) for x in pids)


Out[34]:
'366, 364'

In [35]:
pids = [344, 366]

In [36]:
sql_query = "SELECT * FROM Summary WHERE PROPID == "

In [37]:
sql_query += "({})".format(pid) for pid in pids


  File "<ipython-input-37-de1fa48c5cd4>", line 1
    sql_query += "({})".format(pid) for pid in pids
                                      ^
SyntaxError: invalid syntax

In [ ]:
sql_query += ", ".join(list("({})".format(pid) for pid in pids))

In [ ]:
sql_query

In [ ]:
pids

In [ ]:
df = pd.read_sql_query(sql_query, con=engine)

In [ ]:
df = pd.read_sql_query("SELECT * FROM Summary WHERE PROPID in (362, 366)", con=engine)

In [ ]:
df.head()

In [ ]:
pids = [366]

In [ ]:
df.query('propID == @pids')

In [ ]:
df.propID.unique()

In [ ]:
df = oss.OpSimOutput.fromOpSimDB(dbName, subset='ddf')

In [ ]:


In [ ]:
df.summary.head()

In [ ]:
pids = df.propIDVals('wfd', df.propIDDict, df.proposalTable)

In [ ]:
df.summary.query('propID == {}'.format(pids)).propID.unique()

In [ ]:
import sqlite3

In [ ]:
engine

In [ ]:
alldf = oss.summarize_opsim.OpSimOutput.fromOpSimDB(dbName, subset='_all')

In [ ]:
len(alldf.summary.drop_duplicates())

In [ ]:
len(alldf.summary.reset_index().drop_duplicates(subset='obsHistID'))== len(alldf.summary)

In [ ]:
len(alldf.summary)

In [ ]:
unique_alldf = oss.summarize_opsim.OpSimOutput.fromOpSimDB(dbName, subset='unique_all')

In [ ]:
len(unique_alldf.summary)

In [ ]:
unique_alldf.summary.query('propID == [364, 366]').propID.unique()

In [ ]:
def myfunc(*args):
    if len
    return args[0]**2

In [ ]:
myfunc(2)

In [ ]:
alldf.

In [ ]:
alldf.propIDDict

In [ ]:
alldf.summary.head()

In [ ]: