Index file check

This is a quick look at the first and last few rows in some of the index files, to check if all values seem OK.

Let's look at "agn" as one example ...


In [1]:
import numpy as np
import astropy.units as u
from gammapy.data import DataStore

In [2]:
ds = DataStore.from_dir('1dc/1dc/index/agn')

In [3]:
ds.obs_table.info()


<ObservationTable length=1920>
      name       dtype 
--------------- -------
         OBS_ID   int64
         RA_PNT float64
        DEC_PNT float64
       GLON_PNT float64
       GLAT_PNT float64
        ZEN_PNT float64
        ALT_PNT float64
         AZ_PNT   int64
         ONTIME float64
       LIVETIME float64
          DEADC float64
         TSTART float64
          TSTOP float64
       DATE_OBS   str10
       TIME_OBS    str8
       DATE_END   str10
       TIME_END    str8
         N_TELS   int64
         OBJECT   str14
          CALDB    str3
            IRF   str13
EVENTS_FILENAME   str50
    EVENT_COUNT   int64

In [4]:
ds.obs_table.info('stats')


<ObservationTable length=1920>
      name           mean            std             min           max     
--------------- ------------- ----------------- ------------- -------------
         OBS_ID      510959.5     554.256183246        510000        511919
         RA_PNT 176.142887379     96.1833488986  1.9469000101  356.26071167
        DEC_PNT 19.2987040441     27.9481261874      -17.3806   75.57460022
       GLON_PNT 153.167896435     98.9692443233 2.37542045991 351.290560535
       GLAT_PNT 21.9062138121     44.9647316086  -72.38125429 84.9571427298
        ZEN_PNT           0.0               0.0           0.0           0.0
        ALT_PNT          90.0               0.0          90.0          90.0
         AZ_PNT           0.0               0.0             0             0
         ONTIME        1800.0               0.0        1800.0        1800.0
       LIVETIME        1764.0               0.0        1764.0        1764.0
          DEADC 0.98000001907 2.22044604925e-16 0.98000001907 0.98000001907
         TSTART   686686080.0     13966172.8015   662774400.0   710597760.0
          TSTOP   686687872.0     13966172.8015   662776192.0   710599552.0
       DATE_OBS            --                --            --            --
       TIME_OBS            --                --            --            --
       DATE_END            --                --            --            --
       TIME_END            --                --            --            --
         N_TELS           0.0               0.0             0             0
         OBJECT            --                --            --            --
          CALDB            --                --            --            --
            IRF            --                --            --            --
EVENTS_FILENAME            --                --            --            --
    EVENT_COUNT 96416.2583333     8027.03702508         90874        329003

In [5]:
ds.obs_table[:20]


Out[5]:
ObservationTable length=20
OBS_IDRA_PNTDEC_PNTGLON_PNTGLAT_PNTZEN_PNTALT_PNTAZ_PNTONTIMELIVETIMEDEADCTSTARTTSTOPDATE_OBSTIME_OBSDATE_ENDTIME_ENDN_TELSOBJECTCALDBIRFEVENTS_FILENAMEEVENT_COUNT
int64float64float64float64float64float64float64int64float64float64float64float64float64str10str8str10str8int64str14str3str13str50int64
51000035.66500091643.035499573140.142934442-16.76693118170.090.001800.01764.00.98000001907662774400.0662776192.02021-01-0111:58:512021-01-0112:28:510AGN monitoring1dcNorth_z20_50h1dc/1dc/data/baseline/agn/agn_baseline_510000.fits94306
510001110.4720001271.343399048143.98107369828.01733297810.090.001800.01764.00.98000001907662776320.0662778112.02021-01-0112:30:512021-01-0113:00:510AGN monitoring1dcNorth_z40_50h1dc/1dc/data/baseline/agn/agn_baseline_510001.fits102550
510002186.22700521.379600525255.07360534281.6598844160.090.001800.01764.00.98000001907662778240.0662780032.02021-01-0113:02:512021-01-0113:32:510AGN monitoring1dcNorth_z20_50h1dc/1dc/data/baseline/agn/agn_baseline_510002.fits110043
510003228.21749878-9.106351.29056053540.12965134130.090.001800.01764.00.98000001907662780160.0662781952.02021-01-0113:34:512021-01-0114:04:510AGN monitoring1dcNorth_z40_50h1dc/1dc/data/baseline/agn/agn_baseline_510003.fits122025
510004166.1195983938.207298279179.83172043565.03628207010.090.001800.01764.00.98000001907662782080.0662783872.02021-01-0114:06:512021-01-0114:36:510AGN monitoring1dcNorth_z20_50h1dc/1dc/data/baseline/agn/agn_baseline_510004.fits96144
510005194.05999756-5.8046305.12800325757.046670040.090.001800.01764.00.98000001907662784000.0662785792.02021-01-0114:38:512021-01-0115:08:510AGN monitoring1dcNorth_z40_50h1dc/1dc/data/baseline/agn/agn_baseline_510005.fits98200
51000695.7350006133.428100586179.9471373229.153230015810.090.001800.01764.00.98000001907662785920.0662787712.02021-01-0115:10:512021-01-0115:40:510AGN monitoring1dcNorth_z20_50h1dc/1dc/data/baseline/agn/agn_baseline_510006.fits91658
510007219.2687988356.66040039197.714030684154.98806660540.090.001800.01764.00.98000001907662787840.0662789632.02021-01-0115:42:512021-01-0116:12:510AGN monitoring1dcNorth_z20_50h1dc/1dc/data/baseline/agn/agn_baseline_510007.fits91513
5100081.946900010147.154701233115.270529969-15.06667589520.090.001800.01764.00.98000001907662789760.0662791552.02021-01-0116:14:512021-01-0116:44:510AGN monitoring1dcNorth_z20_50h1dc/1dc/data/baseline/agn/agn_baseline_510008.fits92055
51000912.671600342-9.5104122.31973606-72.381254290.090.001800.01764.00.98000001907662791680.0662793472.02021-01-0116:46:512021-01-0117:16:510AGN monitoring1dcNorth_z40_50h1dc/1dc/data/baseline/agn/agn_baseline_510009.fits98531
51001024.20789909447.901298523130.755637828-14.27968482320.090.001800.01764.00.98000001907662793600.0662795392.02021-01-0117:18:512021-01-0117:48:510AGN monitoring1dcNorth_z20_50h1dc/1dc/data/baseline/agn/agn_baseline_510010.fits91715
51001194.413299561-17.2813224.912833167-15.0864800150.090.001800.01764.00.98000001907662795520.0662797312.02021-01-0117:50:512021-01-0118:20:510AGN monitoring1dcNorth_z40_50h1dc/1dc/data/baseline/agn/agn_baseline_510011.fits98463
510012121.3686981275.57460022138.89468191430.79663256220.090.001800.01764.00.98000001907662797440.0662799232.02021-01-0118:22:512021-01-0118:52:510AGN monitoring1dcNorth_z40_50h1dc/1dc/data/baseline/agn/agn_baseline_510012.fits99277
510013178.52479553-0.1693273.80966548659.4209493340.090.001800.01764.00.98000001907662799360.0662801152.02021-01-0118:54:512021-01-0119:24:510AGN monitoring1dcNorth_z20_50h1dc/1dc/data/baseline/agn/agn_baseline_510013.fits91753
510014185.4020996128.247299194201.5926682483.30358005580.090.001800.01764.00.98000001907662801280.0662803072.02021-01-0119:26:512021-01-0119:56:510AGN monitoring1dcNorth_z20_50h1dc/1dc/data/baseline/agn/agn_baseline_510014.fits101367
510015187.5859069825.376899719232.0702286484.95714272980.090.001800.01764.00.98000001907662803200.0662804992.02021-01-0119:58:512021-01-0120:28:510AGN monitoring1dcNorth_z20_50h1dc/1dc/data/baseline/agn/agn_baseline_510015.fits92430
510016206.00079346-17.3806320.3968166443.70135734080.090.001800.01764.00.98000001907662805120.0662806912.02021-01-0120:30:512021-01-0121:00:510AGN monitoring1dcNorth_z40_50h1dc/1dc/data/baseline/agn/agn_baseline_510016.fits98548
510017226.0950927710.48830032311.365132730454.58485110740.090.001800.01764.00.98000001907662807040.0662808832.02021-01-0121:02:512021-01-0121:32:510AGN monitoring1dcNorth_z20_50h1dc/1dc/data/baseline/agn/agn_baseline_510017.fits91636
510018228.064407352.07399988172.3754204599148.00736628730.090.001800.01764.00.98000001907662808960.0662810752.02021-01-0121:34:512021-01-0122:04:510AGN monitoring1dcNorth_z20_50h1dc/1dc/data/baseline/agn/agn_baseline_510018.fits91829
510019259.7981872617.75020027239.520695984628.10186877050.090.001800.01764.00.98000001907662810880.0662812672.02021-01-0122:06:512021-01-0122:36:510AGN monitoring1dcNorth_z20_50h1dc/1dc/data/baseline/agn/agn_baseline_510019.fits91620

In [6]:
ds.hdu_table.info()


<HDUIndexTable length=11520>
   name   dtype
--------- -----
   OBS_ID int64
 HDU_TYPE  str6
HDU_CLASS str10
 FILE_DIR str42
FILE_NAME str24
 HDU_NAME str21

In [7]:
ds.hdu_table.info('stats')


<HDUIndexTable length=11520>
   name     mean        std       min    max  
--------- -------- ------------- ------ ------
   OBS_ID 510959.5 554.256183246 510000 511919
 HDU_TYPE       --            --     --     --
HDU_CLASS       --            --     --     --
 FILE_DIR       --            --     --     --
FILE_NAME       --            --     --     --
 HDU_NAME       --            --     --     --

In [8]:
ds.hdu_table[:20]


Out[8]:
HDUIndexTable length=20
OBS_IDHDU_TYPEHDU_CLASSFILE_DIRFILE_NAMEHDU_NAME
int64str6str10str42str24str21
510000eventsevents../../data/baseline/agnagn_baseline_510000.fitsEVENTS
510000gtigti../../data/baseline/agnagn_baseline_510000.fitsEVENTS
510000aeffaeff_2d../../caldb/data/cta/1dc/bcf/North_z20_50hirf_file.fitsEFFECTIVE AREA
510000edispedisp_2d../../caldb/data/cta/1dc/bcf/North_z20_50hirf_file.fitsENERGY DISPERSION
510000psfpsf_3gauss../../caldb/data/cta/1dc/bcf/North_z20_50hirf_file.fitsPOINT SPREAD FUNCTION
510000bkgbkg_3d../../caldb/data/cta/1dc/bcf/North_z20_50hirf_file.fitsBACKGROUND
510001eventsevents../../data/baseline/agnagn_baseline_510001.fitsEVENTS
510001gtigti../../data/baseline/agnagn_baseline_510001.fitsEVENTS
510001aeffaeff_2d../../caldb/data/cta/1dc/bcf/North_z40_50hirf_file.fitsEFFECTIVE AREA
510001edispedisp_2d../../caldb/data/cta/1dc/bcf/North_z40_50hirf_file.fitsENERGY DISPERSION
510001psfpsf_3gauss../../caldb/data/cta/1dc/bcf/North_z40_50hirf_file.fitsPOINT SPREAD FUNCTION
510001bkgbkg_3d../../caldb/data/cta/1dc/bcf/North_z40_50hirf_file.fitsBACKGROUND
510002eventsevents../../data/baseline/agnagn_baseline_510002.fitsEVENTS
510002gtigti../../data/baseline/agnagn_baseline_510002.fitsEVENTS
510002aeffaeff_2d../../caldb/data/cta/1dc/bcf/North_z20_50hirf_file.fitsEFFECTIVE AREA
510002edispedisp_2d../../caldb/data/cta/1dc/bcf/North_z20_50hirf_file.fitsENERGY DISPERSION
510002psfpsf_3gauss../../caldb/data/cta/1dc/bcf/North_z20_50hirf_file.fitsPOINT SPREAD FUNCTION
510002bkgbkg_3d../../caldb/data/cta/1dc/bcf/North_z20_50hirf_file.fitsBACKGROUND
510003eventsevents../../data/baseline/agnagn_baseline_510003.fitsEVENTS
510003gtigti../../data/baseline/agnagn_baseline_510003.fitsEVENTS

Check IRF values

Now let's look at which values appear for IRF in the OBS and HDU index table ...


In [9]:
ds = DataStore.from_dir('1dc/1dc/index/all')

In [10]:
irf = ds.obs_table.to_pandas()['IRF']

In [11]:
irf.value_counts()


Out[11]:
South_z20_50h    3428
North_z20_50h    2760
North_z40_50h    1166
South_z40_50h     778
Name: IRF, dtype: int64

In [12]:
irf2 = ds.hdu_table.to_pandas()
irf2 = irf2.query('HDU_TYPE == "aeff"')
irf2 = irf2['FILE_DIR'].str.split('/').apply(lambda _: _[-1])

In [13]:
irf2.value_counts()


Out[13]:
South_z20_50h    3428
North_z20_50h    2760
North_z40_50h    1166
South_z40_50h     778
Name: FILE_DIR, dtype: int64

Check loading

Try to load all info for some of the runs ... should work for all runs.


In [14]:
ds = DataStore.from_dir('1dc/1dc/index/all')
obs_ids = ds.obs_table['OBS_ID'].data[::500]
obs_ids


Out[14]:
array([510000, 510500, 511000, 511500, 210080, 210580, 211080, 310309,
       310809, 311309, 110138, 110638, 111138, 111638, 112138, 112638,
       113138])

In [15]:
for obs_id in obs_ids:
    print('loading', obs_id)
    obs = ds.obs(obs_id)
    print('events', len(obs.events.table))
    print('aeff', obs.aeff.data.data.max())
    print('edisp', obs.edisp.data.data.max())
    print('psf', float(obs.psf.containment_radius(energy=1*u.TeV, theta=0*u.deg).deg))
    print('bkg', obs.bkg.data.data.max())


loading 510000
events 94306
aeff 1303965.5 m2
edisp 16039.921875
psf 0.058499993105930244
bkg 0.4422248601913452 1 / (MeV s sr)
loading 510500
events 92214
aeff 1303965.5 m2
edisp 16039.921875
psf 0.058499993105930244
bkg 0.4422248601913452 1 / (MeV s sr)
loading 511000
events 98852
aeff 1726487.75 m2
edisp 40505.125
psf 0.05799999268419844
bkg 0.23380520939826965 1 / (MeV s sr)
loading 511500
events 98615
aeff 1726487.75 m2
edisp 40505.125
psf 0.05799999268419844
bkg 0.23380520939826965 1 / (MeV s sr)
loading 210080
events 81928
aeff 5371581.0 m2
edisp 10595.85546875
psf 0.050999992403622255
bkg 0.4214513599872589 1 / (MeV s sr)
loading 210580
events 82078
aeff 5371581.0 m2
edisp 10595.85546875
psf 0.050999992403622255
bkg 0.4214513599872589 1 / (MeV s sr)
loading 211080
events 76178
aeff 1303965.5 m2
edisp 16039.921875
psf 0.058499993105930244
bkg 0.4422248601913452 1 / (MeV s sr)
loading 310309
events 108178
aeff 5371581.0 m2
edisp 10595.85546875
psf 0.050999992403622255
bkg 0.4214513599872589 1 / (MeV s sr)
loading 310809
events 107963
aeff 5371581.0 m2
edisp 10595.85546875
psf 0.050999992403622255
bkg 0.4214513599872589 1 / (MeV s sr)
loading 311309
events 98921
aeff 5371581.0 m2
edisp 10595.85546875
psf 0.050999992403622255
bkg 0.4214513599872589 1 / (MeV s sr)
loading 110138
events 109541
aeff 7238342.0 m2
edisp 27173.3203125
psf 0.05149999282535407
bkg 0.2585585415363312 1 / (MeV s sr)
loading 110638
events 100864
aeff 5371581.0 m2
edisp 10595.85546875
psf 0.050999992403622255
bkg 0.4214513599872589 1 / (MeV s sr)
loading 111138
events 103113
aeff 5371581.0 m2
edisp 10595.85546875
psf 0.050999992403622255
bkg 0.4214513599872589 1 / (MeV s sr)
loading 111638
events 101312
aeff 5371581.0 m2
edisp 10595.85546875
psf 0.050999992403622255
bkg 0.4214513599872589 1 / (MeV s sr)
loading 112138
events 92079
aeff 1303965.5 m2
edisp 16039.921875
psf 0.058499993105930244
bkg 0.4422248601913452 1 / (MeV s sr)
loading 112638
events 92293
aeff 1303965.5 m2
edisp 16039.921875
psf 0.058499993105930244
bkg 0.4422248601913452 1 / (MeV s sr)
loading 113138
events 91877
aeff 1303965.5 m2
edisp 16039.921875
psf 0.058499993105930244
bkg 0.4422248601913452 1 / (MeV s sr)

In [16]:
ds.info()


Data store summary info:
name: noname

HDU index table:
BASE_DIR: 1dc/1dc/index/all
Rows: 48792
OBS_ID: 110000 -- 511919
HDU_TYPE: ['aeff', 'bkg', 'edisp', 'events', 'gti', 'psf']
HDU_CLASS: ['aeff_2d', 'bkg_3d', 'edisp_2d', 'events', 'gti', 'psf_3gauss']

Observation table:
Number of observations: 8132

Sorted?

Check if OBS_ID is sorted ...


In [17]:
names = 'agn', 'gps', 'gc', 'egal', 'agn'
for name in names:
    print('checking', name)
    ds = DataStore.from_dir('1dc/1dc/index/' + name)
    obs_id = ds.obs_table['OBS_ID']
    print(np.all(np.diff(obs_id) > 0))
    
    obs_id = ds.hdu_table['OBS_ID']
    print(np.all(np.diff(obs_id) >= 0))


checking agn
True
True
checking gps
True
True
checking gc
True
True
checking egal
True
True
checking agn
True
True

In [ ]: