In [1]:

    

import numpy as np
import pandas as pd
import os
from tqdm import tqdm


    

In [2]:

    

import matplotlib.pyplot as plt
%matplotlib inline


    

In [3]:

    

mydir = '/home/fedor/notebook/ml4dc/ok_files'
paths = []
for dirpath, dirnames, filenames in os.walk(mydir):
    if not dirnames:
        paths.append(dirpath)
        print dirpath, "has 0 subdirectories and", len(filenames), "files"


    

    




/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016G-18Apr2017-v1/170808_215231/0000 has 0 subdirectories and 3 files
/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016G-18Apr2017-v1/170808_215231/0001 has 0 subdirectories and 2 files
/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016F-18Apr2017-v1/170808_215223/0000 has 0 subdirectories and 362 files
/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016C-18Apr2017-v1/170808_215201/0000 has 0 subdirectories and 26 files
/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016E-18Apr2017-v1/170808_215215/0000 has 0 subdirectories and 396 files
/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016D-18Apr2017-v1/170808_215208/0000 has 0 subdirectories and 665 files
/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016H-18Apr2017-v1/170808_215239/0000 has 0 subdirectories and 978 files
/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016H-18Apr2017-v1/170808_215239/0001 has 0 subdirectories and 104 files
/home/fedor/notebook/ml4dc/ok_files/SinglePhoton/crab_SinglePhotonRun_2016H-18Apr2017-v1/170808_215104/0000 has 0 subdirectories and 39 files
/home/fedor/notebook/ml4dc/ok_files/SinglePhoton/crab_SinglePhotonRun_2016D-18Apr2017-v1/170808_215023/0000 has 0 subdirectories and 245 files
/home/fedor/notebook/ml4dc/ok_files/SinglePhoton/crab_SinglePhotonRun_2016C-18Apr2017-v1/170808_215010/0000 has 0 subdirectories and 441 files
/home/fedor/notebook/ml4dc/ok_files/SinglePhoton/crab_SinglePhotonRun_2016F-18Apr2017-v1/170808_215043/0000 has 0 subdirectories and 305 files
/home/fedor/notebook/ml4dc/ok_files/SinglePhoton/crab_SinglePhotonRun_2016G-18Apr2017-v1/170808_215053/0000 has 0 subdirectories and 916 files
/home/fedor/notebook/ml4dc/ok_files/SinglePhoton/crab_SinglePhotonRun_2016G-18Apr2017-v1/170808_215053/0001 has 0 subdirectories and 80 files
/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016E-18Apr2017-v1/170808_214925/0000 has 0 subdirectories and 344 files
/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016D-18Apr2017-v1/170808_214916/0000 has 0 subdirectories and 704 files
/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000 has 0 subdirectories and 898 files
/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0001 has 0 subdirectories and 162 files
/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000 has 0 subdirectories and 945 files
/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0001 has 0 subdirectories and 31 files
/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016C-18Apr2017-v1/170808_214908/0000 has 0 subdirectories and 415 files

In [4]:

    

not_presented_everywhere = ['/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016E-18Apr2017-v1/170808_214925/0000',
                            '/home/fedor/notebook/ml4dc/ok_files/SinglePhoton/crab_SinglePhotonRun_2016F-18Apr2017-v1/170808_215043/0000',
                            '/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016E-18Apr2017-v1/170808_215215/0000',
                            '/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016F-18Apr2017-v1/170808_215223/0000']


    

In [5]:

    

working_paths = np.sort(list(set(paths) - set(not_presented_everywhere)))
working_paths


    

    
Out[5]:





array([ '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016C-18Apr2017-v1/170808_214908/0000',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016D-18Apr2017-v1/170808_214916/0000',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0001',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0001',
       '/home/fedor/notebook/ml4dc/ok_files/SinglePhoton/crab_SinglePhotonRun_2016C-18Apr2017-v1/170808_215010/0000',
       '/home/fedor/notebook/ml4dc/ok_files/SinglePhoton/crab_SinglePhotonRun_2016D-18Apr2017-v1/170808_215023/0000',
       '/home/fedor/notebook/ml4dc/ok_files/SinglePhoton/crab_SinglePhotonRun_2016G-18Apr2017-v1/170808_215053/0000',
       '/home/fedor/notebook/ml4dc/ok_files/SinglePhoton/crab_SinglePhotonRun_2016G-18Apr2017-v1/170808_215053/0001',
       '/home/fedor/notebook/ml4dc/ok_files/SinglePhoton/crab_SinglePhotonRun_2016H-18Apr2017-v1/170808_215104/0000',
       '/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016C-18Apr2017-v1/170808_215201/0000',
       '/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016D-18Apr2017-v1/170808_215208/0000',
       '/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016G-18Apr2017-v1/170808_215231/0000',
       '/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016G-18Apr2017-v1/170808_215231/0001',
       '/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016H-18Apr2017-v1/170808_215239/0000',
       '/home/fedor/notebook/ml4dc/ok_files/ZeroBias/crab_ZeroBiasRun_2016H-18Apr2017-v1/170808_215239/0001'],
      dtype='|S107')

In [6]:

    

sample = np.load(working_paths[0] +'/'+ os.listdir(working_paths[0])[0], encoding="latin1")


    

In [7]:

    

all_features = sample.dtype.names


    

In [8]:

    

Ids_labels = ['runId','lumiId','lumi','isSig','subsystemQuality']


    

In [9]:

    

subsystemNames = ['L1tcalo', 'L1tmu', 'Hlt', 'Pix', 'Strip', 'Ecal', 'Hcal', 'Dt',
                    'Rpc', 'Es', 'Csc', 'Track', 'Egamma', 'Muon', 'Jetmet', 'Lumi']
len(subsystemNames)


    

    
Out[9]:





16

In [10]:

    

4+16


    

    
Out[10]:





20

In [11]:

    

def get_channel_frame(paths, features):
    why_number = []
    why_name = []
    frames = []
    for dir in paths:
        sm = os.listdir(dir)
        for f in tqdm(sm):
            b = {}
            a = np.load(dir + "/" + f, encoding="latin1")

            replace = {True:1 , False:0}    
            for i, s_name in enumerate(subsystemNames):
                bb = [] 
                for s_labels in a["subsystemQuality"]:    
                    bb.append(replace[s_labels[i]])
                b[s_name] = bb   

            for name in features:
                for i in range(7):
                    bb = []
                    for k, a_7 in enumerate(a[name]):
                        if len(a_7)<7:
                            why_name.append(dir+"/"+f+"/"+str(a['runId'][k])+"/"+str(a['lumiId'][k]))#+"/"+name)
                            why_number.append(len(a_7))    
                        if i < len(a_7):
                            bb.append(a_7[i])
                        else:    
                            bb.append(None)
                    b[name + '_' + str(i)] = bb

            df = pd.DataFrame(a[Ids_labels[:-1]])
            df_explicit = pd.concat([df, pd.DataFrame.from_dict(b)], axis = 1)
            frames.append(df_explicit)

    result = pd.concat(frames)
    return result, why_number, why_name


    

In [12]:

    

Muon_features = [s for s in all_features if (s[:3] == 'qMu') and (s[3:7] != 'Cosm')]


    

In [13]:

    

removelist = ['qMuCh_', 'qMuChi2_']
Muon_features = [v for v in Muon_features if v not in removelist]


    

In [14]:

    

Muon_features


    

    
Out[14]:





['qMu0Pt',
 'qMu1Pt',
 'qMu2Pt',
 'qMu3Pt',
 'qMu4Pt',
 'qMu5Pt',
 'qMu0Eta',
 'qMu1Eta',
 'qMu2Eta',
 'qMu3Eta',
 'qMu4Eta',
 'qMu5Eta',
 'qMu0Phi',
 'qMu1Phi',
 'qMu2Phi',
 'qMu3Phi',
 'qMu4Phi',
 'qMu5Phi',
 'qMu0En',
 'qMu1En',
 'qMu2En',
 'qMu3En',
 'qMu4En',
 'qMu5En',
 'qMuPt',
 'qMuEta',
 'qMuPhi',
 'qMuEn_']

In [15]:

    

len(Muon_features)


    

    
Out[15]:





28

In [16]:

    

len(Muon_features)*7+20


    

    
Out[16]:





216

In [17]:

    

result_m, num_m, name_m = get_channel_frame(working_paths[0:6], Muon_features)


    

    




100%|██████████| 415/415 [00:43<00:00, 10.33it/s]
100%|██████████| 704/704 [01:14<00:00,  9.59it/s]
100%|██████████| 945/945 [01:40<00:00,  9.10it/s]
100%|██████████| 31/31 [00:03<00:00,  9.53it/s]
100%|██████████| 898/898 [01:34<00:00,  9.41it/s]
100%|██████████| 162/162 [00:17<00:00,  8.92it/s]

In [18]:

    

np.unique(name_m)


    

    
Out[18]:





array([ '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016C-18Apr2017-v1/170808_214908/0000/AODTree_115.npy/276243/592',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016C-18Apr2017-v1/170808_214908/0000/AODTree_365.npy/276243/1',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016C-18Apr2017-v1/170808_214908/0000/AODTree_378.npy/276244/4',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016D-18Apr2017-v1/170808_214916/0000/AODTree_1.npy/276326/3',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016D-18Apr2017-v1/170808_214916/0000/AODTree_560.npy/276655/726',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016D-18Apr2017-v1/170808_214916/0000/AODTree_571.npy/276659/141',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016D-18Apr2017-v1/170808_214916/0000/AODTree_594.npy/276775/1252',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_119.npy/280021/1',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_119.npy/280021/2',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_119.npy/280022/1',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_28.npy/280327/49',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_301.npy/279480/1',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_301.npy/279480/4',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_301.npy/279480/5',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_301.npy/279480/6',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_301.npy/279480/7',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_302.npy/279488/29',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_60.npy/278821/36',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_60.npy/278821/37',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_626.npy/280019/1',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_666.npy/280189/1',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016G-18Apr2017-v1/170808_214948/0000/AODTree_999.npy/280349/626',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_144.npy/282923/38',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_144.npy/282923/39',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_2.npy/281613/101',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_2.npy/281613/117',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_2.npy/281613/118',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_3.npy/281613/121',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_3.npy/281613/122',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_3.npy/281613/123',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_3.npy/281613/126',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_3.npy/281613/128',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_3.npy/281613/130',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_3.npy/281613/133',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_3.npy/281613/135',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_3.npy/281613/136',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_3.npy/281613/137',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_363.npy/283408/79',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/170',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/171',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/64',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/65',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/66',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/67',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/68',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/69',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/70',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/71',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/72',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/73',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/74',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/75',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_45.npy/281663/76',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/100',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/101',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/102',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/79',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/80',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/81',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/83',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/84',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/85',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/86',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/87',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/89',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/90',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/91',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/92',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/94',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/95',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/98',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281663/99',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/1',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/11',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/12',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/13',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/14',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/15',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/16',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/17',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/18',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/19',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/2',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/20',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/3',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/4',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/7',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_46.npy/281674/9',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/21',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/23',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/24',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/26',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/28',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/29',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/30',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/33',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/35',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/36',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/37',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/38',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/41',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/42',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/43',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/44',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281674/45',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/1',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/10',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/11',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/12',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/13',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/14',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/15',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/16',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/17',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/18',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/19',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/2',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/20',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/21',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/22',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/3',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/4',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/5',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/7',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/8',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_47.npy/281680/9',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/105',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/106',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/107',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/108',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/110',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/112',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/113',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/114',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/115',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/116',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/118',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/119',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/120',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/121',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/122',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/123',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/124',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/125',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/126',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/127',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/156',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/157',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/158',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/159',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/160',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/161',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/162',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/163',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/164',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/165',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/166',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_49.npy/281663/168',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_6.npy/281613/256',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_639.npy/283059/127',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_639.npy/283059/128',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_68.npy/282707/81',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_68.npy/282710/8',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_68.npy/282711/1',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_7.npy/281613/250',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_7.npy/281613/251',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_7.npy/281613/253',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_7.npy/281613/255',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_7.npy/281613/258',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_7.npy/281613/259',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_7.npy/281613/260',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_726.npy/283675/11',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_726.npy/283675/12',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_726.npy/283675/7',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_726.npy/283676/2',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_759.npy/283413/1',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_760.npy/283413/5',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_760.npy/283415/4',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_760.npy/283415/5',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_829.npy/283408/2533',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_829.npy/283408/2535',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_868.npy/283416/4',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_868.npy/283416/7',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_869.npy/283416/29',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_869.npy/283416/31',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_869.npy/283416/32',
       '/home/fedor/notebook/ml4dc/ok_files/SingleMuon/crab_SingleMuonRun_2016H-18Apr2017-v1/170808_215001/0000/AODTree_869.npy/283416/8'],
      dtype='|S131')

In [57]:

    

file_name_m = '/home/fedor/notebook/ml4dc/ok_files/muon_df.pickle'
result_m.to_pickle(file_name_m)
#result_m = pd.read_pickle(file_name_m)


    

In [20]:

    

np.mean(result_m["isSig"])


    

    
Out[20]:





0.9593056910182299

In [21]:

    

result_m.shape


    

    
Out[21]:





(126111, 216)

In [22]:

    

result_m.dropna(axis=0, how='any').shape


    

    
Out[22]:





(125924, 216)

In [23]:

    

Pho_features = [s for s in all_features if s[:4] == 'qPho']


    

In [24]:

    

Pho_features = Pho_features[:-10]


    

In [25]:

    

Pho_features


    

    
Out[25]:





['qPho0Pt',
 'qPho1Pt',
 'qPho2Pt',
 'qPho3Pt',
 'qPho4Pt',
 'qPho5Pt',
 'qPho0Eta',
 'qPho1Eta',
 'qPho2Eta',
 'qPho3Eta',
 'qPho4Eta',
 'qPho5Eta',
 'qPho0Phi',
 'qPho1Phi',
 'qPho2Phi',
 'qPho3Phi',
 'qPho4Phi',
 'qPho5Phi',
 'qPho0En',
 'qPho1En',
 'qPho2En',
 'qPho3En',
 'qPho4En',
 'qPho5En',
 'qPhoPt',
 'qPhoEta',
 'qPhoPhi',
 'qPhoEn_']

In [26]:

    

len(Pho_features)


    

    
Out[26]:





28

In [27]:

    

20+(28*7)


    

    
Out[27]:





216

In [28]:

    

result_ph, num_ph, name_ph = get_channel_frame(working_paths[6:11], Pho_features)


    

    




100%|██████████| 441/441 [00:45<00:00, 10.07it/s]
100%|██████████| 245/245 [00:25<00:00,  9.76it/s]
100%|██████████| 916/916 [01:47<00:00,  8.06it/s]
100%|██████████| 80/80 [00:09<00:00,  8.21it/s]
100%|██████████| 39/39 [00:04<00:00,  8.76it/s]

In [58]:

    

file_name_ph = '/home/fedor/notebook/ml4dc/ok_files/Pho_df.pickle'
result_ph.to_pickle(file_name_ph)
#result_ph = pd.read_pickle(file_name)


    

In [30]:

    

result_ph.shape


    

    
Out[30]:





(68775, 216)

In [31]:

    

result_ph.dropna(axis=0, how='any').shape


    

    
Out[31]:





(68691, 216)

In [32]:

    

np.mean(result_ph["isSig"])


    

    
Out[32]:





0.9812141039621955

In [33]:

    

Cal_features = [s for s in all_features if s[:4] == 'qCal']


    

In [34]:

    

Cal_features = Cal_features[:-10]
len(Cal_features)


    

    
Out[34]:





30

In [35]:

    

Cal_features


    

    
Out[35]:





['qCalJet0Pt',
 'qCalJet1Pt',
 'qCalJet2Pt',
 'qCalJet3Pt',
 'qCalJet4Pt',
 'qCalJet5Pt',
 'qCalJet0Eta',
 'qCalJet1Eta',
 'qCalJet2Eta',
 'qCalJet3Eta',
 'qCalJet4Eta',
 'qCalJet5Eta',
 'qCalJet0Phi',
 'qCalJet1Phi',
 'qCalJet2Phi',
 'qCalJet3Phi',
 'qCalJet4Phi',
 'qCalJet5Phi',
 'qCalJet0En',
 'qCalJet1En',
 'qCalJet2En',
 'qCalJet3En',
 'qCalJet4En',
 'qCalJet5En',
 'qCalJetPt',
 'qCalJetEta',
 'qCalJetPhi',
 'qCalJetEn',
 'qCalMETPt',
 'qCalMETPhi']

In [36]:

    

30*7 + 20


    

    
Out[36]:





230

In [37]:

    

result_c, num_c, name_c = get_channel_frame(working_paths[11:], Cal_features)


    

    




100%|██████████| 26/26 [00:06<00:00,  4.37it/s]
100%|██████████| 665/665 [01:39<00:00,  7.93it/s]
100%|██████████| 3/3 [00:00<00:00,  8.26it/s]
100%|██████████| 2/2 [00:00<00:00,  8.35it/s]
100%|██████████| 978/978 [01:58<00:00,  7.91it/s]
100%|██████████| 104/104 [00:12<00:00,  8.40it/s]

In [60]:

    

file_name_c = '/home/fedor/notebook/ml4dc/ok_files/calo_df.pickle'
result_c.to_pickle(file_name_c)
#result_c = pd.read_pickle(file_name)


    

In [39]:

    

result_c.shape


    

    
Out[39]:





(71094, 230)

In [40]:

    

result_c.dropna(axis=0, how='any').shape


    

    
Out[40]:





(70864, 230)

In [41]:

    

np.mean(result_c["isSig"])


    

    
Out[41]:





0.9478718316594931

In [42]:

    

PF_features = [s for s in all_features if s[:3] == 'qPF']


    

In [43]:

    

PF_features = PF_features[:3*7]
len(PF_features)


    

    
Out[43]:





21

In [44]:

    

PF_features


    

    
Out[44]:





['qPFJetPt',
 'qPFJetEta',
 'qPFJetPhi',
 'qPFJet0Pt',
 'qPFJet1Pt',
 'qPFJet2Pt',
 'qPFJet3Pt',
 'qPFJet4Pt',
 'qPFJet5Pt',
 'qPFJet0Eta',
 'qPFJet1Eta',
 'qPFJet2Eta',
 'qPFJet3Eta',
 'qPFJet4Eta',
 'qPFJet5Eta',
 'qPFJet0Phi',
 'qPFJet1Phi',
 'qPFJet2Phi',
 'qPFJet3Phi',
 'qPFJet4Phi',
 'qPFJet5Phi']

In [45]:

    

21*7 + 20


    

    
Out[45]:





167

In [46]:

    

result_PF,  num_PF, name_PF = get_channel_frame(working_paths[11:], PF_features)


    

    




100%|██████████| 26/26 [00:02<00:00,  9.91it/s]
100%|██████████| 665/665 [01:07<00:00,  9.50it/s]
100%|██████████| 3/3 [00:00<00:00, 10.11it/s]
100%|██████████| 2/2 [00:00<00:00, 10.28it/s]
100%|██████████| 978/978 [01:39<00:00,  9.86it/s]
100%|██████████| 104/104 [00:10<00:00,  9.90it/s]

In [59]:

    

file_name_PF = '/home/fedor/notebook/ml4dc/ok_files/PF_df.pickle'
result_PF.to_pickle(file_name_PF)
#result_PF = pd.read_pickle(file_name)


    

In [48]:

    

result_PF.shape


    

    
Out[48]:





(71094, 167)

In [49]:

    

result_PF.dropna(axis=0, how='any').shape


    

    
Out[49]:





(70935, 167)

In [50]:

    

np.mean(result_PF["isSig"])


    

    
Out[50]:





0.9478718316594931

In [51]:

    

plt.hist(num_PF, normed = True, bins = 5)
plt.title('PF: '+str(len(name_PF)/7), size = 20)
plt.show()


    

In [52]:

    

plt.hist(num_c, normed = True, bins = 5)
plt.title('calo: '+str(len(name_c)/7), size = 20)
plt.show()


    

In [53]:

    

plt.hist(num_m, normed = True, bins = 5)
plt.title('muon: '+str(len(name_m)/7), size = 20)
plt.show()


    

In [54]:

    

plt.hist(num_ph, normed = True, bins = 5)
plt.title('photon: '+str(len(name_ph)/7), size = 20)
plt.show()


    

In [61]:

    

result_m = pd.read_pickle(file_name_m)
result_c = pd.read_pickle(file_name_c)
result_ph = pd.read_pickle(file_name_ph)
result_PF = pd.read_pickle(file_name_PF)


    

In [62]:

    

rs = [result_c, result_PF, result_ph, result_m]
ns = ["c", "PF", "ph", "m"]
for r1, n1 in zip(rs, ns):
    for  r2, n2 in zip(rs, ns):
         print n1, n2, pd.merge(r1, r2, how='inner', on=Ids_labels[:-1]+subsystemNames).shape[0]


    

    




c c 71094
c PF 71094
c ph 11422
c m 64235
PF c 71094
PF PF 71094
PF ph 11422
PF m 64235
ph c 11422
ph PF 11422
ph ph 68775
ph m 62766
m c 64235
m PF 64235
m ph 62766
m m 126111

In [63]:

    

a = pd.merge(result_c, result_PF, how='inner', on=Ids_labels[:-1]+subsystemNames)
b = pd.merge(result_ph, result_m, how='inner', on=Ids_labels[:-1]+subsystemNames)
result = pd.merge(a, b, how='inner', on=Ids_labels[:-1]+subsystemNames)


    

In [64]:

    

result.shape


    

    
Out[64]:





(10949, 769)

In [65]:

    

file_name_merged = '/home/fedor/notebook/ml4dc/ok_files/merged.pickle'
result.to_pickle(file_name_merged)
#result = pd.read_pickle(file_name_merged)


    

In [67]:

    

(len(Muon_features)+len(Cal_features)+len(PF_features)+len(Pho_features))*7+len(Ids_labels[:-1]+subsystemNames)


    

    
Out[67]:





769

In [68]:

    

result = result.dropna(axis=0, how='any')


    

In [69]:

    

result.shape


    

    
Out[69]:





(10872, 769)

In [70]:

    

np.mean(result["isSig"])


    

    
Out[70]:





0.9754415011037527

In [71]:

    

w = []
for s_name in subsystemNames:
    w.append((1.-np.mean(result[s_name]))*result.shape[0])

plt.scatter(np.arange(len(subsystemNames)), w)
plt.xticks(np.arange(len(subsystemNames)), subsystemNames, rotation=90, size = 20)
plt.yticks(size = 20)
plt.show()


    

In [ ]: