In [1]:

    

import xgboost
import numpy as np
import pandas as pd
import shap

# load JS visualization code to notebook
shap.initjs()


    

    




/opt/conda/anaconda3/lib/python3.6/importlib/_bootstrap.py:205: RuntimeWarning: compiletime version 3.5 of module '_catboost' does not match runtime version 3.6
  return f(*args, **kwds)






    

In [2]:

    

data_features = pd.read_hdf('/home/olgako/data/data_features_JetHT.hdf5', "data")
labels = 1-pd.read_hdf('/home/olgako/data/labels_JetHT.hdf5', 'labels')
sublabels = pd.read_hdf('//data/cms2010/data_not_f_JetHT.hdf5', "data")


    

In [3]:

    

data_features.shape, sublabels.shape


    

    
Out[3]:





((162990, 2470), (162990, 20))

In [4]:

    

np.where(sublabels['new_json'] != labels)


    

    
Out[4]:





(array([], dtype=int64),)

In [5]:

    

from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve, auc, roc_auc_score

import time


    

In [6]:

    

indx_train = np.arange(data_features.shape[0]-int(data_features.shape[0]/5), dtype='int32')
indx_test = np.arange(data_features.shape[0]-int(data_features.shape[0]/5),data_features.shape[0], dtype='int32')

#indx_train, indx_test = train_test_split(np.arange(data.shape[0], dtype='int32'), stratify=labels, test_size=0.1, random_state = 1)


    

In [7]:

    

num_good = np.sum(labels)
num_bad = len(labels)-np.sum(labels)

weights = 0.5 / np.where(labels == 0.0, num_good, num_bad)
weights *= len(labels)


    

In [8]:

    

y_train = np.array(labels.iloc[indx_train], 'float32')
#y_val = np.array(labels.iloc[indx_val], 'float32')
y_test = np.array(labels.iloc[indx_test], 'float32')

X_train = np.array(data_features.iloc[indx_train], 'float32')
#X_val = np.array(data_features.iloc[indx_val], 'float32')
X_test = np.array(data_features.iloc[indx_test], 'float32')

weights_train = weights[indx_train]
#weights_val = weights[indx_val]
weights_test = weights[indx_test]

ids_train = sublabels[['runId', 'lumiId']].iloc[indx_train]
ids_test = sublabels[['runId', 'lumiId']].iloc[indx_test]


    

In [9]:

    

feature_names = data_features.columns


    

In [10]:

    

Muon_features = [s for s in feature_names if (s[:3] == 'qMu')]# and (s[3:7] != 'Cosm')]
Pho_features = [s for s in feature_names if s[:4] == 'qPho']
Cal_features = [s for s in feature_names if s[:4] == 'qCal']
PF_features = [s for s in feature_names if s[:3] == 'qPF']

channels_features = dict()

channels_features['muons'] = Muon_features
channels_features['photons'] = Pho_features
channels_features['PF'] = PF_features
channels_features['calo'] = Cal_features

[ (g, len(fs)) for g, fs in channels_features.items() ]


    

    
Out[10]:




[('muons', 439), ('photons', 224), ('PF', 878), ('calo', 280)]

In [11]:

    

import xgboost as xgb
xgb_clf  = xgb.XGBClassifier(n_estimators=80, max_depth=10, eta=0.1, n_jobs=-1, random_state=111, silent=False)
xgb_clf.fit(X_train, y_train)


    

    
Out[11]:





XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
       colsample_bytree=1, eta=0.1, gamma=0, learning_rate=0.1,
       max_delta_step=0, max_depth=10, min_child_weight=1, missing=None,
       n_estimators=80, n_jobs=-1, nthread=None,
       objective='binary:logistic', random_state=111, reg_alpha=0,
       reg_lambda=1, scale_pos_weight=1, seed=None, silent=False,
       subsample=1)

In [12]:

    

model = xgb_clf


    

In [14]:

    

dict_causes = {
 273150:" [[61, 64], [66, 75]],     	Ecal excluded",
 273296:" [[1, 94]],			Pixe excluded",
 273426:" [[1, 59], [64, 64], [66, 69]],inefficeincy in EE",
 273445:" [[5, 6]],			tracker off  seen by many systems",
 274142:" [[99, 100]],			pixel and tracker off",
 274157:" [[103, 534]],		EB minus has region with low efficiency",
 274282:" [[86, 88]],			tracker is off --> particle flow object are affected",
 275326:" [[1, 169]],			sistrip excl",
 275757:" [[104, 122]],		low DCS: HBHE. nothing on PF plot",
 275758:" [[1, 4]],			part of hcal not available --> JetMET reconstruction suffers",
 275764:" [[1, 31]],			pixel off",
 275766:" [[1, 23], [25, 60]],		lower tracker efficiency",
 275768:" [[1, 79]],			lower tracker efficiency",
 275769:" [[1, 22]],			pixel not in Data Acquisition",
 275781:" [[1, 29]],			EB+16, +17, +18 are excluded due to a problem with VME crateS206h",
 275783:" [[1, 1634]],			strip EXCL",
 275838:" [[1, 51]],			strip EXCL",
 275922:" [[4, 6]],			low stat",
 276064:" [[1, 22]],			hot trigger tower seen in EE-07 (high occupancy and Et TP) which cause  ~100% deadtime",
 276071:" [[1, 22]],			strip EXCL",
 276095:" [[1, 5]],			low stat",
 276237:" [[1, 5]],			ECAL, HCAL, PIXEL excluded",
 276453:" [[1, 8], [10, 125]],		EB-17 (FED 626): was excluded (because of cooling failure)",
 276455:" [[1, 401]],			strip EXCL",
 276456:" [[1, 182]],			strip EXCL",
 276457:" [[1, 19]],			sistrip not in DAQ",
 277217:" [[12, 14]],			Short collision run with strip in error in DCS and pixel in error in DAQ. For physically meaningful lumisections 33-47, the total rate is zero. L1T flags marked as bad due to this",
 277933:" [[1, 42]],			ECAL EE+09 FED 648 removed from all LS in the run because it was causing 100% dead time. EE+09 was not masked, so all LS in this run are bad",
 278309:" [[1, 10]],			EE-04 FED 607 TT disabled in LS [1-10] according to express dataset (LS# 10)",
 278821:" [[1, 33], [36, 37]],		FED652 in error, EE+04 is off",
 279028:" [[1, 17]],			strip EXCL",
 279995:" [[1, 8]],			hcal water colling issues",
 280002:" [[1, 111]],			ecal excluded",
 280006:" [[1, 68]],			EB-11 token ring missing",
 280007:" [[1, 36]],			Low voltage channel broken in EB",
 280239:" [[1, 9]],			strip EXCL",
 280241:" [[1, 11]],			strip EXCL",
 281663:" [[61, 172]],			Timing shifted by 5ns due to TCDS* problem ",
 281674:" [[1, 45]],			Timing shifted by 5ns due to TCDS* problem",
 281680:" [[1, 31]],			Timing shifted by 5ns due to TCDS* problem",
 281974:" [[79, 85]],			pixel High voltage OFF",
 282408:" [[80, 191]],			strip EXCL",
 282707:" [[79, 81]],			pixel High voltage OFF",
 282796:" [[80, 82]],			pixel High Voltage off",
 282921:" [[1, 152]]		        strip EXCL"

    
}


    

In [16]:

    

shap_values_test = shap.TreeExplainer(model).shap_values(X_test)
shap_values_train = shap.TreeExplainer(model).shap_values(X_train)


    

In [17]:

    

min_num_neg = 2000
for i in range(shap_values_train.shape[0]):
    cur = len(shap_values_train[i,:-1][shap_values_train[i,:-1]<0])
    if cur < min_num_neg:
        min_num_neg = cur
for i in range(shap_values_test.shape[0]):
    cur = len(shap_values_train[i,:-1][shap_values_test[i,:-1]<0])
    if cur < min_num_neg:
        min_num_neg = cur
        
print(min_num_neg)


    

    




479

In [18]:

    

for run in dict_causes.keys():
    if run in sublabels['runId'].values:
        print('')
        print (run, dict_causes[run])
        print('')
        
        if run in ids_train['runId'].values:
            print ('this run from TRAIN set')
            shap_values = shap_values_train
            search_for_ind = ids_train
            X = X_train
            
        else:
            print ('this run from TEST set')
            shap_values = shap_values_test
            search_for_ind = ids_test
            X = X_test
            
        
        inds = np.where(np.array(search_for_ind)==run)[0]

        print ("sum of features' influences from all lumis, 20 the most important")
        print('')
        sum_shap = np.sum(shap_values[inds,:-1], axis=0)
        print (feature_names[np.argsort(sum_shap, axis=0)][:20])
        print('')
        print("the most common features with big shap values")
        print('')
        top_sh_values = np.argsort(shap_values[inds,:-1], axis=1)[:,:50].T.reshape(-1)
        unique, first_index, counts = np.unique(
            top_sh_values,
        return_counts=True, return_index=True)
        print(feature_names[top_sh_values[np.sort(first_index[counts == len(inds)])]])
        print('')
        #random example 
        #shap.force_plot(shap_values[np.random.choice(inds,1),:], feature_names)
        shap.summary_plot(shap_values[inds,:], features=X[inds, :], feature_names=feature_names)
        
        channels_shap = pd.DataFrame()

        for i in shap_values[inds,:-1]:

            pred_for_channnel = {}
            for channel in channels_features:
                current_sum = 0
                for feature in channels_features[channel]:
                    current_sum += i[list(feature_names).index(feature)]
                pred_for_channnel[channel] = current_sum
            channels_shap = channels_shap.append(pred_for_channnel, ignore_index=True)

        print ('shap for channel for lumi')
        print (channels_shap)
        print(" ")
        print ('shap for channel for run')
        print(channels_shap.sum())
        print(" ")
        print("_"*100)
        
    else:
        print (str(run) +' is not found')


    

    




273150 is not found
273296 is not found
273426 is not found
273445 is not found
274142 is not found
274157 is not found
274282 is not found
275326 is not found

275757  [[104, 122]],		low DCS: HBHE. nothing on PF plot

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['qPFJet4CHSEta_3', 'qEEchi2_3', 'qEEchi2_1', 'qPFJetEIEta_3',
       'qSCEtaWidth_0', 'qSCEtaWidth5x5_1', 'qEEenergy_4', 'qESenergy_3',
       'qEEtime_3', 'qCCEta5x5_0', 'qPFMetPt_5', 'qEBchi2_0', 'qCCEta5x5_4',
       'qCCEn5x5_5', 'qSCEtaWidth5x5_3', 'qPFMetPhi_0', 'qESenergy_0',
       'qCCEta5x5_5', 'qHBHEtime_3', 'qPFMetPhi_4'],
      dtype='object')

the most common features with big shap values

Index(['qPFJet4CHSEta_3', 'qEEchi2_3', 'qEEchi2_1', 'qPFJetEIEta_3',
       'qSCEtaWidth_0', 'qSCEtaWidth5x5_1', 'qEEenergy_4', 'qESenergy_3',
       'qEEtime_3', 'qCCEta5x5_0', 'qEBchi2_0', 'qSCEtaWidth5x5_3',
       'qPFMetPhi_0', 'qHBHEtime_3', 'qCCEta5x5_4', 'qCCEta5x5_5',
       'qCCEn5x5_5', 'qESenergy_0', 'qEBchi2_1', 'qEBtime_3', 'qHFiphi_1',
       'qPFMetPt_0', 'qPreShYEta_5'],
      dtype='object')







    













    




shap for channel for lumi
          PF      calo     muons   photons
0  -4.480931 -0.311185  0.061625 -0.085685
1  -4.464724  0.028613  0.044134  0.029672
2  -4.263342  0.024582 -0.097151 -0.027435
3  -4.375052 -0.087929 -0.164838 -0.017423
4  -4.066766 -0.112606 -0.141677  0.068530
5  -4.212752 -0.075685 -0.092109 -0.064989
6  -4.059303 -0.044144 -0.008511 -0.133038
7  -3.807808 -0.080906 -0.007363 -0.016384
8  -3.819469  0.165778 -0.025631 -0.034982
9  -4.016434 -0.022186  0.079831 -0.046430
10 -4.337837 -0.040294 -0.090593 -0.031276
11 -4.081968 -0.013578  0.060605 -0.046782
12 -4.289302 -0.034557 -0.002195 -0.081137
13 -4.237392  0.030834  0.152573 -0.069418
14 -4.084047 -0.014159  0.073333 -0.041775
15 -4.108532  0.066271  0.005398 -0.151719
16 -4.376708  0.113550 -0.019795 -0.076229
 
shap for channel for run
PF        -71.082368
calo       -0.407601
muons      -0.172366
photons    -0.826500
dtype: float64
 
____________________________________________________________________________________________________

275758  [[1, 4]],			part of hcal not available --> JetMET reconstruction suffers

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['qEEchi2_3', 'qPFJet4CHSEta_3', 'qEEchi2_1', 'qPFJetEIEta_3',
       'qSCEtaWidth_0', 'qSCEtaWidth5x5_1', 'qPFMetPhi_1', 'qEEenergy_4',
       'qESenergy_3', 'qEEtime_3', 'qCCEta5x5_5', 'qEBchi2_0', 'qCCEta5x5_0',
       'qCCEn5x5_5', 'qCCEta5x5_4', 'qESenergy_0', 'qHBHEtime_3', 'qEBtime_3',
       'qSCEtaWidth5x5_3', 'qPhor9__0'],
      dtype='object')

the most common features with big shap values

Index(['qPFJet4CHSEta_3', 'qEEchi2_3', 'qEEchi2_1', 'qSCEtaWidth_0',
       'qPFJetEIEta_3', 'qSCEtaWidth5x5_1', 'qEEenergy_4', 'qEEtime_3',
       'qESenergy_3', 'qCCEta5x5_0', 'qCCEta5x5_5', 'qEBchi2_0', 'qCCEn5x5_5',
       'qESenergy_0', 'qSCEtaWidth5x5_3', 'qEBtime_3', 'qPreShEta_5',
       'qPFMetPt_0', 'qEBchi2_1', 'qPreShEn_5'],
      dtype='object')







    













    




shap for channel for lumi
         PF      calo     muons   photons
0 -4.066446  0.043025  0.097570  0.019787
1 -4.539039  0.050187 -0.141858 -0.125326
2 -3.363135  0.137434  0.169866 -0.176469
3 -0.945124  0.020748 -0.232656 -0.343954
 
shap for channel for run
PF        -12.913744
calo        0.251394
muons      -0.107078
photons    -0.625962
dtype: float64
 
____________________________________________________________________________________________________

275764  [[1, 31]],			pixel off

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['qEEchi2_3', 'qPFMetPhi_1', 'qEEchi2_1', 'qPFJet4CHSEta_3',
       'qPFMetPhi_5', 'qPFJetEIEta_3', 'qPFJetPhi_5', 'qSCEtaWidth5x5_1',
       'qEEtime_3', 'qSCEtaWidth_0', 'qEBchi2_0', 'qPFJetPhi_0', 'qMuPhi_5',
       'qEBenergy_5', 'qESenergy_3', 'qPFMetPt_5', 'qEEenergy_4',
       'qPFMetPhi_4', 'lumi', 'qPFMetPhi_0'],
      dtype='object')

the most common features with big shap values

Index(['qEEchi2_3', 'qPFMetPhi_1', 'qEBchi2_0', 'qMuPhi_5', 'qEEtime_3',
       'qESenergy_3', 'qEBchi2_1'],
      dtype='object')







    













    




shap for channel for lumi
          PF      calo     muons   photons
0  -2.691367 -0.208528 -0.637027  0.086595
1  -1.033206 -0.319695 -0.316106 -0.144838
2  -2.770467 -0.151827 -0.387826 -0.423931
3  -4.217427 -0.168139 -0.580438  0.007884
4  -4.800518  0.154297 -0.275665  0.005598
5  -3.500451 -0.339677 -0.395345 -0.225100
6  -0.938572 -0.247349 -0.299980 -0.031750
7  -5.767483  0.108732 -0.340285  0.004691
8  -5.382938  0.127485 -0.208420  0.002612
9  -5.308576 -0.102331 -0.337830 -0.093058
10 -5.941640 -0.005802 -0.204136 -0.015078
11 -6.058883  0.056901 -0.193921 -0.166297
12 -6.183829  0.002377 -0.230265  0.019943
13 -6.003539  0.019372 -0.353853 -0.069530
14 -5.837561  0.066013 -0.211265  0.073161
15 -6.094262  0.036267 -0.178112 -0.149152
16 -5.591837 -0.134409 -0.178414 -0.077475
17 -6.236649  0.025414 -0.282204 -0.019350
18 -6.069001  0.029318 -0.292206  0.000019
19 -6.068723  0.094107 -0.381210 -0.114056
20 -6.148722  0.113655 -0.212591  0.028571
21 -5.847489 -0.040581 -0.279603  0.019531
22 -5.967857  0.016213 -0.380391  0.001440
23 -6.100565 -0.060417 -0.247128 -0.032338
24 -5.055038  0.060420 -0.256964 -0.023746
25 -5.790331 -0.074303 -0.299976 -0.097841
26 -6.053463  0.032443 -0.176299 -0.055699
27 -5.284070 -0.051172 -0.373531  0.017149
28 -5.580186  0.070164 -0.230239 -0.149156
29 -4.517259  0.095847 -0.343102 -0.044134
 
shap for channel for run
PF        -152.841911
calo        -0.795206
muons       -9.084332
photons     -1.665335
dtype: float64
 
____________________________________________________________________________________________________

275766  [[1, 23], [25, 60]],		lower tracker efficiency

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['qPFMetPhi_1', 'qEEchi2_1', 'qPFMetPhi_5', 'qEEchi2_3',
       'qPFJet4CHSEta_3', 'qPFJetPhi_5', 'qPFJetEIEta_3', 'qPFJetPhi_0',
       'qMuPhi_5', 'qEBenergy_5', 'qSCEtaWidth5x5_1', 'qPFMetPt_5',
       'qEEtime_3', 'qSCEtaWidth_0', 'lumi', 'qESenergy_3', 'qPFMetPhi_4',
       'qEEenergy_4', 'qEBchi2_0', 'qEBtime_3'],
      dtype='object')

the most common features with big shap values

Index(['qPFMetPhi_1', 'qPFMetPhi_5', 'qEEchi2_1', 'lumi', 'qPFJetPhi_5',
       'qPFJetPhi_0', 'qPFMetPhi_4', 'qMuPhi_5', 'qPFMetPhi_0', 'qEEtime_3',
       'qPFMetPt_5', 'qESenergy_3', 'qEBchi2_0', 'qEBtime_3', 'qESenergy_0',
       'qPFChMetPhi_0', 'qEBchi2_1'],
      dtype='object')







    













    




shap for channel for lumi
          PF      calo     muons   photons
0  -4.908292  0.039050 -0.279551 -0.036543
1  -6.116286  0.127903 -0.188708  0.025025
2  -4.897636  0.000473 -0.316865  0.137016
3  -6.115940  0.142883 -0.196327 -0.002732
4  -6.040341  0.053795 -0.349177 -0.120878
5  -6.155712  0.037644 -0.250108  0.043688
6  -6.054021 -0.047264 -0.158921 -0.013266
7  -6.035373  0.103220 -0.190470  0.012760
8  -5.677055  0.040499 -0.259400 -0.105246
9  -5.238450  0.085549 -0.211542 -0.079465
10 -6.090275 -0.037493 -0.299963  0.021839
11 -5.651106 -0.055158 -0.079000  0.077782
12 -6.039999 -0.039125 -0.176047 -0.029638
13 -6.007029  0.030210 -0.221989 -0.119813
14 -5.870022  0.157652 -0.217667 -0.084606
15 -5.809810  0.013983 -0.158457 -0.040475
16 -6.047771  0.191054 -0.133288 -0.054626
17 -5.560540 -0.029584 -0.220868 -0.045477
18 -6.363060 -0.018629 -0.341951 -0.020036
19 -5.003927  0.099033 -0.339018 -0.065505
20 -6.261086  0.089416 -0.178929 -0.135411
21 -5.513042 -0.030904 -0.247793 -0.055287
22 -6.242721  0.030258 -0.315685 -0.037715
23 -6.202662  0.116544 -0.336596 -0.008494
24 -6.119557  0.012112 -0.149767 -0.008674
25 -5.507650  0.022190 -0.217225 -0.004480
26 -6.040002  0.104440 -0.135378  0.007136
27 -5.932161 -0.054181 -0.261140 -0.091550
28 -4.690389  0.085178 -0.314514 -0.133262
29 -5.863313  0.016344 -0.302087  0.028825
30 -5.865557  0.115330 -0.220102 -0.082445
31 -5.262765  0.053337 -0.458839 -0.021188
32 -5.940969  0.080720 -0.350954  0.071197
33 -5.543273  0.065830 -0.318158  0.026328
34 -6.083268 -0.016943 -0.326707 -0.015893
35 -6.377951  0.106283 -0.367542 -0.014925
36 -6.210455 -0.000401 -0.397549  0.099656
37 -6.320689  0.023739 -0.379701  0.017177
38 -5.160899  0.072199 -0.385673  0.013364
39 -7.352629 -0.057453 -0.678937  0.022233
40 -8.049681 -0.027752 -0.735282  0.192611
41 -7.495477 -0.046431 -0.454624  0.083642
42 -7.215045 -0.129130 -0.654474  0.153009
43 -7.647434 -0.026978 -0.798585  0.103186
44 -7.618067  0.055095 -0.544021  0.228728
45 -7.825826 -0.031426 -0.752130  0.110480
46 -7.259235  0.031128 -0.620000  0.070396
47 -7.248223  0.033280 -0.725501  0.066034
48 -7.170826 -0.101241 -0.596986  0.054970
49 -7.714452 -0.004706 -0.556017  0.053942
50 -7.136064 -0.024365 -0.662390  0.057987
51 -7.725130  0.000979 -0.544778  0.101349
52 -7.778135 -0.057313 -0.557729  0.056042
53 -4.196930 -0.497134 -0.691573 -0.394397
 
shap for channel for run
PF        -336.254208
calo         0.903742
muons      -19.826686
photons      0.114375
dtype: float64
 
____________________________________________________________________________________________________

275768  [[1, 79]],			lower tracker efficiency

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['qPFMetPhi_1', 'qPFMetPhi_5', 'qEEchi2_1', 'qPFJetPhi_5', 'qPFJetPhi_0',
       'qMuPhi_5', 'qEBenergy_5', 'qPFMetPt_5', 'qEEtime_3', 'lumi',
       'qPFMetPhi_4', 'qESenergy_3', 'qEBtime_3', 'qPFJetPhi_3', 'qPFJetEta_4',
       'qESenergy_0', 'qgedPhor9__5', 'qMuPhi_1', 'qPFMetPhi_0',
       'qPFJetPhi_1'],
      dtype='object')

the most common features with big shap values

Index(['qPFMetPhi_1', 'qPFMetPhi_5', 'qEEchi2_1', 'qPFJetPhi_5', 'qPFJetPhi_0',
       'qMuPhi_5', 'qEEtime_3', 'qPFMetPhi_4', 'lumi', 'qPFMetPhi_0',
       'qESenergy_3', 'qEBtime_3', 'qPFJetPhi_1', 'qgedPhor9__5',
       'qPFChMetPhi_0'],
      dtype='object')







    













    




shap for channel for lumi
          PF      calo     muons   photons
0  -7.514956  0.073680 -0.570627 -0.084646
1  -7.883093 -0.030464 -0.725244  0.083760
2  -7.296569 -0.072055 -0.670896  0.074012
3  -7.692669 -0.011377 -0.646535  0.086953
4  -7.542675 -0.068124 -0.768697  0.133434
5  -7.753272 -0.021956 -0.634180  0.032811
6  -7.774970  0.008443 -0.686979  0.116104
7  -7.406050  0.088079 -0.523167  0.044981
8  -7.269684 -0.084631 -0.655051  0.181659
9  -7.222670  0.061181 -0.603709  0.020746
10 -7.718063 -0.008409 -0.712558  0.121485
11 -7.458782  0.189805 -0.723338  0.085860
12 -7.442698 -0.103592 -0.676952  0.019966
13 -7.122521 -0.024987 -0.632374  0.082890
14 -6.808794 -0.028314 -0.811212  0.118204
15 -7.676033  0.035017 -0.632686  0.136379
16 -7.494928  0.167038 -0.690544  0.051475
17 -7.714595 -0.025889 -0.532001  0.011785
18 -7.947443 -0.065660 -0.709955  0.073051
19 -5.567271 -0.267973 -0.672254  0.072918
20 -7.958593  0.014327 -0.406199  0.025424
21 -7.882937 -0.106560 -0.738073  0.129999
22 -7.665645  0.078646 -0.795341  0.146705
23 -7.795975  0.058972 -0.667149  0.106562
24 -7.818736 -0.012766 -0.458731  0.088063
25 -7.447148  0.013527 -0.592600  0.082621
26 -7.498203 -0.022970 -0.648520  0.071220
27 -7.716961  0.086734 -0.870037  0.035438
28 -7.984844 -0.047445 -0.784547  0.075536
29 -7.623548  0.050828 -0.653559  0.152982
..       ...       ...       ...       ...
46 -7.720680  0.089500 -0.600403  0.111657
47 -7.803153  0.095253 -0.587075  0.040022
48 -7.780576  0.120432 -0.624145  0.102845
49 -7.619498  0.087077 -0.649904  0.123185
50 -5.838120 -0.172159 -0.699910  0.062716
51 -7.178427  0.076021 -0.636305  0.007690
52 -7.159058 -0.104785 -0.586222  0.102594
53 -7.624866  0.107500 -0.600710  0.114709
54 -7.947481  0.066006 -0.756763  0.062838
55 -7.538588  0.120330 -0.721506  0.005458
56 -6.269637  0.060437 -0.668449 -0.059611
57 -7.382842  0.142085 -0.673788  0.132396
58 -7.166947  0.145325 -0.557377 -0.053593
59 -7.625602  0.114036 -0.716830  0.056588
60 -7.543825  0.128091 -0.583205  0.097286
61 -7.561980  0.076555 -0.739899  0.043936
62 -7.807537  0.046488 -0.669985  0.039913
63 -7.955159  0.122509 -0.806402  0.022059
64 -7.971722  0.023525 -0.771059  0.137020
65 -7.552231 -0.132108 -0.774559  0.086544
66 -7.480401 -0.059335 -0.574234  0.098653
67 -7.791198  0.130282 -0.664388  0.078843
68 -7.870004  0.110758 -0.536819  0.119364
69 -7.735523  0.080481 -0.643060  0.055635
70 -6.906496 -0.061033 -0.517811  0.026727
71 -6.067870  0.098179 -0.631248  0.048396
72 -7.910908  0.093588 -0.525778  0.076638
73 -7.334815 -0.082133 -0.614135  0.107828
74 -7.739089  0.071737 -0.476590  0.137307
75 -7.391920 -0.001869 -0.473500  0.136842

[76 rows x 4 columns]
 
shap for channel for run
PF        -568.550757
calo         2.449195
muons      -50.076314
photons      6.132810
dtype: float64
 
____________________________________________________________________________________________________
275769 is not found

275781  [[1, 29]],			EB+16, +17, +18 are excluded due to a problem with VME crateS206h

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['qEBiPhi_0', 'qEBiPhi_5', 'lumi', 'qCalMETMPhi_1', 'qEBiPhi_1',
       'qPFMetPhi_1', 'qEEenergy_3', 'qEBtime_1', 'qgedPhoPhi_1', 'qPFMetPt_0',
       'qEEtime_3', 'qCalMETPhi_1', 'qPFJet8CHSEta_1', 'qCalMETBEPhi_4',
       'qPFJet4CHSEta_4', 'qCalMETBEFOPhi_4', 'qCalMETPhi_5',
       'qCalMETBEFOPhi_5', 'qESenergy_3', 'qCalMETMPhi_5'],
      dtype='object')

the most common features with big shap values

Index(['qEBiPhi_0', 'qEBiPhi_5', 'lumi', 'qCalMETMPhi_1', 'qPFMetPt_0',
       'qEBiPhi_1', 'qPFMetPhi_1', 'qEBtime_1', 'qCalMETPhi_1',
       'qCalMETMPhi_5', 'qPFJet4CHSEta_4', 'qPFJet8CHSEta_1',
       'qCalMETBEFOPhi_5', 'qCalMETPhi_5', 'qCalMETEn_0', 'qPFJetEIEta_4',
       'qPFJet4Eta_6'],
      dtype='object')







    













    




shap for channel for lumi
          PF      calo     muons   photons
0  -0.683443 -1.100250 -0.096531 -0.110985
1  -0.943994 -1.124269  0.092371 -0.020137
2  -1.020036 -1.117755  0.038461 -0.077497
3  -0.862463 -1.167700  0.040273 -0.019720
4  -0.662206 -1.201235  0.057556 -0.201828
5  -1.062818 -1.003624 -0.040736 -0.241518
6  -0.853715 -1.078456  0.111101 -0.049063
7  -0.433053 -1.175389  0.063316 -0.085657
8  -0.869391 -0.959071  0.006789 -0.109844
9  -0.691347 -1.003728 -0.010747 -0.078152
10 -0.498066 -1.015760  0.002909 -0.103174
11 -0.567391 -1.048307  0.107977 -0.031252
12 -0.701145 -1.011767  0.018813 -0.065249
13 -0.826532 -1.184089  0.018391 -0.009254
14 -1.005093 -1.080653  0.056451 -0.008231
15 -0.625206 -1.046084  0.018988 -0.055745
16 -0.592922 -1.089887  0.089458 -0.015623
17 -0.649803 -1.200980 -0.044651  0.005362
18 -0.509336 -1.194661  0.034699 -0.070906
19 -0.473712 -1.091632  0.125411 -0.012934
20 -0.565729 -1.000997  0.015684  0.014543
21 -0.424336 -0.957973  0.011963 -0.026742
22 -0.618593 -1.060872  0.035701 -0.030305
23 -1.026816 -0.952322 -0.001507 -0.033477
24 -0.659980 -1.008812  0.114351 -0.019697
25 -0.560265 -0.974310 -0.009404 -0.083134
26 -0.387798 -1.188847  0.043577  0.018298
27 -0.381219 -1.067510  0.168448  0.060150
28 -0.439470 -1.007667  0.116005 -0.006010
 
shap for channel for run
PF        -19.595876
calo      -31.114609
muons       1.185118
photons    -1.467781
dtype: float64
 
____________________________________________________________________________________________________
275783 is not found
275838 is not found

275922  [[4, 6]],			low stat

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['qEBtime_4', 'qEEtime_3', 'qEBtime_1', 'qESenergy_4', 'qEEenergy_3',
       'qPreShYEn_4', 'qESenergy_3', 'lumi', 'qPreShYEn_5', 'qHBHEauxe_0',
       'qPFJet8CHSPt_4', 'qHFiphi_1', 'qEEchi2_0', 'qPFJet4CHSEta_4',
       'qHFieta_1', 'qESenergy_5', 'qCalJetPt_3', 'qHBHEtime_1', 'qHBHEtime_0',
       'qSCEtaWidth5x5_3'],
      dtype='object')

the most common features with big shap values

Index(['qEBtime_4', 'qEEtime_3', 'qESenergy_4', 'qEBtime_1', 'lumi',
       'qEEenergy_3', 'qHFiphi_1', 'qESenergy_3', 'qPreShYEn_4', 'qPreShYEn_5',
       'qPFJet4CHSEta_4', 'qHBHEauxe_0', 'qHBHEtime_1', 'qCalJetPt_3',
       'qHFieta_1', 'qr9_1', 'qEEchi2_0', 'qESenergy_5', 'qHBHEauxe_1',
       'qHBHEtime_0', 'qSCEtaWidth5x5_3', 'qPreShEn_5', 'qPFJet4CHSPt_0',
       'qPFJetTopCHS3Pt_6'],
      dtype='object')







    













    




shap for channel for lumi
         PF      calo     muons   photons
0 -0.850799 -0.435677 -0.140928 -0.276381
1 -1.025683 -0.573678  0.001643 -0.087827
2 -1.195430 -0.630699 -0.201622 -0.145415
 
shap for channel for run
PF        -3.071912
calo      -1.640054
muons     -0.340906
photons   -0.509622
dtype: float64
 
____________________________________________________________________________________________________
276064 is not found
276071 is not found

276095  [[1, 5]],			low stat

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['lumi', 'qEEchi2_1', 'qEBenergy_5', 'qEBchi2_0', 'qPFJet4CHSEta_3',
       'qPFJetEta_1', 'qPFMetPt_5', 'qEEtime_1', 'qSCEtaWidth_0',
       'qPFJetEta_3', 'qPFJetEIEta_3', 'qEBiPhi_1', 'qPFJet5Eta_0',
       'qCalMETBEPhi_2', 'qPhomaxenxtal__5', 'qEBtime_4', 'qSCEtaWidth5x5_3',
       'qESenergy_3', 'qHBHEtime_3', 'qCCPhi5x5_6'],
      dtype='object')

the most common features with big shap values

Index(['lumi', 'qEEchi2_1', 'qEBchi2_0', 'qPFJetEta_1', 'qEEtime_1',
       'qPFMetPt_5', 'qPFJetEta_3', 'qESenergy_3', 'qEBtime_4', 'qCCEta5x5_3'],
      dtype='object')







    













    




shap for channel for lumi
         PF      calo     muons   photons
0 -1.934225 -0.291967 -0.124227 -0.319623
1 -1.602892 -0.501380 -0.297347 -0.127083
2 -1.271193 -0.442608 -0.304477 -0.483947
3 -1.749956 -0.167036 -0.092144 -0.083618
4 -4.581241 -0.672450 -0.029779 -0.216555
 
shap for channel for run
PF        -11.139507
calo       -2.075439
muons      -0.847975
photons    -1.230827
dtype: float64
 
____________________________________________________________________________________________________
276237 is not found

276453  [[1, 8], [10, 125]],		EB-17 (FED 626): was excluded (because of cooling failure)

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['qEBiPhi_0', 'qEBiPhi_5', 'qEBiPhi_1', 'lumi', 'qEEenergy_3',
       'qEBtime_4', 'qEBtime_1', 'qEBiPhi_4', 'qSCEtaWidth_0',
       'qSCEtaWidth5x5_1', 'qHBHEtime_1', 'qHBHEauxe_0', 'qEBenergy_5',
       'qPFMetPhi_1', 'qCalMETMPhi_1', 'qCalMETBEFOPhi_5', 'qPFJet8CHSEta_1',
       'qgedPhosigmaeta__0', 'qMuCosm3Eta_1', 'qHBHEtime_0'],
      dtype='object')

the most common features with big shap values

Index(['qEBiPhi_5', 'qEBiPhi_1', 'qEBtime_4', 'qEEenergy_3'], dtype='object')







    













    




shap for channel for lumi
           PF      calo     muons   photons
0   -0.367743 -0.378922  0.024032 -0.155805
1   -0.419268 -0.909160  0.082257  0.027472
2   -0.164030 -0.279305  0.048210 -0.027402
3   -0.105581 -0.623691  0.127599 -0.099339
4   -0.214790 -0.391531  0.179732 -0.101739
5   -0.419663 -0.764590  0.011703 -0.012123
6   -0.669243 -0.957081 -0.021449 -0.061395
7   -1.868776 -0.514251  0.111449 -0.515511
8   -1.308416 -0.579156  0.080654 -0.411247
9   -1.080147 -0.410182  0.117272 -0.373114
10  -0.940457 -0.561287  0.025675 -0.474983
11  -0.681657 -0.203601  0.120238 -0.361823
12  -1.006569 -0.204484 -0.176451 -0.368070
13  -0.741003  0.167457 -0.238439 -0.407377
14  -1.079935 -0.239887 -0.097363 -0.363288
15  -0.915830 -0.443253 -0.000032 -0.328944
16  -1.574906 -0.663276 -0.160563 -0.517671
17  -1.622814 -0.090094 -0.206867 -0.679443
18  -1.145109 -0.563352 -0.215112 -0.296028
19  -0.983944 -0.160478 -0.161963 -0.425647
20  -0.730252 -0.577062 -0.157454 -0.279229
21  -0.693270 -0.034609 -0.140629 -0.702985
22  -0.500088 -0.233253 -0.139381 -0.382159
23  -0.820323 -0.580910 -0.219394 -0.138129
24  -0.590479 -0.393662 -0.245888 -0.059373
25  -0.544113 -0.602286 -0.533213  0.027477
26  -0.424841 -0.086805 -0.176347 -0.280017
27  -0.852939 -0.383342 -0.088941 -0.306786
28  -1.001164 -0.370155 -0.133599 -0.390064
29  -0.671023 -0.550613 -0.049277 -0.092499
..        ...       ...       ...       ...
77  -0.002874 -0.897299 -0.016720  0.036214
78  -0.195509 -0.639402  0.095197  0.095289
79  -0.297432  0.130173  0.129274  0.018250
80   0.144834 -0.476518  0.148912  0.001101
81  -0.293513 -0.804498  0.172329  0.023286
82   0.063342 -0.608139  0.032556  0.022968
83  -0.055555 -0.483100 -0.010532  0.068783
84  -0.228385 -0.681769  0.182508 -0.016970
85   0.126428 -0.762685  0.015219  0.021460
86  -0.079931 -0.583132  0.162294  0.039399
87  -0.070854 -0.545249  0.091229  0.005145
88  -0.231920 -0.631747  0.105468  0.006107
89  -0.217833 -0.705586  0.025919  0.047372
90  -0.020651 -0.524684  0.100724  0.077067
91  -0.123789 -0.447531  0.013186 -0.084865
92  -0.246388 -0.514167  0.080504  0.006162
93   0.224787 -0.325957  0.064225  0.082374
94  -0.543162 -0.768565  0.057596  0.026088
95   0.189551 -0.333868 -0.029221  0.098844
96  -0.133890 -0.569464 -0.040688 -0.004987
97  -0.080958 -0.991861  0.072632  0.013752
98  -0.111285 -0.509019  0.172782 -0.010868
99  -0.135554 -0.190134  0.056472  0.019360
100 -0.056457 -0.163201 -0.149107 -0.019090
101 -0.581747 -0.589769  0.105462  0.005623
102 -0.161539 -0.501138  0.132739 -0.044748
103  0.081021 -0.461270  0.161157  0.017227
104 -0.085545 -0.596366  0.012386  0.040099
105  0.126417 -0.514937  0.104243  0.031084
106 -0.671532 -0.921460 -0.023855  0.034984

[107 rows x 4 columns]
 
shap for channel for run
PF        -38.042401
calo      -54.665078
muons      -0.938509
photons    -8.231512
dtype: float64
 
____________________________________________________________________________________________________
276455 is not found
276456 is not found
276457 is not found
277217 is not found
277933 is not found

278309  [[1, 10]],			EE-04 FED 607 TT disabled in LS [1-10] according to express dataset (LS# 10)

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['qCCPhi5x5_0', 'qCCPhi5x5_5', 'qCCEn5x5_5', 'qPFMetPhi_1',
       'qCalJet1Pt_4', 'qCCEta5x5_4', 'qCCPhi5x5_4', 'qEEenergy_3', 'qEEix_4',
       'qCCEta5x5_0', 'qEEiy_0', 'qSCPhi5x5_3', 'qPreShPhi_0', 'qHBHEtime_0',
       'qEBtime_3', 'qEBtime_0', 'qPFJet8CHSEta_1', 'qPreShEn_5', 'qEEtime_3',
       'qEBtime_5'],
      dtype='object')

the most common features with big shap values

Index(['qCCPhi5x5_0', 'qCCPhi5x5_5', 'qPFMetPhi_1', 'qCCEn5x5_5',
       'qCalJet1Pt_4', 'qCCEta5x5_4', 'qEEiy_0', 'qCCPhi5x5_4', 'qEEenergy_3',
       'qEEix_4', 'qCCEta5x5_0', 'qSCPhi5x5_3', 'qEBtime_3', 'qPreShEn_5',
       'qEBtime_0', 'qPFJet8CHSEta_1', 'qEEchi2_4'],
      dtype='object')







    













    




shap for channel for lumi
         PF      calo     muons   photons
0 -0.017121 -0.429733  0.127486  0.022872
1 -0.995557 -0.554841 -0.035827 -0.034690
2 -0.339343 -0.442611  0.093597  0.019072
3 -0.335203 -0.576661  0.016841  0.009385
4 -1.758022 -0.674648 -0.085833 -0.065206
5 -0.199650 -0.426586 -0.036304  0.032424
6 -0.067054 -0.528119  0.085096  0.058332
7  0.076540 -0.441331  0.124999  0.004993
 
shap for channel for run
PF        -3.635410
calo      -4.074530
muons      0.290055
photons    0.047183
dtype: float64
 
____________________________________________________________________________________________________

278821  [[1, 33], [36, 37]],		FED652 in error, EE+04 is off

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['qCCPhi5x5_0', 'qPFMetPhi_1', 'qCCPhi5x5_5', 'qCCEta5x5_0',
       'qCCEn5x5_5', 'qCCPhi5x5_4', 'qCalJet1Pt_4', 'qEEiy_0', 'qEEenergy_3',
       'qEEix_4', 'qCalJet3Pt_4', 'qHBHEtime_0', 'qPreShPhi_0', 'qgedPhor9__5',
       'qCalMETPhi_3', 'qCalMETBEFOPhi_3', 'qSCEtaWidth5x5_3', 'qPreShYPhi_1',
       'qMuCosm1Pt_0', 'qEBtime_3'],
      dtype='object')

the most common features with big shap values

Index(['qCCPhi5x5_0', 'qPFMetPhi_1', 'qCCEta5x5_0', 'qCCEn5x5_5',
       'qCCPhi5x5_4', 'qEEiy_0', 'qEEenergy_3', 'qEEix_4', 'qPreShPhi_0',
       'qCCEta5x5_3', 'qCalMETPhi_3', 'qEBtime_3', 'qPFJet8CHSEta_1',
       'qPreShEn_5', 'qSCPhi5x5_3'],
      dtype='object')







    













    




shap for channel for lumi
          PF      calo     muons   photons
0  -1.526824 -0.522516 -0.178109  0.054819
1  -2.034884 -0.684983 -0.065346 -0.200901
2  -2.096270 -0.609091 -0.023937  0.049069
3  -2.229792 -0.543472  0.028532 -0.002406
4  -2.242228 -0.692549 -0.030018 -0.034707
5  -1.889997 -0.605115 -0.046488  0.014733
6  -2.709890 -0.271474 -0.067628 -0.005725
7  -0.759280 -0.464586 -0.125208  0.083225
8  -3.120661 -0.340108  0.037049  0.064131
9  -2.430041 -0.679483 -0.081424  0.029959
10 -1.578063 -0.630096 -0.035183 -0.047202
11 -1.028936 -0.648706  0.163547  0.056610
12 -1.634445 -0.752388 -0.061436 -0.035359
13 -0.156432 -0.549681  0.076858  0.028234
14 -2.180285 -0.516534 -0.074565 -0.003812
15 -0.716921 -0.512866  0.036888  0.038872
16 -2.106353 -0.815535 -0.119088  0.010872
17 -1.009543 -0.444646 -0.051950 -0.025501
18 -0.307258 -0.610163  0.046323  0.046316
19 -1.566120 -0.738035 -0.021772  0.013501
20 -2.242798 -0.436218 -0.048522 -0.030528
21 -1.379151 -0.731144  0.288393  0.026610
22 -0.142495 -0.641713 -0.044012  0.011624
23 -1.801402 -0.495534 -0.049339  0.062194
24 -1.728504 -0.770198 -0.080452  0.079348
25 -1.550710 -0.562874  0.134360  0.047274
26 -2.320824 -0.747829 -0.042314 -0.033111
27 -0.378311 -0.539023  0.158254  0.020369
28 -1.742242 -0.787928  0.174383 -0.011439
29  0.029314 -0.603207  0.091633  0.044104
30 -0.631863 -0.488214 -0.004824 -0.021501
31 -2.188599 -0.667462 -0.028085  0.000791
32 -2.089011 -0.578187 -0.032599 -0.025776
 
shap for channel for run
PF        -51.490820
calo      -19.681560
muons      -0.076079
photons     0.304688
dtype: float64
 
____________________________________________________________________________________________________
279028 is not found

279995  [[1, 8]],			hcal water colling issues

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['qPFJet4CHSEta_3', 'qPFMetPhi_3', 'qgedPhor2x5__1', 'qPFJetEIEta_3',
       'qEStime_1', 'qCCPhi_2', 'qUNdrSumEt_0', 'qPFJet8CHS2Pt_4', 'qNVtx_5',
       'qHFieta_1', 'qCalMETEn_4', 'qPFJet4CHSPhi_1', 'qPFMetPt_5',
       'qEBtime_4', 'qPFMetPhi_1', 'qUNSigmaIPhi_3', 'qCalMETBEPhi_3',
       'qPhor9__5', 'qCalMETBEFOEn_3', 'qPFJet4CHSPhi_5'],
      dtype='object')

the most common features with big shap values

Index(['qPFJet4CHSEta_3', 'qPFMetPhi_3', 'qgedPhor2x5__1', 'qPFJetEIEta_3',
       'qEStime_1', 'qCCPhi_2', 'qUNdrSumEt_0', 'qPFJet8CHS2Pt_4', 'qNVtx_5',
       'qHFieta_1', 'qCalMETEn_4', 'qPFJet4CHSPhi_1', 'qPFMetPt_5',
       'qEBtime_4', 'qPFMetPhi_1', 'qUNSigmaIPhi_3', 'qCalMETBEPhi_3',
       'qPhor9__5', 'qCalMETBEFOEn_3', 'qPFJet4CHSPhi_5', 'qCalMETMPhi_1',
       'qPFJetEta_3', 'qPFJet4CHSEta_5', 'qESenergy_4', 'qMu5Eta_3',
       'qSCPhiWidth5x5_0', 'qNVtx_3', 'qCalMETBEFOPhi_3', 'qPFJet4CHS4Pt_6',
       'qEEenergy_3', 'qCalJet2Pt_5', 'qMuCosmLegEta_5', 'qPFMetPt_4',
       'qCalJetPhi_5', 'qPFJet4CHS4Pt_5', 'qEBtime_3', 'qHFiphi_1',
       'qCalJetEn_0', 'qMu4Phi_3', 'qPFJetPhi_3', 'qPFJetPhi_1',
       'qMuCosmEta_1', 'qHBHEtime_0', 'qPFMetPt_0', 'qPFJetTopCHS1Eta_5',
       'qMuCosmLeg0Phi_2', 'qSigmaIEta_3', 'qCalMETPhi_1', 'qPFJet4CHS4Pt_2',
       'qPFJet8CHSSD3Phi_5'],
      dtype='object')







    













    




shap for channel for lumi
         PF      calo     muons   photons
0 -5.181176 -1.047123 -0.300973 -0.320993
 
shap for channel for run
PF        -5.181176
calo      -1.047123
muons     -0.300973
photons   -0.320993
dtype: float64
 
____________________________________________________________________________________________________
280002 is not found

280006  [[1, 68]],			EB-11 token ring missing

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['qPFMetPhi_1', 'qCalMETBEPhi_1', 'qHFieta_1', 'qCalMETBEPhi_3',
       'qSCPhi_1', 'qEBtime_0', 'qCalMETBEFOPhi_3', 'qPFMetPt_5',
       'qCalJet3Pt_4', 'qEBenergy_5', 'qEBiPhi_1', 'qCCPhi_1', 'qgedPhor9__5',
       'qPreShEn_4', 'qPFMetPhi_3', 'qPFJet4CHSPhi_0', 'qCalJetPt_3',
       'qEBtime_3', 'qPFJet8CHS5Pt_1', 'qCalMETEn_4'],
      dtype='object')

the most common features with big shap values

Index(['qPFMetPhi_1', 'qHFieta_1', 'qSCPhi_1', 'qCalMETBEFOPhi_3',
       'qPFMetPt_5', 'qEBiPhi_1', 'qPFMetPhi_3', 'qCalJetPt_3'],
      dtype='object')







    













    




shap for channel for lumi
          PF      calo     muons   photons
0  -6.058166 -2.462731 -0.100720 -0.100980
1  -5.891794 -1.842641  0.107395  0.017513
2  -3.963906 -1.743998 -0.212764 -0.217888
3  -6.106144 -2.505621 -0.011331 -0.075714
4  -6.036377 -2.532761 -0.129589 -0.078085
5  -5.925825 -2.080910  0.031239 -0.079204
6  -5.841641 -2.513426 -0.007466 -0.163330
7  -6.125124 -2.559343  0.051405 -0.026739
8  -5.961860 -2.526858  0.099739 -0.185145
9  -6.155635 -2.539698 -0.012126 -0.002515
10 -5.903775 -2.483135  0.135610 -0.212210
11 -6.003523 -2.642015 -0.038475 -0.024331
12 -6.110198 -2.638155 -0.116084 -0.135680
13 -6.164195 -2.650187  0.039890 -0.044096
14 -6.212756 -2.549198 -0.164612 -0.031139
15 -4.646801 -2.289896  0.018758 -0.192752
16 -5.945683 -2.565698  0.032617 -0.072839
17 -5.712776 -2.281295 -0.068980  0.083087
18 -1.378419 -2.482317 -0.018051 -0.147915
19 -4.379586 -2.637528 -0.031325 -0.088598
20 -6.133733 -3.323997  0.014786 -0.086857
21 -5.723449 -2.377832 -0.089209 -0.115084
22 -5.752967 -2.437275 -0.038735  0.034098
23 -6.040580 -2.524871  0.109701 -0.131136
24 -5.946011 -2.507767  0.009320 -0.112624
25 -5.770868 -1.797237 -0.100641  0.072798
26 -6.185401 -2.252595  0.080885 -0.128697
27 -6.153526 -2.345725 -0.010886 -0.057864
28 -6.033446 -2.407262 -0.014432 -0.151440
29 -5.563927 -2.466006  0.065518 -0.112111
..       ...       ...       ...       ...
36 -6.022815 -2.240958  0.047664 -0.032351
37 -6.016934 -2.272921  0.101205 -0.031937
38 -5.709172 -1.612400  0.001833  0.034322
39 -6.091192 -2.051072  0.029144 -0.096292
40 -6.152472 -2.426686 -0.040115 -0.108329
41 -5.836416 -2.411977  0.102140 -0.004071
42 -5.493624 -2.415049  0.013545  0.010877
43 -5.970802 -2.219084 -0.032673  0.072284
44 -6.537876 -2.318046  0.028027 -0.110183
45 -5.947123 -2.198954  0.151541 -0.066620
46 -5.602267 -1.099070  0.023257 -0.134957
47 -6.309902 -2.455034 -0.047763 -0.134527
48 -5.883113 -2.683028  0.067364 -0.007730
49 -6.009362 -2.368463  0.107039 -0.015444
50 -5.868992 -2.373975  0.157350  0.048730
51 -5.923150 -2.550535  0.066525 -0.157832
52 -6.339670 -2.699235  0.067198 -0.035114
53 -5.586887 -2.297423 -0.155141 -0.088500
54 -6.482104 -1.738656 -0.012037 -0.224149
55 -5.406219 -2.444089 -0.003676 -0.142088
56 -5.489250 -2.609119  0.078752 -0.126970
57 -5.890080 -2.328828 -0.075569 -0.025390
58 -6.193305 -2.367807 -0.143025 -0.003483
59 -6.055849 -2.426617  0.010714 -0.085715
60 -5.611762 -0.961898 -0.057125 -0.126212
61 -6.170348 -1.791062 -0.049477 -0.092661
62 -5.787853 -1.371624  0.036049  0.019375
63 -5.597144 -2.409341  0.045208 -0.041195
64 -5.910287 -2.394813  0.033069 -0.038832
65 -6.082456 -1.532331 -0.260441 -0.034771

[66 rows x 4 columns]
 
shap for channel for run
PF        -382.951082
calo      -150.666569
muons       -0.411385
photons     -4.822202
dtype: float64
 
____________________________________________________________________________________________________

280007  [[1, 36]],			Low voltage channel broken in EB

this run from TRAIN set
sum of features' influences from all lumis, 20 the most important

Index(['qPFJet4CHSEta_3', 'qPFJetEIEta_3', 'qEBenergy_5', 'qPFMetPt_5',
       'qHFieta_1', 'qPFMetPhi_1', 'qEBtime_0', 'qCalMETBEEn_5',
       'qPhomaxenxtal__5', 'qPFJet8CHSSD0Phi_1', 'qPFJet8CHSEta_4',
       'qCalMETBEFOEn_3', 'qPFJet4CHSEta_4', 'qPFJet8CHS2Phi_1', 'qCCPhi_2',
       'qCalJetPt_3', 'qCalJetEta_0', 'qCalMETEn_4', 'qPFJetTopCHSPt_3',
       'qPFJetTopCHS1Eta_5'],
      dtype='object')

the most common features with big shap values

Index(['qPFJet4CHSEta_3', 'qPFJetEIEta_3', 'qEBenergy_5', 'qPFMetPt_5',
       'qHFieta_1', 'qEBtime_0', 'qCalMETBEEn_5', 'qPFJet8CHSEta_4',
       'qPhomaxenxtal__5', 'qPFJet8CHSSD0Phi_1', 'qCalJetPt_3',
       'qCalMETBEFOEn_3', 'qPFMetPt_0', 'qPFJet4CHSEta_4', 'qPFJetTopCHSPt_3',
       'qPFJetTopCHS1Eta_5', 'qCalJetEta_4', 'qMuCosmLeg2Phi_1', 'qCalMETEn_4',
       'qPhosigmaIeta__3', 'qPFMetPt_4', 'qPhoEn__3', 'qCalMETMEn_0',
       'qEEix_1', 'qPFJet8CHS0Eta_4', 'qgedPhor2x5__0', 'qPFJet8CHSSDEta_4',
       'qEBchi2_0', 'qPhosigmaeta__3'],
      dtype='object')







    













    




shap for channel for lumi
         PF      calo     muons   photons
0 -6.852889 -0.727082 -0.020850 -0.449455
1 -7.386769 -0.578108 -0.025756 -0.404102
2 -6.166199 -0.584850 -0.402144 -0.324461
 
shap for channel for run
PF        -20.405857
calo       -1.890039
muons      -0.448751
photons    -1.178018
dtype: float64
 
____________________________________________________________________________________________________
280239 is not found
280241 is not found
281663 is not found
281674 is not found
281680 is not found
281974 is not found
282408 is not found
282707 is not found
282796 is not found
282921 is not found

In [19]:

    

inds = np.where(np.array(ids_train)==275768)[0]
inds


    

    
Out[19]:





array([582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594,
       595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607,
       608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620,
       621, 622, 623, 624, 625, 626, 627, 628, 629, 630, 631, 632, 633,
       634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, 646,
       647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657])

In [20]:

    

shap.force_plot(shap_values_train[583,:], feature_names[:])


    

    
Out[20]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [21]:

    

shap.force_plot(shap_values_train[617,:], feature_names[:])


    

    
Out[21]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [22]:

    

shap.force_plot(shap_values_train[656,:], feature_names[:])


    

    
Out[22]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [23]:

    

np.where(np.array(ids_train)==279995)


    

    
Out[23]:





(array([108963]), array([0]))

In [24]:

    

shap.force_plot(shap_values_train[108963,:], feature_names[:])


    

    
Out[24]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [25]:

    

inds = np.where(np.array(ids_train)==275764)[0]
inds


    

    
Out[25]:





array([494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506,
       507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519,
       520, 521, 522, 523])

In [26]:

    

shap.force_plot(shap_values_train[500,:], feature_names[:])


    

    
Out[26]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [27]:

    

shap.force_plot(shap_values_train[523,:], feature_names[:])


    

    
Out[27]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [28]:

    

np.where(np.array(ids_train)==275758)


    

    
Out[28]:





(array([476, 477, 478, 479]), array([0, 0, 0, 0]))

In [29]:

    

shap.force_plot(shap_values_train[476,:], feature_names[:])


    

    
Out[29]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [30]:

    

shap.force_plot(shap_values_train[478,:], feature_names[:])


    

    
Out[30]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [31]:

    

np.where(np.array(ids_train)==275757)[0]


    

    
Out[31]:





array([459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471,
       472, 473, 474, 475])

In [32]:

    

shap.force_plot(shap_values_train[459,:], feature_names)


    

    
Out[32]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [33]:

    

shap.force_plot(shap_values_train[460,:], feature_names)


    

    
Out[33]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [34]:

    

shap.force_plot(shap_values_train[461,:], feature_names)


    

    
Out[34]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [35]:

    

shap.force_plot(shap_values_train[475,:], feature_names)


    

    
Out[35]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [36]:

    

np.where(np.array(ids_train)==280007)


    

    
Out[36]:





(array([109030, 109031, 109032]), array([0, 0, 0]))

In [37]:

    

shap.force_plot(shap_values_train[109030,:], feature_names=feature_names)


    

    
Out[37]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [38]:

    

shap.force_plot(shap_values_train[109031,:], feature_names=feature_names)


    

    
Out[38]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [39]:

    

shap.force_plot(shap_values_train[109032,:], feature_names=feature_names)


    

    
Out[39]:







  Visualization omitted, Javascript library not loaded!

  Have you run `initjs()` in this notebook? If this notebook was from another
  user you must also trust this notebook (File -> Trust notebook). If you are viewing
  this notebook on github the Javascript has been stripped for security.

 

In [ ]: