In [1]:
import numpy as np
import pandas as pd
import os
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import LabelEncoder
from scipy.stats import norm
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from scipy import stats
import warnings
warnings.filterwarnings('ignore')
In [2]:
DATAFILE = os.path.join('data','data.csv')
TARGETFILE = os.path.join('data','target.csv')
OUTDIR = os.path.join('results')
In [3]:
#DA AGGIUNGERE GTARGET E TDI
train = pd.read_table(DATAFILE,sep=',')
target = pd.read_table(TARGETFILE,sep=',')
dataset = pd.concat([train,target],axis=1)
In [4]:
#Original dataset size
original_size = np.shape(dataset)[0]
#Delete rows associated with "extreme" values: Y=200 || Y = -200
dataset = dataset[dataset.Y!=-200]
dataset = dataset[dataset.Y!=200]
#New dataset size and analysis of lost data
new_size = np.shape(dataset)[0]
print('Records lost: ',(1.0-new_size/original_size)*100, '%')
#Description of the dataset
dataset.describe()
Out[4]:
In [5]:
dataset['Y'].describe()
Out[5]:
In [6]:
sns.distplot(dataset['Y'])
plt.show()
In [10]:
#correlation matrix
corrmat = dataset.corr()
sns.heatmap(corrmat, vmax=.8, square=True)
plt.show()
In [11]:
#Target correlation matrix
k = 7 #number of variables for heatmap
cols = corrmat.nlargest(k, 'Y')['Y'].index
cm = np.corrcoef(dataset[cols].values.T)
sns.set(font_scale=1.25)
hm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)
plt.show()
In [54]:
#scatterplot
sns.set()
sns.pairplot(dataset, size = 2.5)
plt.show();
In [7]:
dataset_trans = dataset.copy()
dataset_trans.to_csv('dataset_trans.csv',sep=',',index=False)
In [61]:
#GC
var = 'GT'
sns.distplot(dataset[var])
plt.show()
res = stats.probplot(dataset[var], plot=plt)
plt.show()
#log-transformation and standard scaling
scaler = MinMaxScaler().fit(dataset_trans[var].reshape(-1,1))
dataset_trans[var] = scaler.transform(dataset_trans[var].reshape(-1,1))
sns.distplot(dataset_trans[var])
plt.show()
In [63]:
#min_risk
var = 'min_risk'
sns.distplot(dataset[var])
plt.show()
res = stats.probplot(dataset[var], plot=plt)
plt.show()
#log-transformation and standard scaling
dataset_trans[var] = np.log(dataset[var])
scaler = StandardScaler().fit(dataset_trans[var].reshape(-1,1))
dataset_trans[var] = scaler.transform(dataset_trans[var].reshape(-1,1))
sns.distplot(dataset_trans[var])
plt.show()
res = stats.probplot(dataset_trans[var], plot=plt)
plt.show()
In [64]:
#CR
var = 'CR'
sns.distplot(dataset[var])
plt.show()
res = stats.probplot(dataset[var], plot=plt)
plt.show()
#log-transformation and standard scaling
dataset_trans[var] = np.log(dataset[var])
scaler = StandardScaler().fit(dataset_trans[var].reshape(-1,1))
dataset_trans[var] = scaler.transform(dataset_trans[var].reshape(-1,1))
sns.distplot(dataset_trans[var])
plt.show()
res = stats.probplot(dataset_trans[var], plot=plt)
plt.show()
In [65]:
#CF
var = 'CF'
sns.distplot(dataset[var])
plt.show()
res = stats.probplot(dataset[var], plot=plt)
plt.show()
#log-transformation and standard scaling
dataset_trans[var] = np.log(dataset[var])
scaler = StandardScaler().fit(dataset_trans[var].reshape(-1,1))
dataset_trans[var] = scaler.transform(dataset_trans[var].reshape(-1,1))
sns.distplot(dataset_trans[var])
plt.show()
res = stats.probplot(dataset_trans[var], plot=plt)
plt.show()
In [66]:
#u2ss
var = 'u2ss'
sns.distplot(dataset[var])
plt.show()
res = stats.probplot(dataset[var], plot=plt)
plt.show()
#log-transformation and standard scaling
dataset_trans[var] = np.log(dataset[var])
scaler = StandardScaler().fit(dataset_trans[var].reshape(-1,1))
dataset_trans[var] = scaler.transform(dataset_trans[var].reshape(-1,1))
sns.distplot(dataset_trans[var])
plt.show()
res = stats.probplot(dataset_trans[var], plot=plt)
plt.show()
In [67]:
#Vmx
var = 'Vmx'
sns.distplot(dataset[var])
plt.show()
res = stats.probplot(dataset[var], plot=plt)
plt.show()
#log-transformation and standard scaling
dataset_trans[var] = np.log(dataset[var])
scaler = StandardScaler().fit(dataset_trans[var].reshape(-1,1))
dataset_trans[var] = scaler.transform(dataset_trans[var].reshape(-1,1))
sns.distplot(dataset_trans[var])
plt.show()
res = stats.probplot(dataset_trans[var], plot=plt)
plt.show()
In [68]:
#kp3
var = 'kp3'
sns.distplot(dataset[var])
plt.show()
res = stats.probplot(dataset[var], plot=plt)
plt.show()
#log-transformation and standard scaling
dataset_trans[var] = np.log(dataset[var])
scaler = StandardScaler().fit(dataset_trans[var].reshape(-1,1))
dataset_trans[var] = scaler.transform(dataset_trans[var].reshape(-1,1))
sns.distplot(dataset_trans[var])
plt.show()
res = stats.probplot(dataset_trans[var], plot=plt)
plt.show()
In [18]:
#Gt
var = 'Gt'
sns.distplot(dataset[var])
plt.show()
res = stats.probplot(dataset[var], plot=plt)
plt.show()
#log-transformation and standard scaling
dataset_trans[var] = np.log(dataset[var])
scaler = StandardScaler().fit(dataset_trans[var].reshape(-1,1))
dataset_trans[var] = scaler.transform(dataset_trans[var].reshape(-1,1))
sns.distplot(dataset_trans[var])
plt.show()
res = stats.probplot(dataset_trans[var], plot=plt)
plt.show()
In [69]:
#Ib
var = 'Ib'
sns.distplot(dataset[var])
plt.show()
res = stats.probplot(dataset[var], plot=plt)
plt.show()
#log-transformation and standard scaling
dataset_trans[var] = np.log(dataset[var])
scaler = StandardScaler().fit(dataset_trans[var].reshape(-1,1))
dataset_trans[var] = scaler.transform(dataset_trans[var].reshape(-1,1))
sns.distplot(dataset_trans[var])
plt.show()
res = stats.probplot(dataset_trans[var], plot=plt)
plt.show()
In [70]:
#BW
var = 'BW'
sns.distplot(dataset[var])
plt.show()
res = stats.probplot(dataset[var], plot=plt)
plt.show()
#log-transformation and standard scaling
dataset_trans[var] = np.log(dataset[var])
scaler = StandardScaler().fit(dataset_trans[var].reshape(-1,1))
dataset_trans[var] = scaler.transform(dataset_trans[var].reshape(-1,1))
sns.distplot(dataset_trans[var])
plt.show()
res = stats.probplot(dataset_trans[var], plot=plt)
plt.show()
In [71]:
#ROC
var = 'ROC'
enc = LabelEncoder()
a = enc.fit_transform(np.asarray(dataset_trans[var]))
n = np.shape(enc.classes_)[0]
b = np.zeros([np.shape(a)[0],n])
for i in np.arange(np.shape(a)[0]):
b[i,a[i]] = 1
columns = ["" for x in range(n)]
for i in np.arange(n):
columns[i] = var + str(enc.classes_[i])
d = pd.DataFrame(data=b,columns=columns)
del dataset_trans[var]
dataset_trans = pd.concat([d, dataset_trans],axis=1)
In [72]:
#var_class
var = 'var_class'
enc = LabelEncoder()
a = enc.fit_transform(np.asarray(dataset_trans[var]))
n = np.shape(enc.classes_)[0]
b = np.zeros([np.shape(a)[0],n])
for i in np.arange(np.shape(a)[0]):
b[i,a[i]] = 1
columns = ["" for x in range(n)]
for i in np.arange(n):
columns[i] = var + str(enc.classes_[i])
d = pd.DataFrame(data=b,columns=columns)
del dataset_trans[var]
dataset_trans = pd.concat([d, dataset_trans],axis=1)
In [ ]:
#scatterplot
sns.set()
sns.pairplot(dataset_trans, size = 2.5)
plt.show();
In [ ]:
#correlation matrix
corrmat = dataset_trans.corr()
sns.heatmap(corrmat, vmax=.8, square=True)
plt.show()
In [ ]:
#Target correlation matrix
k = 10 #number of variables for heatmap
cols = corrmat.nlargest(k, 'Y')['Y'].index
cm = np.corrcoef(dataset_trans[cols].values.T)
sns.set(font_scale=1.25)
hm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)
plt.show()
In [30]:
dataset_trans
Out[30]:
In [31]:
dataset_trans.to_csv('dataset_trans.csv',sep=',',index=False)
In [ ]: