In [ ]:
# importing pandas package with alias pd
import pandas as pd
In [ ]:
#create a data frame - dictionary is used here where keys get converted to column names and values to row values.
data = pd.DataFrame({'Country': ['Russia','Colombia','Chile','Equador','Nigeria'],
'Rank':[121,40,100,130,11]})
data
In [ ]:
# describe() method computes summary statistics of integer / double variables
data.describe()
In [ ]:
# info() gives a more detailed statistics about data in dataframe
data.info()
In [ ]:
data = pd.DataFrame({'group':['a', 'a', 'a', 'b','b', 'b', 'c', 'c','c'],'ounces':[4, 3, 12, 6, 7.5, 8, 3, 5, 6]})
# head(n) gives first n rows of dataframe
data.head(3)
In [ ]:
#Let's sort the data frame by ounces - inplace = True will make changes to the data
data.sort_values(by=['ounces'],ascending=True,inplace=False)
In [ ]:
# Sorting on multiple columns
data.sort_values(by=['group','ounces'],ascending=[True,False],inplace=False)
Often, we get data sets with duplicate rows, which is nothing but noise. Therefore, before training the model, we need to make sure we get rid of such inconsistencies in the data set. Let's see how we can remove duplicate rows.
In [ ]:
data = pd.DataFrame({'k1':['one']*3 + ['two']*4, 'k2':[3,2,1,3,3,4,4]})
data.sort_values(by='k2')
In [ ]:
#remove duplicates
data.drop_duplicates()
Here, we removed duplicates based on matching row values across all columns. Alternatively, we can also remove duplicates based on a particular column. Let's remove duplicate values from the k1 column
In [ ]:
data.drop_duplicates(subset='k1')
Lets see how we can add a new column to our DataFrame
In [ ]:
import numpy as np
data = pd.DataFrame({'food': ['bacon', 'pulled pork', 'bacon', 'Pastrami','corned beef', 'Bacon', 'pastrami', 'honey ham','nova lox'],
'ounces': [4, 3, 12, 6, 7.5, 8, 3, 5, 6]})
# Creates a new column protien and assigns 9 random values to it
data = data.assign(protien = np.random.random(9))
data
In [ ]:
# Lets remove the added columns
data.drop('protien',axis='columns',inplace=True)
data
Lets see how we can use slice on DataFrames
In [ ]:
dates = pd.date_range('20130101',periods=6)
df = pd.DataFrame(np.random.randn(6,4),index=dates,columns=list('ABCD'))
df
In [ ]:
#get first n rows from the data frame
df[:3]
In [ ]:
#slice based on date range
df['20130101':'20130104']
In [ ]:
#slicing based on column names
df.loc[:,['A','B']]
In [ ]:
df.loc['20130102':'20130103',['A','B']]
In [ ]:
#slicing based on index of columns
df.iloc[3] #returns 4th row (index is 3rd)
In [ ]:
#returns a specific range of rows
df.iloc[2:4, 0:2] # Selects rows from 2:4 and columns from 0:2
In [ ]:
# Comapring
df['B'] > 1
In [ ]:
# Boolean indexing based on column values as well. This helps in filtering a data set based on a pre-defined condition
df[df['B'] > 1]
We can also use a query method to select columns based on a criterion
In [ ]:
#list all columns where A is greater than C
df.query('A > C')
In [ ]:
#using OR condition
df.query('A < B | C > A')
In [ ]:
# Reading the csv file using pandas into a DataFrame
iris = pd.read_csv('Iris.csv')
iris.head(10)
In [ ]:
iris.describe()
In [ ]:
iris.info()
In [ ]:
iris['Species'].value_counts()
In [ ]:
# We can plot things is using the .plot extension from Pandas dataframes
# We'll use this to make a scatterplot of the Iris features.
%matplotlib inline
iris.plot(kind="scatter", x="SepalLengthCm", y="SepalWidthCm")
In [ ]:
# Importing some visualization libraries
import warnings
warnings.filterwarnings("ignore")
import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
sns.set(style="white", color_codes=True)
In [ ]:
# One piece of information missing in the plots above is what species each plant is
# We'll use seaborn's FacetGrid to color the scatterplot by species
sns.FacetGrid(iris, hue="Species", size=5) \
.map(plt.scatter, "SepalLengthCm", "SepalWidthCm") \
.add_legend()
In [ ]:
# We can look at an individual feature in Seaborn through a boxplot
sns.boxplot(x="Species", y="PetalLengthCm", data=iris)
In [ ]:
# Another useful seaborn plot is the pairplot, which shows the bivariate relation
# between each pair of features
#
# From the pairplot, we'll see that the Iris-setosa species is separataed from the other
# two across all feature combinations
sns.pairplot(iris.drop("Id", axis=1), hue="Species", size=3)
In [ ]:
# Box plots can also be made using DataFrame
iris.drop("Id", axis=1).boxplot(by="Species",figsize=(12, 12))
Since our Species column which contains these three class labels contain categorical data, the first thing to do would be to encode them numerically as follows:
In [ ]:
target_map = {'Iris-setosa':0, 'Iris-versicolor':1,'Iris-virginica':2 }
# Use the pandas apply method to numerically encode our attrition target variable
iris['Species'] = iris['Species'].apply(lambda x: target_map[x])
iris
In [ ]:
# importing alll the necessary packages to use the various classification algorithms
from sklearn.linear_model import LogisticRegression # for Logistic Regression algorithm
from sklearn.cross_validation import train_test_split #to split the dataset for training and testing
from sklearn import metrics #for checking the model accuracy
from sklearn.tree import DecisionTreeClassifier #for using Decision Tree Algoithm
from sklearn.ensemble import RandomForestClassifier # A combine model of many decision trees
When we train any algorithm, the number of features and their correlation plays an important role. If there are features and many of the features are highly correlated, then training an algorithm with all the featues will reduce the accuracy. Thus features selection should be done carefully
In [ ]:
X = iris[['SepalLengthCm', 'SepalWidthCm', 'PetalLengthCm', 'PetalWidthCm']]
Y = iris['Species']
In [ ]:
plt.figure(figsize=(10,8))
sns.heatmap(X.corr(),annot=True,cmap='cubehelix_r') #draws heatmap with input as the correlation matrix calculted by(iris.corr())
plt.show()
The Sepal Width and Length are not correlated The Petal Width and Length are highly correlated
We will use all the features for training the algorithm and check the accuracy.
Then we will use 1 Petal Feature and 1 Sepal Feature to check the accuracy of the algorithm as we are using only 2 features that are not correlated. Thus we can have a variance in the dataset which may help in better accuracy. We will check it later
Split the dataset into training and testing dataset. The testing dataset is generally smaller than training one as it will help in training the model better.
Select any algorithm based on the problem (classification or regression) whatever you feel may be good.
Then pass the training dataset to the algorithm to train it.
Then pass the testing data to the trained algorithm to predict the outcome.
We then check the accuracy by passing the predicted outcome and the actual output to the model.
In [ ]:
train, test = train_test_split(iris, test_size = 0.3, random_state=1212)# in this our main data is split into train and test
# the attribute test_size=0.3 splits the data into 70% and 30% ratio. train=70% and test=30%
print(train.shape)
print(test.shape)
In [ ]:
train_X = train[['SepalLengthCm','SepalWidthCm','PetalLengthCm','PetalWidthCm']]# taking the training data features
train_y = train.Species# output of our training data
test_X = test[['SepalLengthCm','SepalWidthCm','PetalLengthCm','PetalWidthCm']] # taking test data features
test_y = test.Species #output value of test data
In [ ]:
model = LogisticRegression()
model.fit(train_X,train_y)
prediction = model.predict(test_X)
print('The accuracy of the Logistic Regression is',metrics.accuracy_score(prediction,test_y))
In [ ]:
model = DecisionTreeClassifier()
model.fit(train_X,train_y)
prediction = model.predict(test_X)
print('The accuracy of the Decision Tree is',metrics.accuracy_score(prediction,test_y))
We used all the features of iris in above models. Now we will use Petals and Sepals Seperately
In [ ]:
petal=iris[['PetalLengthCm','PetalWidthCm','Species']]
sepal=iris[['SepalLengthCm','SepalWidthCm','Species']]
train_p,test_p=train_test_split(petal,test_size=0.3,random_state=0) #petals
train_x_p=train_p[['PetalWidthCm','PetalLengthCm']]
train_y_p=train_p.Species
test_x_p=test_p[['PetalWidthCm','PetalLengthCm']]
test_y_p=test_p.Species
train_s,test_s=train_test_split(sepal,test_size=0.3,random_state=0) #Sepal
train_x_s=train_s[['SepalWidthCm','SepalLengthCm']]
train_y_s=train_s.Species
test_x_s=test_s[['SepalWidthCm','SepalLengthCm']]
test_y_s=test_s.Species
In [ ]:
model = LogisticRegression()
model.fit(train_x_p,train_y_p)
prediction=model.predict(test_x_p)
print('The accuracy of the Logistic Regression using Petals is:',metrics.accuracy_score(prediction,test_y_p))
model.fit(train_x_s,train_y_s)
prediction=model.predict(test_x_s)
print('The accuracy of the Logistic Regression using Sepals is:',metrics.accuracy_score(prediction,test_y_s))
In [ ]:
model=DecisionTreeClassifier()
model.fit(train_x_p,train_y_p)
prediction=model.predict(test_x_p)
print('The accuracy of the Decision Tree using Petals is:',metrics.accuracy_score(prediction,test_y_p))
model.fit(train_x_s,train_y_s)
prediction=model.predict(test_x_s)
print('The accuracy of the Decision Tree using Sepals is:',metrics.accuracy_score(prediction,test_y_s))
In [ ]:
from sklearn.preprocessing import StandardScaler
import plotly.graph_objs as go
import plotly.offline as py
py.init_notebook_mode(connected=True)
import plotly.graph_objs as go
from plotly import tools
X = iris.iloc[:, :2] # Take only the first two features.
y = iris.Species
h = .02 # step size in the mesh
X = StandardScaler().fit_transform(X)
# Implement 3 Logistic Regression models with varying values of C
clf = LogisticRegression(C=0.01)
clf.fit(X, y)
clf2 = LogisticRegression(C=1)
clf2.fit(X, y)
clf3 = LogisticRegression(C=100)
clf3.fit(X, y)
# Define our usual decision surface bounding plots
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h)
, np.arange(y_min, y_max, h))
y_ = np.arange(y_min, y_max, h)
In [ ]:
Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
trace1 = go.Heatmap(x=xx[0], y=y_, z=Z,
colorscale='Viridis',
showscale=True)
trace2 = go.Scatter(x=X[:, 0], y=X[:, 1],
mode='markers',
showlegend=False,
marker=dict(size=10,
color=y,
colorscale='Viridis',
line=dict(color='black', width=1))
)
layout= go.Layout(
autosize= True,
title= 'Logistic Regression (C=0.01)',
hovermode= 'closest',
showlegend= False)
data = [trace1, trace2]
fig = go.Figure(data=data, layout=layout)
py.iplot(fig)
Z = clf2.predict(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
#Z = Z.reshape((xx.shape[0], xx.shape[1], 3))
trace3 = go.Heatmap(x=xx[0], y=y_,
z=Z,
colorscale='Viridis',
showscale=True)
trace4 = go.Scatter(x=X[:, 0], y=X[:, 1],
mode='markers',
showlegend=False,
marker=dict(size=10,
color=y,
colorscale='Viridis',
line=dict(color='black', width=1))
)
layout= go.Layout(
autosize= True,
title= 'Logistic Regression (C=1)',
hovermode= 'closest',
showlegend= False)
data = [trace3, trace4]
fig2 = go.Figure(data=data,layout= layout)
for i in map(str, range(1, 3)):
x = 'xaxis' + i
y = 'yaxis' + i
fig['layout'][x].update(showgrid=False, zeroline=False,
showticklabels=False, ticks='', autorange=True)
fig['layout'][y].update(showgrid=False, zeroline=False,
showticklabels=False, ticks='', autorange=True)
py.iplot(fig2)
del X, y # remove the earlier X and y
X = iris.iloc[:, :2] # Take only the first two features.
y = iris.Species
h = .02 # step size in the mesh
X = StandardScaler().fit_transform(X)
Z = clf3.predict(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
trace5 = go.Heatmap(x=xx[0], y=y_,
z=Z,
colorscale='Viridis',
showscale=True)
trace6 = go.Scatter(x=X[:, 0], y=X[:, 1],
mode='markers',
showlegend=False,
marker=dict(size=10,
color=y,
colorscale='Viridis',
line=dict(color='black', width=1))
)
layout= go.Layout(
autosize= True,
title= 'Logistic Regression (C=100)',
hovermode= 'closest',
showlegend= False)
data = [trace5, trace6]
fig3 = go.Figure(data=data,layout= layout)
py.iplot(fig3)
In [ ]: