Get your data here. The data is related with direct marketing campaigns of a Portuguese banking institution. The marketing campaigns were based on phone calls. Often, more than one contact to the same client was required, in order to access if the product (bank term deposit) would be ('yes') or not ('no') subscribed. There are four datasets:
1) bank-additional-full.csv with all examples (41188) and 20 inputs, ordered by date (from May 2008 to November 2010)
2) bank-additional.csv with 10% of the examples (4119), randomly selected from 1), and 20 inputs.
3) bank-full.csv with all examples and 17 inputs, ordered by date (older version of this dataset with less inputs).
4) bank.csv with 10% of the examples and 17 inputs, randomly selected from 3 (older version of this dataset with less inputs).
The smallest datasets are provided to test more computationally demanding machine learning algorithms (e.g., SVM).
The classification goal is to predict if the client will subscribe (yes/no) a term deposit (variable y).
LabelEncoder useful)
In [170]:
# Standard imports for data analysis packages in Python
import pandas as pd
import numpy as np
import seaborn as sns # for pretty layout of plots
import matplotlib.pyplot as plt
from pprint import pprint # for pretty printing
from sklearn.grid_search import GridSearchCV
from sklearn.metrics import confusion_matrix, classification_report
from sklearn import preprocessing
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.cross_validation import train_test_split
from sklearn.learning_curve import learning_curve
from sklearn.grid_search import GridSearchCV
# This enables inline Plots
%matplotlib inline
In [171]:
# read data
data = pd.read_csv('bank-additional/bank-additional-full.csv', sep=';')
In [172]:
# use labelencode to transform categorical rows with small unique values
le = preprocessing.LabelEncoder()
le.fit(data.default)
data.default = le.transform(data.default)
le.fit(data.marital)
data.marital = le.transform(data.marital)
le.fit(data.job)
data.job = le.transform(data.job)
le.fit(data.education)
data.education = le.transform(data.education)
le.fit(data.housing)
data.housing = le.transform(data.housing)
le.fit(data.loan)
data.loan = le.transform(data.loan)
le.fit(data.contact)
data.contact = le.transform(data.contact)
le.fit(data.month)
data.month = le.transform(data.month)
le.fit(data.day_of_week)
data.day_of_week = le.transform(data.day_of_week)
le.fit(data.poutcome)
data.poutcome = le.transform(data.poutcome)
le.fit(data.y)
data.y = le.transform(data.y)
In [173]:
# assign X and y. Split data
y = data.y
X = data.drop('y', axis=1)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
In [168]:
# Instantiate the estimator using KNN
clf = KNeighborsClassifier(n_neighbors=9, leaf_size=5)
# Fit the estimator to the Training Data
clf.fit(X_train, y_train)
Out[168]:
In [125]:
#plot learning curve for KNN with kd_tree algorithm
%time
_ = plot_learning_curve(KNeighborsClassifier(),'test', X_train, y_train)
In [80]:
#plot learning curve for KNN with brute algorithm
_ = plot_learning_curve(KNeighborsClassifier(algorithm='brute'),'test', X_train, y_train)
In [126]:
#plot learning curve for KNN with ball_tree algorithm
_ = plot_learning_curve(KNeighborsClassifier(algorithm='ball_tree'),'test', X_train, y_train)
In [119]:
# Find and verify best parameters for KNN using grid search
# Since the different algorithms have little to no effect on the learning curve,
# We vary the grid search by using n_neighbors and leaf_size
d = {'n_neighbors': range(1, 10)}
gds = GridSearchCV(KNeighborsClassifier(), d)
In [120]:
gds.fit(X_train, y_train)
gds.best_params_
Out[120]:
In [121]:
e = {'leaf_size' : [1, 3, 5, 10, 15, 30, 60, 120]}
gds = GridSearchCV(KNeighborsClassifier(), e)
In [122]:
gds.fit(X_train, y_train)
gds.best_params_
Out[122]:
In [116]:
# Classification report for KNN
y_pred = clf.predict(X_test)
print classification_report(y_test, y_pred)
In [131]:
In [ ]:
# Repeat steps above for random forest
In [174]:
rf = RandomForestClassifier(n_estimators=90, max_depth=10)
rf.fit(X_train, y_train)
Out[174]:
In [137]:
# Plot learning curve for random forest classifier
%time
_ = plot_learning_curve(RandomForestClassifier(n_estimators=100),'test',X_train, y_train)
In [95]:
# Find the best parameters for random forest
# We will vary the n_estimators and max_depth parameters
In [132]:
d = {'n_estimators' : [2, 4, 8, 16, 25, 30, 60, 90, 100]}
gds = GridSearchCV(RandomForestClassifier(), d)
In [133]:
gds.fit(X_train, y_train)
gds.best_params_
Out[133]:
In [134]:
e = {'max_depth' : [2, 4, 5, 10, 20, 40, 60, 90, 100]}
gds = GridSearchCV(RandomForestClassifier(), e)
In [135]:
gds.fit(X_train, y_train)
gds.best_params_
Out[135]:
In [138]:
# plot learning curve again using the best params
%time
_ = plot_learning_curve(RandomForestClassifier(n_estimators=90, max_depth=10),'test',X_train, y_train)
In [175]:
# Classification report for random forest
y_pred = rf.predict(X_test)
print classification_report(y_test, y_pred)
In [179]:
#Find the most important features
sorted(zip(rf.feature_importances_, data.drop('y', axis=1).columns), reverse=True)[:20]
Out[179]:
In [144]:
# It looks like the top 3 features account for the majority (58%) of importance.
# Rather than using all 20 features, perhaps we can us the top 5 important features
# To build the model with better precision
In [180]:
new_data = data[['duration', 'euribor3m', 'nr.employed', 'pdays', 'cons.conf.idx']]
In [181]:
X_train, X_test, y_train, y_test = train_test_split(new_data, y, test_size=0.3)
In [182]:
rf = RandomForestClassifier(n_estimators=90, max_depth=10)
rf.fit(X_train, y_train)
Out[182]:
In [183]:
# Classification report for random forest
y_pred = rf.predict(X_test)
print classification_report(y_test, y_pred)
In [ ]:
# It looks like the precision for classifying 0 increase by classifying 1 decreased.
# Perhaps using only 5 features doesn't have help increasing the precision after all
In [97]:
#plot learning curve helper function
def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
n_jobs=1, train_sizes=np.linspace(.1, 1.0, 5)):
"""
Generate a simple plot of the test and traning learning curve.
Parameters
----------
estimator : object type that implements the "fit" and "predict" methods
An object of that type which is cloned for each validation.
title : string
Title for the chart.
X : array-like, shape (n_samples, n_features)
Training vector, where n_samples is the number of samples and
n_features is the number of features.
y : array-like, shape (n_samples) or (n_samples, n_features), optional
Target relative to X for classification or regression;
None for unsupervised learning.
ylim : tuple, shape (ymin, ymax), optional
Defines minimum and maximum yvalues plotted.
cv : integer, cross-validation generator, optional
If an integer is passed, it is the number of folds (defaults to 3).
Specific cross-validation objects can be passed, see
sklearn.cross_validation module for the list of possible objects
n_jobs : integer, optional
Number of jobs to run in parallel (default 1).
"""
plt.figure()
plt.title(title)
if ylim is not None:
plt.ylim(*ylim)
plt.xlabel("Training examples")
plt.ylabel("Score")
train_sizes, train_scores, test_scores = learning_curve(
estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
test_scores_mean = np.mean(test_scores, axis=1)
test_scores_std = np.std(test_scores, axis=1)
plt.grid()
plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
train_scores_mean + train_scores_std, alpha=0.1,
color="r")
plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
test_scores_mean + test_scores_std, alpha=0.1, color="g")
plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
label="Training score")
plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
label="Cross-validation score")
plt.legend(loc="best")
return plt
In [158]:
In [ ]: