In [26]:
import re
import numpy as np
import scipy as sp
import pandas as pd
import statsmodels.api as sm
import matplotlib as mpl
import matplotlib.tri as mtri
import matplotlib.pylab as plt
import matplotlib.patches as mpatches
import seaborn as sns
import itertools
from scipy import stats
from sklearn.datasets import make_regression
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.metrics import classification_report
from sklearn import preprocessing
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_curve
# Seaborn setting
sns.set(palette="hls", font_scale=2)
In [6]:
df = pd.read_csv('frames/lc_dataframe.csv')
print list(df.columns)
df.tail()
Out[6]:
In [3]:
var_desc = pd.read_csv('frames/var_description(csv).csv')
var_desc
Out[3]:
In [56]:
df.dtypes
Out[56]:
In [55]:
df.isnull().sum()
Out[55]:
In [257]:
fig = plt.figure(figsize = (15, 10))
corrmat = df.corr()
sns.heatmap(corrmat)
plt.title("Correlation between Features")
plt.show()
In [307]:
corr_stack = corrmat.abs().unstack()
ordered_stack = corr_stack.order(kind="quicksort", ascending=False)
order_ix = []
for num in range(len(ordered_stack)):
if ordered_stack[num] > 0.6 and ordered_stack[num] < 1.0:
order_ix.append(num)
print ordered_stack[min(order_ix):max(order_ix)]
In [7]:
del df["grade"]
del df["sub_grade"]
del df["installment"]
del df["open_acc"]
In [8]:
category_vars = ["emp_title", "emp_length", "home_ownership",
"verification_status", "issue_d", "purpose",
"initial_list_status", "pymnt_plan", "application_type"]
continuous_vars = ["loan_amnt", "int_rate", "annual_inc",
"desc", "dti", "revol_bal", "revol_util", "total_acc",
"pub_rec", "inq_last_6mths", "delinq_2yrs"]
In [310]:
for var in category_vars:
print df.groupby(var).loan_status.value_counts()
In [9]:
del df["pymnt_plan"]
del df["application_type"]
In [10]:
category_vars.remove("pymnt_plan")
category_vars.remove("application_type")
In [11]:
df["home_ownership"].replace([1, 3], 4, inplace=True)
df["home_ownership"].replace(2, 1, inplace=True)
df["home_ownership"].replace(4, 2, inplace=True)
df["home_ownership"].replace(5, 3, inplace=True)
df["home_ownership"].replace(6, 4, inplace=True)
In [327]:
np.unique(df["home_ownership"])
Out[327]:
| Home_ownership Category | Category number |
|---|---|
| Mortgage | 1 |
| Other | 2 |
| Own | 3 |
| Rent | 4 |
In [12]:
df["purpose"].replace([4, 11], 10, inplace=True)
df["purpose"].replace(13, 4, inplace=True)
df["purpose"].replace(14, 11, inplace=True)
In [326]:
np.unique(df["purpose"])
Out[326]:
| Purpose Category | Category number |
|---|---|
| car | 1 |
| credit_card | 2 |
| debt_consolidation | 3 |
| vacation | 4 |
| home_improvement | 5 |
| house | 6 |
| major_purchase | 7 |
| medical | 8 |
| moving | 9 |
| other | 10 |
| wedding | 11 |
| small_business | 12 |
In [332]:
print len(category_vars), len(continuous_vars)
In [377]:
N=4; M=2; # set row and column of the figure
fig = plt.figure(figsize=(10,15)) # figure size
plt.subplots_adjust(top=1, bottom=0, hspace=0, wspace=0) # subplot setting
# make subplot with face data index
for i in range(N):
for j in range(M):
if i * M + j < 7:
ax = fig.add_subplot(N, M, i * M + j + 1)
plt.hist(df[category_vars[i * M + j]], bins=30)
plt.title(category_vars[i * M + j])
else:
pass
plt.tight_layout()
plt.suptitle("Histograms for category variables", y=1.02,fontsize=30)
plt.show()
In [381]:
N=4; M=3; # set row and column of the figure
fig = plt.figure(figsize=(15,15)) # figure size
plt.subplots_adjust(top=1, bottom=0, hspace=0, wspace=0) # subplot setting
# make subplot with face data index
for i in range(N):
for j in range(M):
if i * M + j < 11:
ax = fig.add_subplot(N, M, i * M + j + 1)
plt.hist(df[continuous_vars[i * M + j]], bins=10)
plt.title(continuous_vars[i * M + j])
else:
pass
plt.tight_layout()
plt.suptitle("Histograms for continuous variables", y=1.02,fontsize=30)
plt.show()
In [420]:
N=4; M=3; # set row and column of the figure
fig = plt.figure(figsize=(15,15)) # figure size
plt.subplots_adjust(top=1, bottom=0, hspace=0, wspace=0) # subplot setting
# make subplot with face data index
for i in range(N):
for j in range(M):
if i * M + j < 11:
ax = fig.add_subplot(N, M, i * M + j + 1)
sp.stats.probplot(df[continuous_vars[i * M + j]], plot=plt)
plt.title(continuous_vars[i * M + j])
else:
pass
plt.tight_layout()
plt.suptitle("QQ-Plots for continuous variables", y=1.02,fontsize=30)
plt.show()
In [13]:
outliers = int(len(df) * 0.00005)
print "Number of 0.005%: ", outliers
out_ix = np.array([])
for name in continuous_vars:
out_large_list = np.array(df[name].nlargest(outliers).index)
out_small_list = np.array(df[name].nsmallest(outliers).index)
out_ix = np.concatenate((out_ix, out_large_list, out_small_list))
out_ix = np.unique(out_ix)
print "Number of outliers(0.005% * 2): ", out_ix.shape[0]
In [14]:
cleaned_df = df.drop(df.index[list(out_ix)])
cleaned_df.index = range(len(cleaned_df))
cleaned_df.tail()
Out[14]:
In [422]:
N=4; M=3; # set row and column of the figure
fig = plt.figure(figsize=(15,15)) # figure size
plt.subplots_adjust(top=1, bottom=0, hspace=0, wspace=0) # subplot setting
# make subplot with face data index
for i in range(N):
for j in range(M):
if i * M + j < 11:
ax = fig.add_subplot(N, M, i * M + j + 1)
sp.stats.probplot(cleaned_df[continuous_vars[i * M + j]], plot=plt)
plt.title(continuous_vars[i * M + j])
else:
pass
plt.tight_layout()
plt.suptitle("QQ-Plots for continuous variables after removing outliers", y=1.02,fontsize=30)
plt.show()
In [15]:
log_vars = ["int_rate", "annual_inc", "loan_amnt"]
scaling_vars = ["desc", "dti", "revol_bal", "revol_util", "total_acc", "pub_rec", "inq_last_6mths", "delinq_2yrs"]
In [401]:
for i in log_vars:
print cleaned_df[cleaned_df[i].isin([int(0)])].index
In [16]:
cleaned_df[log_vars] = np.log10(cleaned_df[log_vars])
In [17]:
cleaned_df[scaling_vars] = preprocessing.scale(cleaned_df[scaling_vars])
In [419]:
N=4; M=3; # set row and column of the figure
fig = plt.figure(figsize=(15,15)) # figure size
plt.subplots_adjust(top=1, bottom=0, hspace=0, wspace=0) # subplot setting
# make subplot with face data index
for i in range(N):
for j in range(M):
if i * M + j < 11:
ax = fig.add_subplot(N, M, i * M + j + 1)
sp.stats.probplot(cleaned_df[continuous_vars[i * M + j]], plot=plt)
plt.title(continuous_vars[i * M + j])
else:
pass
plt.tight_layout()
plt.suptitle("QQ-Plots for continuous variables after Scaling and Logarithm", y=1.02,fontsize=30)
plt.show()
In [426]:
N=4; M=3; # set row and column of the figure
fig = plt.figure(figsize=(15,15)) # figure size
plt.subplots_adjust(top=1, bottom=0, hspace=0, wspace=0) # subplot setting
# make subplot with face data index
for i in range(N):
for j in range(M):
if i * M + j < 11:
ax = fig.add_subplot(N, M, i * M + j + 1)
plt.hist(cleaned_df[continuous_vars[i * M + j]], bins=10)
plt.title(continuous_vars[i * M + j])
else:
pass
plt.tight_layout()
plt.suptitle("Histograms for continuous variables after Scaling and Logarithm", y=1.02,fontsize=30)
plt.show()
In [18]:
scaled_df = cleaned_df
In [20]:
def under_sampling(df_name):
global X, y, under_X, under_y, under_df
X = df_name.ix[:, df_name.columns != "loan_status"]
y = df_name.ix[:, df_name.columns == "loan_status"]
default_ix = np.array(df_name[df_name.loan_status == 0].index)
paid_ix = np.array(df_name[df_name.loan_status == 1].index)
rand_paid_ix = np.random.choice(paid_ix, len(default_ix), replace=False)
rand_ix = np.concatenate([default_ix, rand_paid_ix])
print "X Shape: ", X.shape
print "y Shape: ", y.shape
print "Default Index Length: ", len(default_ix)
print "Paid Index Length: ", len(rand_paid_ix)
print "Random Index Shape: ", rand_ix.shape
print "-" * 75
under_df = df_name.iloc[rand_ix, :]
under_X = under_df.ix[:, under_df.columns != "loan_status"]
under_y = under_df.ix[:, under_df.columns == "loan_status"]
print "Under DataFrame's X and y", len(under_X), len(under_y)
print "Data Variables Names: under_df, under_X, under_y"
In [21]:
under_sampling(scaled_df)
In [22]:
under_train_X, under_test_X, under_train_y, under_test_y = train_test_split(
under_X, under_y, test_size=0.20, random_state=0)
train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.25, random_state=0)
In [18]:
rf = RandomForestClassifier(max_features=None, random_state=0)
rf_result = rf.fit(under_train_X, under_train_y)
In [23]:
def plot_confusion_matrix(cm, classes,
normalize=False,
title='Confusion matrix',
cmap=plt.cm.Oranges):
"""
This function prints and plots the confusion matrix.
Normalization can be applied by setting `normalize=True`.
"""
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
thresh = cm.max() / 2.
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j, i, cm[i, j],
horizontalalignment="center",
color="white" if cm[i, j] > thresh else "black")
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
In [20]:
# Compute confusion matrix
cnf_matrix = confusion_matrix(under_test_y, rf_result.predict(under_test_X))
np.set_printoptions(precision=2)
# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=[0,1],
title='Cofusion Matrix(predict with under sampled test set)')
plt.show()
print classification_report(under_test_y, rf_result.predict(under_test_X))
In [21]:
# Compute confusion matrix
cnf_matrix = confusion_matrix(y, rf_result.predict(X))
np.set_printoptions(precision=2)
# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=[0,1],
title='Cofusion Matrix(predict with whole test set)')
plt.show()
print classification_report(y, rf_result.predict(X))
In [24]:
cv = StratifiedKFold(n_splits=8, random_state=0, shuffle=False)
sgd = SGDClassifier(loss="log", fit_intercept=True,
average=1000, n_iter=30, n_jobs=8,
random_state=0)
lda = LinearDiscriminantAnalysis()
lr = LogisticRegression(n_jobs=8)
In [113]:
dummies_df = scaled_df[continuous_vars]
for var in category_vars:
dummies_df = dummies_df.join(pd.get_dummies(scaled_df[var], prefix=var))
print dummies_df.shape
print dummies_df.columns
dummies_df = dummies_df.join(scaled_df["loan_status"])
dummies_df.tail()
Out[113]:
In [109]:
under_sampling(dummies_df)
In [124]:
print "SGD CV Accuracy for under sampled dataset"
sgd_cv_under = cross_val_score(sgd, under_X,
np.array(under_y).reshape(len(under_y, )),
cv = cv, scoring="accuracy", n_jobs=8)
print "Each Validation's Accuracy: ", sgd_cv_under
print "Average Accuracy: ", np.mean(sgd_cv_under)
print '-' * 75
print "SGD CV Accuracy for whole dataset"
sgd_cv_whole = cross_val_score(sgd, X,
np.array(y).reshape(len(y, )),
cv = cv, scoring="accuracy", n_jobs=8)
print "Each Validation's Accuracy: ", sgd_cv_whole
print "Average Accuracy: ", np.mean(sgd_cv_whole)
In [80]:
print "LDA CV Accuracy for under sampled dataset"
lda_cv_under = cross_val_score(lda, under_X,
np.array(under_y).reshape(len(under_y, )),
cv = cv, scoring="accuracy", n_jobs=8)
print "Each Validation's Accuracy: ", lda_cv_under
print "Average Accuracy: ", np.mean(lda_cv_under)
print '-' * 75
print "LDA CV Accuracy for whole dataset"
lda_cv_whole = cross_val_score(lda, X,
np.array(y).reshape(len(y, )),
cv = cv, scoring="accuracy", n_jobs=8)
print "Each Validation's Accuracy: ", lda_cv_whole
print "Average Accuracy: ", np.mean(lda_cv_whole)
In [125]:
print "LogisticRegression CV Accuracy for under sampled dataset"
lr_cv_under = cross_val_score(lr, under_X,
np.array(under_y).reshape(len(under_y, )),
cv = cv, scoring="accuracy", n_jobs=8)
print "Each Validation's Accuracy: ", lr_cv_under
print "Average Accuracy: ", np.mean(lr_cv_under)
print '-' * 75
print "LogisticRegression CV Accuracy for whole dataset"
lr_cv_whole = cross_val_score(lr, X,
np.array(y).reshape(len(y, )),
cv = cv, scoring="accuracy", n_jobs=8)
print "Each Validation's Accuracy: ", lr_cv_whole
print "Average Accuracy: ", np.mean(lr_cv_whole)
In [27]:
lr_result = lr.fit(under_train_X, under_train_y)
sgd_result = sgd.fit(under_train_X, under_train_y)
lda_result = lda.fit(under_train_X, under_train_y)
fpr0, tpr0, thresholds0 = roc_curve(y, sgd_result.decision_function(X), pos_label=1)
fpr1, tpr1, thresholds1 = roc_curve(y, lr_result.decision_function(X), pos_label=1)
fpr2, tpr2, thresholds2 = roc_curve(y, lda_result.decision_function(X), pos_label=1)
plt.plot(fpr0, tpr0, "r-", label="LogisticRegression")
plt.plot(fpr1, tpr1, "g-", label="SGD")
plt.plot(fpr2, tpr2, "y-", label="LDA")
plt.plot([0, 1], [0, 1], 'k--', label="random guess")
plt.xlim(-0.05, 1.0)
plt.ylim(0, 1.05)
plt.xlabel('False Positive Rate (Fall-Out)')
plt.ylabel('True Positive Rate (Recall)')
plt.title('Receiver operating characteristic Curve')
plt.legend(loc="lower right")
plt.show()
In [469]:
importances = rf_result.feature_importances_
indices = np.argsort(importances)[::-1]
for f in range(X.shape[1]):
print("{}. {} / feature_n: {} / importance: {}".format(f+1, list(X.columns)[f], indices[f], importances[indices[f]]))
std = np.std([tree.feature_importances_ for tree in rf_result.estimators_],
axis=0)
plt.figure()
plt.title("Feature importances")
plt.bar(range(X.shape[1]), importances[indices],
yerr=std[indices], align="center")
plt.xticks(range(X.shape[1]), indices)
plt.xlim([-1, X.shape[1]])
plt.show()
In [167]:
model = sm.Logit.from_formula("loan_status ~ loan_amnt + int_rate + C(emp_title) + C(emp_length) + annual_inc + C(verification_status) + C(home_ownership)",
data=under_df)
result = model.fit()
print result.pred_table()
result.summary()
Out[167]:
In [184]:
iter_df = scaled_df.drop("emp_length", 1)
under_sampling(iter_df)
In [185]:
under_train_X, under_test_X, under_train_y, under_test_y = train_test_split(
under_X, under_y, test_size=0.20, random_state=0)
train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.25, random_state=0)
In [159]:
def rf_fit():
rf = RandomForestClassifier(max_features=None, random_state=0)
rf_result = rf.fit(under_train_X, under_train_y)
importances = rf_result.feature_importances_
indices = np.argsort(importances)[::-1]
for f in range(X.shape[1]):
print("{}. {} / feature_n: {} / importance: {}".format(f+1, list(X.columns)[f], indices[f], importances[indices[f]]))
std = np.std([tree.feature_importances_ for tree in rf_result.estimators_],
axis=0)
plt.figure()
plt.title("Feature importances")
plt.bar(range(X.shape[1]), importances[indices],
yerr=std[indices], align="center")
plt.xticks(range(X.shape[1]), indices)
plt.xlim([-1, X.shape[1]])
plt.show()
In [187]:
rf_fit()
In [164]:
model = sm.Logit.from_formula("loan_status ~ loan_amnt + int_rate + C(emp_title) + C(issue_d) + annual_inc + C(verification_status) + C(home_ownership)",
data=under_df)
result = model.fit()
print result.pred_table()
result.summary()
Out[164]:
In [188]:
iter_df["issue_d"] = iter_df["issue_d"].replace([1, 9, 12], 1)
iter_df["issue_d"] = iter_df["issue_d"].replace([2, 3, 4, 5, 6, 7, 8, 10, 11], 0)
print np.unique(iter_df["issue_d"])
under_sampling(iter_df)
In [189]:
under_train_X, under_test_X, under_train_y, under_test_y = train_test_split(
under_X, under_y, test_size=0.20, random_state=0)
train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.25, random_state=0)
In [190]:
rf_fit()
In [176]:
model = sm.Logit.from_formula("loan_status ~ loan_amnt + int_rate + C(emp_title) + C(issue_d) + annual_inc + C(verification_status) + C(home_ownership)",
data=under_df)
result = model.fit()
print result.pred_table()
result.summary()
Out[176]:
In [191]:
fitted_cols = ["loan_status", "emp_title", "issue_d", "verification_status",
"home_ownership", "loan_amnt", "int_rate", "annual_inc"]
fitted_df = iter_df.reindex(columns=fitted_cols)
fitted_df.tail()
Out[191]:
In [192]:
under_sampling(fitted_df)
In [196]:
under_train_X, under_test_X, under_train_y, under_test_y = train_test_split(
under_X, under_y, test_size=0.20, random_state=0)
train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.25, random_state=0)
In [194]:
rf_fit()
In [200]:
rf = RandomForestClassifier(max_features=None, random_state=0)
rf_result = rf.fit(under_train_X, under_train_y)
In [201]:
# Compute confusion matrix
cnf_matrix = confusion_matrix(under_test_y, rf_result.predict(under_test_X))
np.set_printoptions(precision=2)
# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=[0,1],
title='Cofusion Matrix(predict with under sampled test set)')
plt.show()
print classification_report(under_test_y, rf_result.predict(under_test_X))
In [202]:
# Compute confusion matrix
cnf_matrix = confusion_matrix(y, rf_result.predict(X))
np.set_printoptions(precision=2)
# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=[0,1],
title='Cofusion Matrix(predict with whole test set)')
plt.show()
print classification_report(y, rf_result.predict(X))