In [1]:
# Import libraries
import numpy as np
import pandas as pd
In [2]:
# Read student data
student_data = pd.read_csv("student-data.csv")
print "Student data read successfully!"
# Note: The last column 'passed' is the target/label, all other are feature columns
Now, can you find out the following facts about the dataset?
Use the code block below to compute these values. Instructions/steps are marked using TODOs.
In [3]:
# TODO: Compute desired values - replace each '?' with an appropriate expression/function call
shape = student_data.shape
n_students = shape[0]
n_features = shape[1]-1 # the last column is the target
n_passed = len(student_data[student_data.passed == 'yes'])
n_failed = len(student_data[student_data.passed == 'no'])
grad_rate = 100*float(n_passed)/n_students
print "Total number of students: {}".format(n_students)
print "Number of students who passed: {}".format(n_passed)
print "Number of students who failed: {}".format(n_failed)
print "Number of features: {}".format(n_features)
print "Graduation rate of the class: {:.2f}%".format(grad_rate)
In this section, we will prepare the data for modeling, training and testing.
It is often the case that the data you obtain contains non-numeric features. This can be a problem, as most machine learning algorithms expect numeric data to perform computations with.
Let's first separate our data into feature and target columns, and see if any features are non-numeric.
Note: For this dataset, the last column ('passed') is the target or label we are trying to predict.
In [4]:
# Extract feature (X) and target (y) columns
feature_cols = list(student_data.columns[:-1]) # all columns but last are features
target_col = student_data.columns[-1] # last column is the target/label
print "Feature column(s):-\n{}".format(feature_cols)
print "Target column: {}".format(target_col)
X_all = student_data[feature_cols] # feature values for all students
y_all = student_data[target_col] # corresponding targets/labels
print "\nFeature values:-"
print X_all.head() # print the first 5 rows
As you can see, there are several non-numeric columns that need to be converted! Many of them are simply yes/no, e.g. internet. These can be reasonably converted into 1/0 (binary) values.
Other columns, like Mjob and Fjob, have more than two values, and are known as categorical variables. The recommended way to handle such a column is to create as many columns as possible values (e.g. Fjob_teacher, Fjob_other, Fjob_services, etc.), and assign a 1 to one of them and 0 to all others.
These generated columns are sometimes called dummy variables, and we will use the pandas.get_dummies() function to perform this transformation.
In [5]:
# Preprocess feature columns
def preprocess_features(X):
outX = pd.DataFrame(index=X.index) # output dataframe, initially empty
# Check each column
for col, col_data in X.iteritems():
# If data type is non-numeric, try to replace all yes/no values with 1/0
if col_data.dtype == object:
col_data = col_data.replace(['yes', 'no'], [1, 0])
# Note: This should change the data type for yes/no columns to int
# If still non-numeric, convert to one or more dummy variables
if col_data.dtype == object:
col_data = pd.get_dummies(col_data, prefix=col) # e.g. 'school' => 'school_GP', 'school_MS'
outX = outX.join(col_data) # collect column(s) in output dataframe
return outX
X_all = preprocess_features(X_all)
print "Processed feature columns ({}):-\n{}".format(len(X_all.columns), list(X_all.columns))
In [6]:
# First, decide how many training vs test samples you want
num_all = student_data.shape[0] # same as len(student_data)
num_train = 300 # about 75% of the data
num_test = num_all - num_train
# TODO: Then, select features (X) and corresponding labels (y) for the training and test sets
# Note: Shuffle the data or randomly select samples to avoid any bias due to ordering in the dataset
indices = range(num_all)
import random
random.shuffle(indices)
train_indices = indices[:num_train]
test_indices = indices[-num_test:]
X_train = X_all.iloc[train_indices]
y_train = y_all[train_indices]
X_test = X_all.iloc[test_indices]
y_test = y_all[test_indices]
print "Training set: {} samples".format(X_train.shape[0])
print "Test set: {} samples".format(X_test.shape[0])
# Note: If you need a validation set, extract it from within training data
Choose 3 supervised learning models that are available in scikit-learn, and appropriate for this problem. For each model:
Produce a table showing training time, prediction time, F1 score on training set and F1 score on test set, for each training set size.
Note: You need to produce 3 such tables - one for each model.
In [7]:
# Train a model
import time
def train_classifier(clf, X_train, y_train):
print "Training {}...".format(clf.__class__.__name__)
start = time.time()
clf = clf.fit(X_train, y_train)
end = time.time()
print "Done!\nTraining time (secs): {:.3f}".format(end - start)
# TODO: Choose a model, import it and instantiate an object
from sklearn.tree import DecisionTreeClassifier
clf = DecisionTreeClassifier(max_depth=3)
# Fit model to training data
train_classifier(clf, X_train, y_train) # note: using entire training set here
print clf # you can inspect the learned model by printing it
In [8]:
# Predict on training set and compute F1 score
from sklearn.metrics import f1_score
def predict_labels(clf, features, target):
print "Predicting labels using {}...".format(clf.__class__.__name__)
start = time.time()
y_pred = clf.predict(features)
end = time.time()
print "Done!\nPrediction time (secs): {:.3f}".format(end - start)
return f1_score(target.values, y_pred, pos_label='yes')
train_f1_score = predict_labels(clf, X_train, y_train)
print "F1 score for training set: {}".format(train_f1_score)
In [9]:
# Predict on test data
print "F1 score for test set: {}".format(predict_labels(clf, X_test, y_test))
In [10]:
# Train and predict using different training set sizes
def train_predict(clf, X_train, y_train, X_test, y_test):
print "------------------------------------------"
print "Training set size: {}".format(len(X_train))
train_classifier(clf, X_train, y_train)
print "F1 score for training set: {}".format(predict_labels(clf, X_train, y_train))
print "F1 score for test set: {}".format(predict_labels(clf, X_test, y_test))
num_all = student_data.shape[0] # same as len(student_data)
num_test = 95
test_indices = indices[-num_test:]
X_test = X_all.iloc[test_indices]
y_test = y_all[test_indices]
indices = range(num_all)
import random
random.shuffle(indices)
def try_different_training_sizes(clf):
# TODO: Run the helper function above for desired subsets of training data
# Note: Keep the test set constant
for size in (100, 200, 300):
train_indices = indices[:size]
X_train = X_all.iloc[train_indices]
y_train = y_all[train_indices]
train_predict(clf, X_train, y_train, X_test, y_test)
In [11]:
# TODO: Train and predict using two other models
from sklearn.ensemble import BaggingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.cross_validation import cross_val_score
from sklearn.metrics import make_scorer, f1_score
svc = SVC(random_state=0)
bc = BaggingClassifier()
# Using cross validation to figure out the max depth for the Decision Tree Classifier
# Reference: http://blog.kaggle.com/2015/06/29/scikit-learn-video-7-optimizing-your-model-with-cross-validation/
all_scores = []
for md in range(1,11):
clf = DecisionTreeClassifier(max_depth=md, random_state=0)
scores = cross_val_score(clf, X_all, y_all, scoring=make_scorer(f1_score, pos_label='yes'))
all_scores.append((md, scores.mean(),))
print all_scores
# The maximum f1 score occurs for max_depth 1, so using it!
dtc = DecisionTreeClassifier(max_depth=1,random_state=0)
try_different_training_sizes(dtc)
try_different_training_sizes(bc)
try_different_training_sizes(svc)
| Training Size | Training Time | Prediction Time | Training F1 Score | Testing F1 Score |
|---|---|---|---|---|
| 100 | 0.000 | 0.000 | 0.82 | 0.82 |
| 200 | 0.000 | 0.000 | 0.84 | 0.82 |
| 300 | 0.001 | 0.000 | 0.81 | 0.82 |
| Training Size | Training Time | Prediction Time | Training F1 Score | Testing F1 Score |
|---|---|---|---|---|
| 100 | 0.002 | 0.001 | 0.882 | 0.800 |
| 200 | 0.005 | 0.001 | 0.824 | 0.810 |
| 300 | 0.008 | 0.002 | 0.832 | 0.800 |
| Training Size | Training Time | Prediction Time | Training F1 Score | Testing F1 Score |
|---|---|---|---|---|
| 100 | 0.014 | 0.001 | 0.97 | 0.7438 |
| 200 | 0.013 | 0.001 | 0.98 | 0.857 |
| 300 | 0.015 | 0.002 | 0.99 | 0.894 |
In [14]:
# TODO: Fine-tune your model and report the best F1 score
from sklearn.grid_search import GridSearchCV
from sklearn.metrics import make_scorer, f1_score
dt_parameters = {'max_depth':(3,2,1),}
clf = DecisionTreeClassifier(random_state=0)
classifier = GridSearchCV(clf,dt_parameters, scoring=make_scorer(f1_score, pos_label='yes'))
# Fit the learner to the training data to obtain the best parameter set
print "Final Model: "
train_classifier(classifier, X_train, y_train)
# pick the best classifier
classifier = classifier.best_estimator_
print "Best model: ", classifier
print predict_labels(classifier, X_test, y_test)
In [ ]:
In [ ]: