In [1]:
import pandas as pd
import numpy as np
import pickle
import xlwt
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.cross_validation import cross_val_score
from sklearn.cross_validation import train_test_split
from sklearn.metrics import confusion_matrix
In [2]:
df_file = pd.read_csv('../data/df_dropSub_less20_dropNaResult.csv',delimiter=",", skip_blank_lines = True,
error_bad_lines=False)
df_file = df_file.drop('Unnamed: 0',axis=1)
df_file = df_file.fillna(0)
df_file = df_file.replace(['A', 'B+', 'B', 'C+', 'C' , 'D+' , 'D' , 'F' , 'W' , 'S' , 'S#' , 'U' , 'U#'],
[8, 7, 7, 6 , 6, 5, 5, 4, 3, 2, 2, 1, 1])
In [3]:
df_file
Out[3]:
In [4]:
count_courseId = df_file["3COURSEID"].value_counts()
more20 = count_courseId
headers=list(df_file.columns.values)
subjects = []
countSub = 0
In [5]:
count = 0
subjects.sort()
precision_rf={}
df_precision = more20.drop('CS231').copy()
list_allsub = df_file.columns[4:]
allSubject_df = pd.DataFrame(columns=[subjects],index=[list_allsub])
top10_df = pd.DataFrame(columns=[subjects])
In [6]:
headers=list(df_file.columns.values)
subjects = []
countSub = 0
#Create dictionary of list subjects
for sub in df_file[headers[1]]:
if sub not in subjects:
subjects.append(sub)
countSub = countSub+1
In [7]:
subjects.sort()
In [8]:
subjects.remove('CS231')
In [9]:
len(subjects)
Out[9]:
In [10]:
subject = 'CS401'
df_sub = df_file[df_file['3COURSEID'] == subject]
df_sub = df_sub.iloc[np.random.permutation(len(df_sub))]
count_enrollment = df_sub['3COURSEID'].value_counts()
#print "Number of %s enrollment: %s"%(subject,count_enrollment)
A = df_sub.as_matrix()
X = A[:,6:116]
X = X.astype(np.int64, copy=False)
y = A[:,2]
y = y.astype(np.int64, copy=False)
In [11]:
subject
Out[11]:
In [12]:
X
Out[12]:
In [13]:
X.shape
Out[13]:
In [14]:
y.shape
Out[14]:
In [15]:
# Split the data into a training set and a test set
X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=0.2,random_state=0)
In [16]:
X_train
Out[16]:
In [17]:
X.shape
Out[17]:
In [18]:
X_test
Out[18]:
In [19]:
X_test.shape
Out[19]:
In [20]:
X_train.shape
Out[20]:
In [21]:
y_train
Out[21]:
In [22]:
y_train.shape
Out[22]:
In [23]:
y_test
Out[23]:
In [24]:
y_test.shape
Out[24]:
In [25]:
forest = RandomForestClassifier(n_estimators=10, max_depth=None,
min_samples_split=1, random_state=None, max_features=None)
clf = forest.fit(X_train, y_train)
In [26]:
y_pred = clf.predict(X_test)
In [27]:
y_pred.shape
Out[27]:
In [28]:
# Compute confusion matrix
cm = confusion_matrix(y_test, y_pred)
print('Confusion matrix, without normalization')
print(cm)
plt.figure()
Out[28]:
In [29]:
y_pred
Out[29]:
In [30]:
y_test
Out[30]:
In [31]:
Grade = ['A', 'B', 'C' , 'D' , 'F' , 'W' , 'S' , 'U' ,'na']
row = []
for cls in y_train:
if cls not in row:
row.append(cls)
row.sort()
print row
row_cm = []
for i in xrange(len(row)):
Grade = ['A', 'B', 'C' , 'D' , 'F' , 'W' , 'S' , 'U' ,'na']
grade = Grade[::-1][row[i]]
print grade
row_cm.append(grade)
print row_cm
In [32]:
row_cm
Out[32]:
In [55]:
def plot_confusion_matrix(cm, title=subject, cmap=plt.cm.Blues):
for y in range(cm.shape[0]):
for x in range(cm.shape[1]):
plt.text(x, y, '%.1f' % cm[y, x])
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(row_cm))
plt.xticks(tick_marks, row_cm, rotation=45)
plt.yticks(tick_marks, row_cm)
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.pcolor(cm, cmap=cmap)
plt.yticks(np.arange(len(row_cm)),row_cm,label="#Training Data")
plt.xticks(np.arange(len(row_cm)),row_cm,label='#Class')
plt.colorbar()
In [56]:
# Compute confusion matrix
cm = confusion_matrix(y_test, y_pred)
np.set_printoptions(precision=2)
print('Confusion matrix')
print(row_cm)
print(cm)
plt.figure()
plot_confusion_matrix(cm)
In [57]:
cm
Out[57]:
In [58]:
plt.show()
In [46]:
for subject in subjects:
#Create new Dataframe
#print subject
df_sub = df_file[df_file['3COURSEID'] == subject]
df_sub = df_sub.iloc[np.random.permutation(len(df_sub))]
count_enrollment = df_sub['3COURSEID'].value_counts()
#print "Number of %s enrollment: %s"%(subject,count_enrollment)
A = df_sub.as_matrix()
X = A[:,6:117]
X = X.astype(np.int64, copy=False)
y = A[:,2]
y = y.astype(np.int64, copy=False)
# Split the data into a training set and a test set
X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=0.2,random_state=0)
forest = RandomForestClassifier(n_estimators=10, max_depth=None,
min_samples_split=1, random_state=None, max_features=None)
clf = forest.fit(X_train, y_train)
y_pred = clf.predict(X_test)
cm = confusion_matrix(y_test, y_pred)
print('Confusion matrix of %s'%subject)
print(cm)
plt.figure()
#plot_confusion_matrix(cm)
In [1]:
from sklearn.metrics import mean_squared_error
In [234]:
import pandas as pd
import numpy as np
from collections import defaultdict
df_file = pd.read_csv('../data/CS_table_No2_No4_new.csv',delimiter=";", skip_blank_lines = True,
error_bad_lines=False)
In [235]:
for subject in subjects:
df_sub = df_file[df_file['3COURSEID'] == subject]
df_sub = df_sub.iloc[np.random.permutation(len(df_sub))]
count_enrollment = df_sub['3COURSEID'].value_counts()
#print "Number of %s enrollment: %s"%(subject,count_enrollment)
A = df_sub.as_matrix()
X = A[:,6:116]
X = X.astype(np.int64, copy=False)
y = A[:,2]
y = y.astype(np.int64, copy=False)
#Training data
forest = RandomForestClassifier(n_estimators=10, max_depth=None,
min_samples_split=1, random_state=None, max_features=None)
clf = forest.fit(X, y)
f = "tree_drop/tree%s.pic"%subject
with open(f, 'wb') as pickleFile:
pickle.dump(clf, pickleFile, pickle.HIGHEST_PROTOCOL)
Out[235]:
In [ ]: