In [41]:
import pandas as pd
import numpy as np
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split
from sklearn.metrics import accuracy_score
# visulaize the important characteristics of the dataset
import matplotlib.pyplot as plt
In [16]:
dataframe_all = pd.read_csv("https://d396qusza40orc.cloudfront.net/predmachlearn/pml-training.csv")
num_rows = dataframe_all.shape[0]
print('No. of rows:', num_rows)
In [42]:
dataframe_all.head()
Out[42]:
Each row represents a different person and each column is on of many physical measurments lke the position of their arm, or forearm and each person gets one of 5 labels (classes) like sitting, standing, jumping, running and jogging.
In [43]:
#List all fators from our response variable
dataframe_all.classe.unique()
Out[43]:
In [44]:
# count the number of missing elements (NaN) in each column
counter_nan = dataframe_all.isnull().sum()
counter_without_nan = counter_nan[counter_nan==0]
print('Columns without Nan:', counter_without_nan )
# remove the columns with missing elements
dataframe_all = dataframe_all[counter_without_nan.keys()]
# remove the first 7 columns which contain no discriminative information
dataframe_all = dataframe_all.iloc[:,7:]
# the list of columns (the last column is the class label)
columns = dataframe_all.columns
print (columns)
In [45]:
# get x and convert it to numpy array
x = dataframe_all.iloc[:,:-1].values
standard_scaler = StandardScaler()
x_std = standard_scaler.fit_transform(x)
In [46]:
# get class label data
y = dataframe_all.iloc[:,-1].values
# encode the class label
class_labels = np.unique(y)
label_encoder = LabelEncoder()
y = label_encoder.fit_transform(y)
In [47]:
test_percentage = 0.3
x_train, x_test, y_train, y_test = train_test_split(x_std, y, test_size = test_percentage, random_state = 0)
In [48]:
from sklearn.manifold import TSNE
tsne = TSNE(n_components=2, random_state=0)
x_test_2d = tsne.fit_transform(x_test)
In [52]:
markers=('s', 'd', 'o', '^', 'v')
color_map = {0:'red', 1:'blue', 2:'lightgreen', 3:'purple', 4:'cyan'}
plt.figure()
plt.figure(figsize=(10,10))
for idx, cl in enumerate(np.unique(y_test)):
plt.scatter(x=x_test_2d[y_test==cl,0], y=x_test_2d[y_test==cl,1], c=color_map[idx], marker=markers[idx], label=cl)
plt.xlabel('X in t-SNE')
plt.ylabel('Y in t-SNE')
plt.legend(loc='upper right')
plt.title('t-SNE visualization of test data')
plt.show()