In [1]:
import sys
sys.path.append('/usr/lib/python2.7/dist-packages')
import numpy as np
import cv2
import urllib
from sklearn import cross_validation
import pandas as pd
from matplotlib import pyplot as plt
In [2]:
# Data is the Crowdflower.com Dress Patterns data set.
# https://www.crowdflower.com/wp-content/uploads/2016/07/dress_patterns.csv
# load dataset and one-hot encode output classes.
imgdf = pd.read_csv('dress_patterns.csv', header=0)
categories = pd.get_dummies(imgdf[['category']]).as_matrix()
In [48]:
#import glob
#filenames = glob.glob('/home/nick/Documents/LewisUniversity/MachineLearning/Project/images/*.png')
In [49]:
#fn2=[]
#for g in filenames:
# fn2.append(g[-40:])
In [50]:
#imgdf.image_url[3][-40:] in fn2
Out[50]:
In [52]:
#for u in imgdf.image_url:
# if u[-40:] in fn2:
# continue
# else:
# #url = urllib.urlopen(u)
# #resource = url.read()
# #outfile = open("/home/nick/Documents/LewisUniversity/MachineLearning/Project/" + u[-40:],"wb")
# #outfile.write(resource)
# #outfile.close()
# f = open('/home/nick/Documents/LewisUniversity/MachineLearning/Project/' + u[-40:] + '.txt', 'w')
# f.write('file Not found')
# f.close()
In [60]:
##imgdf.image_url[15701]
#filelist = pd.DataFrame(filenames)
In [5]:
#print X.shape
#print categories.shape
#print A.shape
#f = open('/home/nick/Documents/LewisUniversity/MachineLearning/Project/status/statuses.txt', 'a')
#f.write('hello'+'world'+'\n')
#f.close()
In [70]:
#dl = imgdf.image_url
#for u in fn2:
# if
#dl[1][-40:]
Out[70]:
In [75]:
#for locs in imgdf.image_url[0:2]:
# fn = "/home/nick/Documents/LewisUniversity/MachineLearning/Project/images/" + locs[-40:]
# url = urllib.urlopen(fn)
# resource = url.read()
# imgarr = np.asarray(bytearray(resource), dtype=np.uint8)
# img = cv2.imdecode(imgarr,-1)
# cv2.imshow("images", img)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
In [3]:
redness = [0,0,255]
bound = np.array(redness, dtype = "uint8")
In [183]:
X = np.empty(shape=[1]+[3]+[50]+[50], dtype='float32')
A = np.empty(shape=[0]+[3]+[50]+[50], dtype='float32')
for locs in imgdf.image_url:
try:
fn = "/home/nick/Documents/LewisUniversity/MachineLearning/Project/images/" + locs[-40:]
url = urllib.urlopen(fn)
resource = url.read()
imgarr = np.asarray(bytearray(resource), dtype=np.uint8)
img = cv2.imdecode(imgarr,-1)
#save the file locally too
#outfile = open(locs[-40:],"wb")
#outfile.write(resource)
#outfile.close()
# find the colors within the specified boundaries and apply
mask = cv2.inRange(img, bound, bound)
output = cv2.bitwise_and(img, img, mask = mask)
# find the corners using goodFeaturesToTrack
gray = cv2.cvtColor(output,cv2.COLOR_BGR2GRAY)
corners = cv2.goodFeaturesToTrack(gray,8,0.01,10)
corners = np.int0(corners)
# Get the min and max corners for our rectangle definition.
x1 = None
x2 = None
y1 = None
y2 = None
for i in corners:
a = i[0][0]
b = i[0][1]
if x1 == None:
x1 = a
x2 = a
if y1 == None:
y1 = b
y2 = b
if x1 != None and a < x1:
x1 = a
if x2 != None and a > x2:
x2 = a
if y1 != None and b < y1:
y1 = b
if y2 != None and b > y2:
y2 = b
#crop the image to a square based on the middle of rectangle.
# This is so we can have consistent shaped data across all observations.
if (y2-y1) < (x2-x1):
ymin = y1
ymax = y2
xmin = x1+(((x2-x1)-(y2-y1))/2)
xmax = x2-(((x2-x1)-(y2-y1))/2)
else:
ymin = y1+(((y2-y1)-(x2-x1))/2)
ymax = y2-(((y2-y1)-(x2-x1))/2)
xmin = x1
xmax = x2
# show computed image range and display image
#print ymin, ymax, xmin, xmax, ymax-ymin, xmax-xmin
crop_img = img[ymin:ymax, xmin:xmax] # Crop image
# resize image to scaled 50 by 50
resized_img = cv2.resize(crop_img, (50, 50))
#hsv_img = cv2.cvtColor(resized_img, cv2.COLOR_RGB2HSV)
# Had messed with using HSV color format, but didn't match with Keras examples. Back to RGB.
X_temp = resized_img/255.
# re-arrange array to be a set of 3,50,50 instead of 50,50,3 for the RGB images.
for i in range(3):
for j in range(50):
for k in range(50):
X[0,i,j,k] = X_temp[j,k,i]
#X = X_temp[:,:,:].flatten()
#Append the record to the array.
A = np.vstack([A, X])
#Save a status so we can see how we are doing
f = open('/home/nick/Documents/LewisUniversity/MachineLearning/Project/status/statuses.txt', 'a')
f.write(locs[-40:]+'\n')
f.close()
except:
#print "Bad file. ", fn
f = open('/home/nick/Documents/LewisUniversity/MachineLearning/Project/status/badimages.txt', 'a')
f.write(locs+'\n')
f.close()
#cv2.imshow("images", img)
#cv2.waitKey(0)
#cv2.destroyAllWindows()
In [184]:
A.shape
Out[184]:
In [185]:
# Because of a few bad images (grayscale, red rectangle on only 2 sides, etc.), need to remove these from the label set y.
badimages = pd.read_csv('/home/nick/Documents/LewisUniversity/MachineLearning/Project/status/badimages.txt', header=None)
#categories = pd.get_dummies(imgdf[['category']]).as_matrix()
#pd.get_dummies(imgdf[['category']])
cleanup = imgdf['image_url'].isin( badimages[0])
categories = pd.get_dummies(imgdf.loc[~cleanup].category).as_matrix()
In [8]:
#imgdf.loc[~cleanup]
#imgdf.category [~badimages]
#badimages
In [48]:
#imgdf.loc[~cleanup].category.shape
#categories[5]
#predicted[5]
#imgdf[['category']]
#categories
#pd.get_dummies(imgdf.loc[~cleanup].category)
#categories.idxmax(1)
#pd.DataFrame(categories).idxmax(1)
#categories.shape
#y_test.shape
#pd.DataFrame(y_test).idxmax(1)
In [187]:
#save the data set
#print locs[-40:]
np.save('/home/nick/Documents/LewisUniversity/MachineLearning/Project/visionmatrix',A)
In [188]:
#print X.shape
print categories.shape
print A.shape
In [ ]:
In [190]:
# split up data into train and test sets.
X_train, X_test, y_train, y_test = cross_validation.train_test_split(
A, categories, test_size=0.3, random_state=20)
In [ ]:
In [191]:
print X_train.shape
print y_train.shape
print X_test.shape
print y_test.shape
In [6]:
# re-load the saved data if needed
A = np.load('/home/nick/Documents/LewisUniversity/MachineLearning/Project/visionmatrix.npy')
In [192]:
#Let's start with the model parameters defined in the Week6 notebook for this data, changing the input shape as appropriate.
from keras.models import Sequential
from keras.layers import Dense, Dropout, Convolution2D, MaxPooling2D, Flatten
from keras.regularizers import l2, l1
from keras.optimizers import SGD
from keras import backend as K
K.set_image_dim_ordering('th')
model = Sequential()
model.add(Convolution2D(32, 5, 5, border_mode='valid', input_shape=(3, 50, 50), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dense(17, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
In [193]:
model.summary()
In [194]:
#Because of time constraints, we'll just run 10 epochs instead of say, 20.
model = model.fit(X_train, y_train, batch_size = 256,
nb_epoch = 10, verbose=2, validation_data=(X_test,y_test))
In [195]:
predicted = model.model.predict_classes(X_test)
In [196]:
y_train.shape
Out[196]:
In [197]:
predicted
Out[197]:
In [198]:
from sklearn.metrics import classification_report, f1_score, accuracy_score, confusion_matrix
# put the y_test back into a format of non-one-hot encoded for comparison
y_test_orig = pd.DataFrame(y_test).idxmax(1)
print "Convolution Network Results\nConfusion Matrix: "
print confusion_matrix(y_test_orig,predicted)
print "\n Classifcation Report"
print classification_report(y_test_orig,predicted)
This model is by no means great, but it does predict with .63 recall and .54 precision.
In [199]:
# Let's try with the Keras documentation example
# apply a 5x5 convolution with 32 output filters on a 50x50 image:
model = Sequential()
model.add(Convolution2D(16, 3, 3, border_mode='valid', input_shape=(3, 50, 50), activation='relu'))
# add a 3x3 convolution on top, with 16 output filters:
#model.add(Convolution2D(16, 3, 3, border_mode='same', activation='relu'))
# and flatten it and add a dense layer to get it to match the 17 categories.
model.add(Flatten())
model.add(Dense(17, activation='relu'))
# Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
In [200]:
model.summary()
In [201]:
model = model.fit(X_train, y_train, batch_size = 256,
nb_epoch = 5, verbose=2, validation_data=(X_test,y_test))
In [202]:
predicted = model.model.predict_classes(X_test)
In [203]:
# put the y_test back into a format of non-one-hot encoded for comparison
y_test_orig = pd.DataFrame(y_test).idxmax(1)
print "Convolution Network Results\nConfusion Matrix: "
print confusion_matrix(y_test_orig,predicted)
print "\n Classifcation Report"
print classification_report(y_test_orig,predicted)
This is a much worse model, and it is always predicting a value of 7.
In [205]:
# Let's try something totally different, a neural network based on the homework from week 5:
model = Sequential()
model.add(Flatten(input_shape=(3, 50, 50)))
model.add(Dense(output_dim=100,
activation='sigmoid', W_regularizer=l2(0.01)))
model.add(Dense(output_dim=500, activation='sigmoid', W_regularizer=l2(0.01)))
model.add(Dense(output_dim=17, activation='sigmoid', W_regularizer=l2(0.01)))
# Compile model
sgd = SGD(lr=0.1)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
In [206]:
model.summary()
In [207]:
# Fit the model
model = model.fit(X_train, y_train, batch_size = 256,
nb_epoch = 20, verbose=2, validation_data=(X_test,y_test))
In [208]:
predicted = model.model.predict_classes(X_test)
In [209]:
# put the y_test back into a format of non-one-hot encoded for comparison
y_test_orig = pd.DataFrame(y_test).idxmax(1)
print "Neural Network Results\nConfusion Matrix: "
print confusion_matrix(y_test_orig,predicted)
print "\n Classifcation Report"
print classification_report(y_test_orig,predicted)
Again, this is a lousy model like the previous one, as it just predicts 9.
In [210]:
# let's try logistic regression.
# Stochastic Logistic Regression
model = Sequential()
# validation loss
model.add(Flatten(input_shape=(3, 50, 50)))
model.add(Dense(output_dim=17, activation='sigmoid', W_regularizer=l1(0.01)))
# Compile model
sgd = SGD(lr=0.1)
model.compile(loss='mean_squared_error', optimizer=sgd, metrics=['accuracy'])
In [211]:
model.summary()
In [214]:
# Fit the model
model = model.fit(X_train, y_train, batch_size = 256,
nb_epoch = 100, verbose=0, validation_data=(X_test,y_test))
In [215]:
predicted = model.model.predict_classes(X_test)
In [216]:
# put the y_test back into a format of non-one-hot encoded for comparison
y_test_orig = pd.DataFrame(y_test).idxmax(1)
print "Logistic Regression Results\nConfusion Matrix: "
print confusion_matrix(y_test_orig,predicted)
print "\n Classifcation Report"
print classification_report(y_test_orig,predicted)
These neural network based models are giving the same results, except for the convolutional network. We'll try it again for a logistic regression, but pre-flattening the data.
In [217]:
# reshape the training and test data
X_train_new = np.empty(shape=[X_train.shape[0]] + [7500], dtype='float32')
for i in range(X_train.shape[0]):
X_train_new[i,:] = X_train[i,:,:,:].flatten()
X_test_new = np.empty(shape=[X_test.shape[0]] + [7500], dtype='float32')
for i in range(X_test.shape[0]):
X_test_new[i,:] = X_test[i,:,:,:].flatten()
In [218]:
print X_test_new.shape
print X_train_new.shape
In [219]:
# Stochastic Logistic Regression
model = Sequential()
# validation loss
model.add(Dense(output_dim=17, input_shape=[7500],
activation='sigmoid', W_regularizer=l2(0)))
# Compile model
sgd = SGD(lr=0.1)
model.compile(loss='categorical_crossentropy', optimizer=sgd)
In [220]:
model.summary()
In [223]:
# Fit the model
model = model.fit(X_train_new, y_train, batch_size = 256,
nb_epoch = 100, verbose=0, validation_data=(X_test_new,y_test))
In [225]:
predicted = model.model.predict_classes(X_test_new)
In [226]:
# put the y_test back into a format of non-one-hot encoded for comparison
y_test_orig = pd.DataFrame(y_test).idxmax(1)
print "Logistic Regression Results\nConfusion Matrix: "
print confusion_matrix(y_test_orig,predicted)
print "\n Classifcation Report"
print classification_report(y_test_orig,predicted)
And the same results as with the previous logistic regression.
In [227]:
# We will try sklearn's logistic regression.
from sklearn.linear_model import LogisticRegression
# regular
y_train_orig = pd.DataFrame(y_train).idxmax(1)
y_test_orig = pd.DataFrame(y_test).idxmax(1)
lr = LogisticRegression()
lr.fit(X_train_new, y_train_orig)
predicted = lr.predict(X_test_new)
In [228]:
print "Logistic Regression Results\nConfusion Matrix: "
print confusion_matrix(y_test_orig,predicted)
print "\n Classifcation Report"
print classification_report(y_test_orig,predicted)
With SciKitLearn's logistic regression algorithm, we get much better (but not good) precision than with the previous methods (excluding convolutional neural networks), but worse recall.
In [229]:
#Finally, we'll try the random forest, which is not known for being great with image data.
from sklearn.ensemble import RandomForestClassifier
from sklearn.grid_search import GridSearchCV
rfc = RandomForestClassifier(random_state=47, n_estimators=100)
rfc.fit(X_train_new, y_train_orig)
Out[229]:
In [230]:
predicted = rfc.predict(X_test_new)
print "Decision Tree Results\nConfusion Matrix: "
print confusion_matrix(y_test_orig,predicted)
print "\n Classifcation Report"
print classification_report(y_test_orig,predicted)
The random forest is actually working much better than the neural networks and logistic regression models, and better precision than even the convolutional neural network. The convolutional neural network still has the best recall, however. It's still not a great model, but it has precision and recall better then 50%, however, most test records are being predicted as 9 as in the other models. Apparently, our data is unbalanced among the classes - there are tons of category 'plain' compared to others.
In [267]:
import matplotlib.pyplot as plt
g=imgdf.groupby(['category']).count()['_unit_id']
objects = g.index
y_pos = np.arange(len(objects))
performance = g
plt.bar(y_pos, performance, align='center', alpha=0.5)
#plt.xticks(y_pos, objects)
plt.xticks(y_pos, objects, rotation='vertical')
plt.ylabel('Class Counts')
plt.title('Dress Classes in Data')
plt.show()