This notebook will show the general procedure to use our project data directories and how to do a regression task using convnets
In [1]:
#Create references to important directories we will use over and over
import os, sys
DATA_HOME_DIR = '/home/nathan/olin/spring2017/line-follower/line-follower/data'
In [2]:
#import modules
import numpy as np
from glob import glob
from PIL import Image
from tqdm import tqdm
from scipy.ndimage import zoom
from keras.models import Sequential
from keras.metrics import categorical_crossentropy, categorical_accuracy
from keras.layers.convolutional import *
from keras.preprocessing import image
from keras.layers.core import Flatten, Dense
from keras.optimizers import Adam
from keras.layers.normalization import BatchNormalization
from matplotlib import pyplot as plt
import seaborn as sns
%matplotlib inline
Create paths to data directories
In [3]:
%cd $DATA_HOME_DIR
path = DATA_HOME_DIR
train_path=path + '/sun_apr_16_office_full_line_1'
valid_path=path + '/sun_apr_16_office_full_line_2'
Throughout the notebook, we will take advantage of helper functions to cleanly process our data.
In [4]:
def resize_vectorized4D(data, new_size=(64, 64)):
"""
A vectorized implementation of 4d image resizing
Args:
data (4D array): The images you want to resize
new_size (tuple): The desired image size
Returns: (4D array): The resized images
"""
fy, fx = np.asarray(new_size, np.float32) / data.shape[1:3]
return zoom(data, (1, fy, fx, 1), order=1) # order is the order of spline interpolation
In [5]:
def lowerHalfImage(array):
"""
Returns the lower half rows of an image
Args: array (array): the array you want to extract the lower half from
Returns: The lower half of the array
"""
return array[round(array.shape[0]/2):,:,:]
In [6]:
def folder_to_numpy(image_directory_full):
"""
Read sorted pictures (by filename) in a folder to a numpy array.
We have hardcoded the extraction of the lower half of the images as
that is the relevant data
USAGE:
data_folder = '/train/test1'
X_train = folder_to_numpy(data_folder)
Args:
data_folder (str): The relative folder from DATA_HOME_DIR
Returns:
picture_array (np array): The numpy array in tensorflow format
"""
# change directory
print ("Moving to directory: " + image_directory_full)
os.chdir(image_directory_full)
# read in filenames from directory
g = glob('*.png')
if len(g) == 0:
g = glob('*.jpg')
print ("Found {} pictures".format(len(g)))
# sort filenames
g.sort()
# open and convert images to numpy array - then extract the lower half of each image
print("Starting pictures to numpy conversion")
picture_arrays = np.array([lowerHalfImage(np.array(Image.open(image_path))) for image_path in g])
# reshape to tensorflow format
# picture_arrays = picture_arrays.reshape(*picture_arrays.shape, 1)
print ("Shape of output: {}".format(picture_arrays.shape))
# return array
return picture_arrays
return picture_arrays.astype('float32')
In [7]:
def flip4DArray(array):
""" Produces the mirror images of a 4D image array """
return array[..., ::-1,:] #[:,:,::-1] also works but is 50% slower
In [8]:
def concatCmdVelFlip(array):
""" Concatentaes and returns Cmd Vel array """
return np.concatenate((array, array*-1)) # multiply by negative 1 for opposite turn
Because we are using a CNN and unordered pictures, we can flip our data and concatenate it on the end of all training and validation data to make sure we don't bias left or right turns.
Extract and store the training data in X_train and Y_train
In [9]:
%cd $train_path
Y_train = np.genfromtxt('cmd_vel.csv', delimiter=',')[:,1] # only use turning angle
Y_train = np.concatenate((Y_train, Y_train*-1))
X_train = folder_to_numpy(train_path + '/raw')
X_train = np.concatenate((X_train, flip4DArray(X_train)))
Test the shape of the arrays:
X_train: (N, 240, 640, 3)
Y_train: (N,)
In [10]:
X_train.shape, Y_train.shape
Out[10]:
Visualize the training data, currently using a hacky method to display the numpy matrix as this is being run over a remote server and I can't view new windows
In [11]:
%cd /tmp
img = Image.fromarray(X_train[0], 'RGB')
img.save("temp.jpg")
image.load_img("temp.jpg")
Out[11]:
Follow the same steps for as the training data for the validation data.
In [12]:
%cd $valid_path
Y_valid = np.genfromtxt('cmd_vel.csv', delimiter=',')[:,1]
Y_valid = np.concatenate((Y_valid, Y_valid*-1))
X_valid = folder_to_numpy(valid_path + '/raw')
X_valid = np.concatenate((X_valid, flip4DArray(X_valid)))
Test the shape of the arrays:
X_valid: (N, 240, 640, 3)
Y_valid: (N,)
In [13]:
X_valid.shape, Y_valid.shape
Out[13]:
When we train the network, we don't want to be dealing with (240, 640, 3) images as they are way too big. Instead, we will resize the images to something more managable, like (64, 64, 3) or (128, 128, 3). In terms of network predictive performance, we are not concerned with the change in aspect ratio, but might want to test a (24, 64, 3) images for faster training
In [7]:
img_rows, img_cols = (64, 64)
In [15]:
print(img_rows)
print(img_cols)
In [16]:
X_train = resize_vectorized4D(X_train, (img_rows, img_cols))
X_valid = resize_vectorized4D(X_valid, (img_rows, img_cols))
In [17]:
print(X_train.shape)
print(X_valid.shape)
Visualize newly resized image.
In [18]:
%cd /tmp
img = Image.fromarray(X_train[np.random.randint(0, X_train.shape[0])], 'RGB')
img.save("temp.jpg")
image.load_img("temp.jpg")
Out[18]:
gen allows us to normalize and augment our images. We will just use it to rescale the images.
In [19]:
gen = image.ImageDataGenerator(
# rescale=1. / 255 # normalize data between 0 and 1
)
Next, create the train and valid generators, these are shuffle and have a batch size of 32 by default
In [20]:
train_generator = gen.flow(X_train, Y_train)#, batch_size=batch_size, shuffle=True)
valid_generator = gen.flow(X_valid, Y_valid)#, batch_size=batch_size, shuffle=True)
# get_batches(train_path, batch_size=batch_size,
# target_size=in_shape,
# gen=gen)
# val_batches = get_batches(valid_path, batch_size=batch_size,
# target_size=in_shape,
# gen=gen)
In [21]:
data, category = next(train_generator)
print ("Shape of data: {}".format(data[0].shape))
%cd /tmp
img = Image.fromarray(data[np.random.randint(0, data.shape[0])].astype('uint8'), 'RGB')
img.save("temp.jpg")
image.load_img("temp.jpg")
Out[21]:
In [8]:
in_shape = (img_rows, img_cols, 3)
Our test model will use a VGG like structure with a few changes. We are removing the final activation function. We will also use either mean_absolute_error or mean_squared_error as our loss function for regression purposes.
In [9]:
def get_model():
model = Sequential([
Convolution2D(32,3,3, border_mode='same', activation='relu', input_shape=in_shape),
MaxPooling2D(),
Convolution2D(64,3,3, border_mode='same', activation='relu'),
MaxPooling2D(),
Convolution2D(128,3,3, border_mode='same', activation='relu'),
MaxPooling2D(),
Flatten(),
Dense(2048, activation='relu'),
Dense(1024, activation='relu'),
Dense(512, activation='relu'),
Dense(1)
])
model.compile(loss='mean_absolute_error', optimizer='adam')
return model
In [10]:
model = get_model()
In [11]:
model.summary()
In [ ]:
history = model.fit_generator(train_generator,
samples_per_epoch=train_generator.n,
nb_epoch=2500,
validation_data=valid_generator,
nb_val_samples=valid_generator.n,
verbose=True)
In [ ]:
# %cd $DATA_HOME_DIR
# model.save_weights('epoche_2500.h5')
In [12]:
%cd $DATA_HOME_DIR
model.load_weights('epoche_2500.h5')
In [ ]:
val_plot = np.convolve(history.history['val_loss'], np.repeat(1/10, 10), mode='valid')
train_plot = np.convolve(history.history['loss'], np.repeat(1/10, 10), mode='valid')
In [ ]:
sns.tsplot(val_plot)
In [ ]:
X_preds = model.predict(X_valid).reshape(X_valid.shape[0],)
for i in range(len(X_valid)):
print("{:07f} | {:07f}".format(Y_valid[i], X_preds[i]))
In [ ]:
X_train_preds = model.predict(X_train).reshape(X_train.shape[0],)
for i in range(len(X_train_preds)):
print("{:07f} | {:07f}".format(Y_train[i], X_train_preds[i]))
Notes
In [ ]: