In [1]:
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
import math
import numpy as np
from PIL import Image
import cv2
import matplotlib.pyplot as plt
from os import getcwd
import csv
# Fix error with TF and Keras
import tensorflow as tf
In [2]:
def preprocess_image(img):
'''
Adds gaussian blur and transforms BGR to YUV.
'''
new_img = cv2.GaussianBlur(img, (3,3), 0)
new_img = cv2.cvtColor(new_img, cv2.COLOR_BGR2YUV)
return new_img
def random_distortion(img):
'''
Adds random distortion to training dataset: random brightness, shadows and a random vertical shift
of the horizon position
'''
new_img = img.astype(float)
# Add random brightness
value = np.random.randint(-28, 28)
new_img[:,:,0] = np.minimum(np.maximum(new_img[:,:,0],0),255)
# Add random shadow covering the entire height but random width
img_height, img_width = new_img.shape[0:2]
middle_point = np.random.randint(0,img_width)
darkening = np.random.uniform(0.6,0.8)
if np.random.rand() > .5:
new_img[:,0:middle_point,0] *= darkening
else:
new_img[:,middle_point:img_width,0] *= darkening
# Applying a perspective transform at the beginning of the horizon line
horizon = 2*img_height/5 # Assumes horizon to be located at 2/5 of image height
v_shift = np.random.randint(-img_height/8,img_height/8) # Shifting horizon by up to 1/8
# First points correspond to a rectangle surrounding the image below the horizon line
pts1 = np.float32([[0,horizon],[img_width,horizon],[0,img_height],[img_width,img_height]])
# Second set of points correspond to same rectangle plus a random vertical shift
pts2 = np.float32([[0,horizon+v_shift],[img_width,horizon+v_shift],[0,img_height],[img_width,img_height]])
# Getting the perspective transformation
M = cv2.getPerspectiveTransform(pts1,pts2)
# pplying the perspective transformation
new_img = cv2.warpPerspective(new_img,M,(img_width,img_height), borderMode=cv2.BORDER_REPLICATE)
return new_img.astype(np.uint8)
def generator(image_paths, steering_angles, batch_size=32, validation_flag=False):
'''
Training batches generator. Does not distort the images if "validation_flag" is set to True
'''
num_samples = len(image_paths)
while 1:
image_paths, steering_angles = shuffle(image_paths, steering_angles)
for offset in range(0, num_samples, batch_size):
batch_images = image_paths[offset:offset+batch_size]
batch_angles = steering_angles[offset:offset+batch_size]
images = []
angles = []
for batch_angle ,batch_image in zip(batch_angles,batch_images):
img = cv2.imread(batch_image)
img = preprocess_image(img)
if not validation_flag:
img = random_distortion(img)
# Randomly flipping the image to augment data
# Only augmenting rare examples (angle > ~0.3)
if abs(batch_angle) > 0.3 and np.random.random_sample() >= 0.5:
img = cv2.flip(img, 1)
batch_angle *= -1
images.append(img)
angles.append(batch_angle)
X_train = np.array(images)
y_train = np.array(angles)
yield shuffle(X_train, y_train)
def generate_data(image_paths, angles, batch_size=20, validation_flag=False):
'''
Loads, preprocess and distorts images batch.
If 'validation_flag' is true the image is not distorted.
'''
image_batch = []
label_batch = []
image_paths, angles = shuffle(image_paths, angles)
for i in range(batch_size):
img = cv2.imread(image_paths[i])
angle = angles[i]
img = preprocess_image(img)
if not validation_flag:
img = random_distortion(img)
image_batch.append(img)
label_batch.append(angle)
return np.array(image_batch), np.array(label_batch)
In [3]:
# Name of folders containing data provided by Udacity and data collected by myself
data_folders = ['./data','./data2']
image_paths = []
steering_angles = []
minimum_speed = 10.0
speeds = []
for data_folder in data_folders:
with open(data_folder + '/driving_log.csv') as csvfile:
reader = csv.reader(csvfile)
for line in reader:
if line[0] == 'center':
# Ignore header
continue
else:
speeds.append(float(line[6]))
if float(line[6]) > minimum_speed:
# get center image path and angle
source_path = data_folder + '/IMG/'+line[0].split('/')[-1]
image_paths.append(source_path)
steering_angles.append(float(line[3]))
# get left image path and angle
source_path = data_folder + '/IMG/'+line[1].split('/')[-1]
image_paths.append(source_path)
steering_angles.append(float(line[3])+0.25)
# get left image path and angle
source_path = data_folder + '/IMG/'+line[2].split('/')[-1]
image_paths.append(source_path)
steering_angles.append(float(line[3])-0.25)
image_paths = np.array(image_paths)
steering_angles = np.array(steering_angles)
print('Samples in dataset:', image_paths.shape[0])
The speed of the car influences the steering angle. We can't just ignore how fast or slow was the car driving when collecting the data. I decided to remove those examples where the car was traveling at less than a minimum speed. This minimum speed was set to 10.0 based on the following distribution and on the speeed with which we are going to test the model:
In [4]:
plt.figure(figsize=(8,8))
plt.hist(speeds, bins=50)
plt.title('Distribution of training speed')
plt.show()
In [10]:
# Number of bins was determined experimentally
num_bins = 40
hist, bins = np.histogram(steering_angles, num_bins)
avg_samples_per_bin = np.mean(hist)
plt.figure(figsize=(10,8))
plt.hist(steering_angles, bins=bins)
plt.plot((np.min(steering_angles), np.max(steering_angles)), (avg_samples_per_bin, avg_samples_per_bin), 'k-')
plt.ylabel('Number of training samples', fontsize=20)
plt.xlabel('Steering angle', fontsize=20)
plt.show()
We can see that most of the training examples correspond to driving straight forward. We need to try to flat this distribution to achieve better performance when training.
In [11]:
# First, determine keep probability for each bin. If number of samples is larger than avg, we will remove
# proportionally to the number of samples above the average; otherwise we keep all samples.
new_target_avg = avg_samples_per_bin * 1.0
keep_probs = []
for i in range(num_bins):
if hist[i] < new_target_avg:
keep_probs.append(1.)
else:
keep_probs.append(1./(hist[i]/new_target_avg))
In [12]:
idx_to_remove = []
for i in range(len(steering_angles)):
for j in range(num_bins):
if steering_angles[i] >= bins[j] and steering_angles[i] <= bins[j+1]:
# Delete with probability 1-keep_prob
if np.random.random_sample() > keep_probs[j]:
idx_to_remove.append(i)
In [13]:
image_paths = np.delete(image_paths, idx_to_remove, axis=0)
steering_angles = np.delete(steering_angles, idx_to_remove)
In [16]:
hist, bins = np.histogram(steering_angles, num_bins)
plt.figure(figsize=(10,8))
plt.hist(steering_angles, bins=bins)
plt.plot((np.min(steering_angles), np.max(steering_angles)), (avg_samples_per_bin, avg_samples_per_bin), 'k-')
plt.ylabel('Number of training samples', fontsize=20)
plt.xlabel('Steering angle', fontsize=20)
plt.show()
print('Samples in dataset:', image_paths.shape[0])
In [17]:
img = cv2.cvtColor(cv2.imread('./assets/example.jpeg'), cv2.COLOR_BGR2RGB)
plt.figure(figsize=(7,7))
plt.imshow(img)
plt.show()
In [18]:
img = cv2.imread('./assets/example.jpeg')
img = preprocess_image(img)
temp_img = cv2.cvtColor(img, cv2.COLOR_YUV2RGB) # For visualization
plt.figure(figsize=(7,7))
plt.imshow(temp_img)
plt.show()
In [20]:
new_img = img.astype(float)
value = np.random.randint(-28, 28)
new_img[:,:,0] = np.minimum(np.maximum(img[:,:,0],0),255)
temp_img = cv2.cvtColor(new_img.astype(np.uint8), cv2.COLOR_YUV2RGB) # For visualization
plt.figure(figsize=(7,7))
plt.imshow(temp_img)
plt.show()
In [22]:
# Add random shadow covering the entire height but random width
img_height, img_width = new_img.shape[0:2]
middle_point = np.random.randint(0,img_width)
darkening = np.random.uniform(0.6,0.8)
if np.random.rand() > .5:
new_img[:,0:middle_point,0] *= darkening
else:
new_img[:,middle_point:img_width,0] *= darkening
temp_img = cv2.cvtColor(new_img.astype(np.uint8), cv2.COLOR_YUV2RGB) # For visualization
plt.figure(figsize=(7,7))
plt.imshow(temp_img)
plt.show()
In [24]:
def add_horizon_shift(img):
new_img = img.astype(float)
img_height, img_width = new_img.shape[0:2]
# Applying a perspective transform at the beginning of the horizon line
horizon = 2*img_height/5 # Assumes horizon to be located at 2/5 of image height
v_shift = np.random.randint(-img_height/8,img_height/8) # Shifting horizon by up to 1/8
# First points correspond to a rectangle surrounding the image below the horizon line
pts1 = np.float32([[0,horizon],[img_width,horizon],[0,img_height],[img_width,img_height]])
# Second set of points correspond to same rectangle plus a random vertical shift
pts2 = np.float32([[0,horizon+v_shift],[img_width,horizon+v_shift],[0,img_height],[img_width,img_height]])
M = cv2.getPerspectiveTransform(pts1,pts2)
new_img = cv2.warpPerspective(new_img,M,(img_width,img_height), borderMode=cv2.BORDER_REPLICATE)
return new_img.astype(np.uint8)
In [26]:
new_img = add_horizon_shift(new_img)
temp_img = cv2.cvtColor(new_img, cv2.COLOR_YUV2RGB) # For visualization
plt.figure(figsize=(7,7))
plt.imshow(temp_img)
plt.show()