In [1]:
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import collections
import cv2 #this needs to be imported before TF!!
import tensorflow as tf
from tensorflow.contrib.layers import flatten
import random
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
from six.moves import map
import re
import os
from datetime import datetime
%matplotlib inline
In [16]:
"""Embed a YouTube video via its embed url into a notebook."""
from functools import partial
from IPython.display import display, IFrame
width, height = (560, 315, )
def _iframe_attrs(embed_url):
"""Get IFrame args."""
return (
('src', 'width', 'height'),
(embed_url, width, height, ),
)
def _get_args(embed_url):
"""Get args for type to create a class."""
iframe = dict(zip(*_iframe_attrs(embed_url)))
attrs = {
'display': partial(display, IFrame(**iframe)),
}
return ('YouTubeVideo', (object, ), attrs, )
def youtube_video(embed_url):
"""Embed YouTube video into a notebook.
Place this module into the same directory as the notebook.
>>> from embed import youtube_video
>>> youtube_video(url).display()
"""
YouTubeVideo = type(*_get_args(embed_url)) # make a class
return YouTubeVideo() # return an object
In [2]:
# Summary Distribution before preprocessing
driving_log = pd.read_csv("data/driving_log.csv")
driving_log.columns = ['center_image','left_image','right_image','steering_angle','throttle','break','speed']
driving_log.steering_angle.plot.hist(alpha=0.7,bins=100)
_summary = {'mean':driving_log.steering_angle.mean(),
'median':driving_log.steering_angle.median(),
'max':driving_log.steering_angle.max(),
'min':driving_log.steering_angle.min(),
'std': driving_log.steering_angle.std()}
_summary
Out[2]:
In [3]:
# Output smoothing - smooth the ground truth steering angle by applying a rolling mean
# First compute the time difference between window start and end to make sure the frames
# belong to the same sequence
frame_lookback = 5
def compute_timestamp(df):
conv = "%Y%m%d%H%M%S%f"
r = r'(.*)(center_)(.*)(.jpg)'
_res = df.center_image.str.match(r)
df['record_time'] = [datetime.strptime(r[2].replace("_",""),conv) for r in _res]
df['record_diff'] = df['record_time'].diff(frame_lookback) / np.timedelta64(1, 's')
return df
In [4]:
# The module retrieves the time the image is captured and the delta between consecutive images
import functools
driving_log = compute_timestamp(driving_log)
_median_lookback = driving_log.record_diff.median()
print("Median time lapse between {} frames: {}".format(frame_lookback, _median_lookback))
def apply_smoothing(row, lookback):
if row['record_diff'] > lookback:
# Do not smooth otherwise
return row['steering_angle']
else:
return row['smooth_steering_angle']
# Add some leeway to include initial dataset in smoothing
smoothing_fn = functools.partial(apply_smoothing, lookback= _median_lookback + 1 )
driving_log['smooth_steering_angle'] = driving_log['steering_angle'].rolling(frame_lookback).mean()
driving_log['smooth_steering_angle'] = driving_log.apply(smoothing_fn,axis=1).fillna(0)
In [5]:
driving_log[['steering_angle','smooth_steering_angle']].plot()
Out[5]:
In [6]:
# Removing straight angles now
keep_straight_p = 0.25
keep = lambda x: True if np.random.random() < keep_straight_p or x != 0 else False
driving_log['is_kept'] = driving_log.steering_angle.apply(keep)
print("Before filter: {}".format(driving_log.shape[0]))
filtered_driving_log = driving_log[driving_log['is_kept']]
print("After filter: {}".format(filtered_driving_log.shape[0]))
In [7]:
# Ground truth processing after smoothing & filtering
filtered_driving_log.smooth_steering_angle.plot.hist(alpha=0.7,bins=100)
_summary = {'mean':filtered_driving_log.steering_angle.mean(),
'median':filtered_driving_log.smooth_steering_angle.median(),
'max':filtered_driving_log.smooth_steering_angle.max(),
'min':filtered_driving_log.smooth_steering_angle.min(),
'std': filtered_driving_log.smooth_steering_angle.std()}
_summary
Out[7]:
In [8]:
# adjust the steering angle heuristicly
# filter instances of straight driving
# create a mapping from imagenames to angles
# Take the smoothed angle
f = lambda x: "".join(x.split("/")[-1])
correction = s = np.random.normal(0.25, 0.05, filtered_driving_log.shape[0])
center = pd.DataFrame({'image': filtered_driving_log['center_image'].apply(f), 'steering_angle': filtered_driving_log['smooth_steering_angle']})
left = pd.DataFrame({'image': filtered_driving_log['left_image'].apply(f), 'steering_angle': filtered_driving_log['smooth_steering_angle'] + correction })
right = pd.DataFrame({'image': filtered_driving_log['right_image'].apply(f), 'steering_angle': filtered_driving_log['smooth_steering_angle'] - correction})
steering_angles = pd.concat([center, left, right])
target_map = pd.Series(steering_angles.steering_angle.values,index=steering_angles.image)
# # Split training / validation
_target_map_train, _target_map_validation = train_test_split(target_map, test_size= 0.1)
target_map_train, target_map_validation = _target_map_train.to_dict(), _target_map_validation.to_dict()
print("image data size: {}".format(len(target_map_train)))
_dir = "data/IMG/"
# Assert # images == # of annotated images
file_names = os.listdir(_dir)
#assert len(target_map) == len(file_names)
# No NaNs in the target data
assert sum(np.isnan(np.fromiter(iter(target_map_train.values()), dtype=float))) == 0
In [9]:
from data_generator.image import ImageDataGenerator
input_shape, output_shape = (160, 320, 3) , (32, 128, 3)
batch_size = 256
In [10]:
# investigate target distribution
gen = ImageDataGenerator(target_map = target_map_train,
root_path = _dir,
input_shape = input_shape,
output_shape = output_shape,
batch_size = batch_size)
# Generate some data
i = 0
X, y = [], []
while i < 5:
i += 1
_x, _y = next(gen)
X.append(_x)
y.append(_y)
print("done")
# Visualize the distribution of simulated training targets for couple of batches of training data
df = pd.DataFrame(y).T
f = plt.figure(figsize=(10, 10))
df.plot.hist(alpha=0.5, bins=20, legend=False, ax=f.gca())
Out[10]:
In [11]:
nb_images = 10
i = np.random.randint(1,128,nb_images)
#plt.imshow(X[0][5])
plt.figure(figsize=(75,50))
for ix, _i in enumerate(i):
plt.subplot(nb_images,1,ix+1)
plt.imshow(X[0][_i,...])
plt.axis('off')
plt.title("Steering Angle: " + str(np.round(y[0][_i],4)),fontsize=20);
In [12]:
# Set up the training and validation generator
train_gen = ImageDataGenerator(target_map = target_map_train,
root_path = _dir,
input_shape = input_shape,
output_shape = output_shape,
batch_size = batch_size)
val_gen = ImageDataGenerator(target_map = target_map_validation,
root_path = _dir,
input_shape = input_shape,
output_shape = output_shape,
batch_size = batch_size,
training = False)
In [13]:
# Set up the model
from model.model import CNNModel
model = CNNModel(input_shape = output_shape,
nb_fully_connected = 128,
filter_sizes = [8,5,3],
pool_size = (2,2),
dropout_p = 0.5,
l2_reg = 0.005 )
print("Model Summary:")
model.model.summary()
In [14]:
# Call *fit_generator*
hist = model.train(train_gen,
batch_size = 100,
nb_epochs= 100,
val_gen = val_gen
)
In [15]:
#save the model
model.save("model.json", "model.h5" )