In [0]:
! pip3 install kaggle
In [0]:
from google.colab import files
uploaded = files.upload()
In [0]:
! mkdir ~/.kaggle/
! cp ./kaggle.json ~/.kaggle/
In [0]:
! ls /root/.kaggle/
In [0]:
! kaggle competitions download -c dogs-vs-cats-redux-kernels-edition
In [0]:
! rm ./kaggle.json
! rm /root/.kaggle/kaggle.json
! rm -r ./datalab
! rm -r ./data
! rm -r ./sample_data/
In [0]:
! mkdir ./data
! mkdir ./data/train
! mkdir ./data/train/cats
! mkdir ./data/train/dogs
! mkdir ./data/preview
! mkdir ./data/validation
! mkdir ./data/validation/cats/
! mkdir ./data/validation/dogs/
In [0]:
!ls
In [0]:
! mv ./*.zip ./data/
In [0]:
! unzip -q ./data/train.zip -d ./data/train/
In [0]:
! mv ./data/train/train/cat.* ./data/train/cats/
! mv ./data/train/train/dog.* ./data/train/dogs/
In [0]:
! ls ./data/train/train/
In [0]:
! rm -r ./data/train/train
In [0]:
! ls ./data/train/dogs/ -1 | wc -l
In [0]:
! ls ./data/train/cats/ -1 | wc -l
In [0]:
! cd ./data/train/cats/ ; shuf -n 500 -e * | xargs -i mv {} ../../../data/validation/cats/
In [0]:
! cd ./data/train/dogs/ ; shuf -n 500 -e * | xargs -i mv {} ../../../data/validation/dogs/
In [0]:
! ls ./data/train/cats/ -1 | wc -l
In [0]:
! ls ./data/train/dogs/ -1 | wc -l
In [0]:
! ls ./data/validation/cats/ -1 | wc -l
In [0]:
! ls ./data/validation/dogs/ -1 | wc -l
In [0]:
! rm -r ./data/validation/cats/model
! rm -r ./data/validation/cats/datalab
In [0]:
! mkdir ./model/
In order to make the most out of our few training image data, the process of "augmentation" of these images via a number of random transformations is helpful. This process feed the data to the neural network model, so that it would never see twice the exact same picture. The key advantage of implementation such an augmentation strategy is to help prevent overfitting and better generalization by the trained model.
In Keras this can be done via the keras.preprocessing.image.ImageDataGenerator class. This class allows you to:
In [0]:
try:
import warnings
warnings.filterwarnings('ignore')
from keras.preprocessing.image import ImageDataGenerator
except:
print ("Please install Keras (cmd: $sudo pip3 install keras) to run this notebook ...")
In [0]:
datagen = ImageDataGenerator(rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
rescale=1./255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest')
For more information, see the documentation.
In [0]:
from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
In [0]:
datagen = ImageDataGenerator(rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest')
In [0]:
import os, random
random_image = random.choice(os.listdir('./data/train/cats/'))
img = load_img(os.path.join('./data/train/cats/' + str(random_image)))
print ('Loaded image for preview : ' + str(random_image))
x = img_to_array(img)
x = x.reshape((1,) + x.shape)
i = 0
for batch in datagen.flow(x, batch_size=1,
save_to_dir='./data/preview/', save_prefix='cat', save_format='jpeg'):
i += 1
if i > 20:
break
../data/cats_dogs/preview/ directory
In [0]:
import os
import matplotlib.image as mpl_image
import matplotlib.pyplot as plt
from IPython.display import Image as PyImage
In [0]:
def load_images(folder):
images = []
for filename in os.listdir(folder):
img = mpl_image.imread(os.path.join(folder, filename))
if img is not None:
images.append(img)
return images
In [0]:
def stack_plot(stack_size, folder):
rows, cols = stack_size, stack_size
fig,ax = plt.subplots(rows,cols,figsize=[24,24])
i = 0
try:
for filename in os.listdir(folder):
img = mpl_image.imread(os.path.join(folder, filename))
ax[int(i/rows),int(i % rows)].imshow(img)
ax[int(i/rows),int(i % rows)].axis('off')
i += 1
except:
print ("Failed to add an image to the stacked plot ...")
plt.show()
In [0]:
stack_plot(5, './data/preview/')
In [0]:
use_tpu=True
In [0]:
import os
if use_tpu:
TPU_WORKER = 'grpc://' + os.environ['COLAB_TPU_ADDR']
In [0]:
import tensorflow as tf
In [0]:
img_width, img_height = 150, 150
train_data_dir = './data/train/'
validation_data_dir = './data/validation/'
nb_train_samples = 24000
nb_validation_samples = 1000
epochs = 5
batch_size = 16
In [0]:
if tf.keras.backend.image_data_format() == 'channels_first':
input_shape = (3, img_width, img_height)
else:
input_shape = (img_width, img_height, 3)
In [0]:
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Conv2D(32, (3, 3), input_shape=input_shape))
model.add(tf.keras.layers.Activation('relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
model.add(tf.keras.layers.Conv2D(32, (3, 3)))
model.add(tf.keras.layers.Activation('relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
model.add(tf.keras.layers.Conv2D(64, (3, 3)))
model.add(tf.keras.layers.Activation('relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(64))
model.add(tf.keras.layers.Activation('relu'))
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(1))
model.add(tf.keras.layers.Activation('sigmoid'))
In [0]:
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
In [0]:
train_datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
In [0]:
test_datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1. / 255)
In [0]:
train_generator = train_datagen.flow_from_directory(train_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary')
In [0]:
validation_generator = test_datagen.flow_from_directory(validation_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary')
In [0]:
! wget https://github.com/rahulremanan/HIMA/raw/master/examples/Notebooks/01_Image_augmentation/weights/first_try.h5
In [0]:
! mv ./first_try.h5 ./model/
In [0]:
#tf.keras.backend.clear_session()
if use_tpu:
model = tf.contrib.tpu.keras_to_tpu_model(model,
strategy=tf.contrib.tpu.TPUDistributionStrategy(
tf.contrib.cluster_resolver.TPUClusterResolver(TPU_WORKER)))
In [0]:
model_path = './model/first_try.h5'
lr = 1e-4
In [0]:
if os.path.exists(model_path) and use_tpu:
model.load_weights(model_path)
print ('Loaded saved weights file ...')
elif os.path.exists(model_path):
model.load_weights(model_path)
print ('Loaded saved weights file ...')
else:
print ('Failed to load model weights ...')
In [0]:
model.fit_generator(train_generator,
steps_per_epoch=nb_train_samples // batch_size,
epochs=epochs,
validation_data=validation_generator,
validation_steps=nb_validation_samples // batch_size,
callbacks = [tf.contrib.keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=10,
verbose=0,
mode='min'),
tf.contrib.keras.callbacks.ModelCheckpoint(model_path,
monitor='val_loss',
save_best_only=True,
mode='min',
verbose=0),
tf.contrib.keras.callbacks.ReduceLROnPlateau(monitor='val_acc',
patience=2,
mode = 'max',
min_delta=1e-4,
cooldown=1,
verbose=1,
factor=0.5,
min_lr=lr*1e-4)])
model.save_weights(model_path,
overwrite=True)
In [0]:
model.save_weights(model_path)
In [0]:
! ls ./model/
In [0]:
from google.colab import files
files.download(model_path)
In [0]: