In [1]:
import numpy as np
import pandas as pd
import matplotlib
import os
from os import getcwd
from os import listdir
from os.path import isfile, join, isdir
import tensorflow as tf
from __future__ import division, print_function, absolute_import
import tflearn
from tflearn.data_utils import shuffle, to_categorical
from tflearn.metrics import Accuracy
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.estimator import regression
from tflearn.data_preprocessing import ImagePreprocessing
from tflearn.data_augmentation import ImageAugmentation
In [2]:
trainPath = '../data/raw/train'
testPath = '../data/raw/test_stg1'
rawdataPath = '../data/raw'
fish_classes = [f for f in listdir(trainPath) if isdir(join(trainPath, f))]
NUM_CATEGORIES = len(fish_classes)
In [3]:
%%time
#If the training images differ in size to the image input layer of the pretrained network,
# then you must resize or crop the image data
BUILD_HDF5_DATASET = True
IMAGE_SIZE = 32
VALIDATION_SPLIT = True
output_path = join(rawdataPath, 'fish_dataset_{}x{}.h5'.format(IMAGE_SIZE, IMAGE_SIZE))
input_path = join(rawdataPath, 'train')
if BUILD_HDF5_DATASET:
# Build a HDF5 dataset (only required once)
from tflearn.data_utils import build_hdf5_image_dataset
build_hdf5_image_dataset(target_path =input_path,
image_shape =(IMAGE_SIZE, IMAGE_SIZE),
mode ='folder',
output_path =output_path,
categorical_labels =True,
normalize =True)
In [4]:
%%time
from sklearn.model_selection import train_test_split
# Load HDF5 dataset
import h5py
h5f = h5py.File(output_path, 'r')
X_all = h5f['X'][()]
Y_all = h5f['Y'][()]
# Split into
if VALIDATION_SPLIT:
X, X_valid, Y, Y_valid = train_test_split(X_all, Y_all,
test_size =0.2,
random_state =23,
stratify =Y_all)
In [5]:
# Data loading and organization
from tflearn.datasets import cifar10
(CX, CY), (CX_test, CY_test) = cifar10.load_data()
CX, CY = shuffle(CX, CY)
CY = to_categorical(CY, 10)
CY_test = to_categorical(CY_test, 10)
In [6]:
# train and save a model
with tf.Graph().as_default():
# Real-time data preprocessing
img_prep = ImagePreprocessing()
img_prep.add_featurewise_zero_center()
img_prep.add_featurewise_stdnorm()
# Real-time data augmentation
img_aug = ImageAugmentation()
img_aug.add_random_flip_leftright()
img_aug.add_random_rotation(max_angle=25.)
# Convolutional network building
network = input_data(shape=[None, 32, 32, 3],
data_preprocessing=img_prep,
data_augmentation=img_aug)
network = conv_2d(network, 32, 3, activation='relu')
network = max_pool_2d(network, 2)
network = conv_2d(network, 64, 3, activation='relu')
network = conv_2d(network, 64, 3, activation='relu')
network = max_pool_2d(network, 2)
network = fully_connected(network, 512, activation='relu')
network = dropout(network, 0.5)
network = fully_connected(network, 10, activation='softmax')
network = regression(network, optimizer='adam',
loss='categorical_crossentropy',
learning_rate=0.001)
model = tflearn.DNN(network, tensorboard_verbose=0,checkpoint_path='cifar10_cnn.tfl.ckpt')
# Train using classifier
model.fit(CX, CY, n_epoch=1, shuffle=True, validation_set=(CX_test, CY_test),
show_metric=True, batch_size=100, run_id='cifar10_cnn')
model.save("cifar10_cnn.tfl")
In [7]:
# load that trained model and rerun
with tf.Graph().as_default():
# Real-time data preprocessing
img_prep = ImagePreprocessing()
img_prep.add_featurewise_zero_center()
img_prep.add_featurewise_stdnorm()
# Real-time data augmentation
img_aug = ImageAugmentation()
img_aug.add_random_flip_leftright()
img_aug.add_random_rotation(max_angle=25.)
# Convolutional network building
network = input_data(shape=[None, 32, 32, 3],
data_preprocessing=img_prep,
data_augmentation=img_aug)
network = conv_2d(network, 32, 3, activation='relu')
network = max_pool_2d(network, 2)
network = conv_2d(network, 64, 3, activation='relu')
network = conv_2d(network, 64, 3, activation='relu')
network = max_pool_2d(network, 2)
network = fully_connected(network, 512, activation='relu')
network = dropout(network, 0.5)
network = fully_connected(network, 10, activation='softmax')
network = regression(network, optimizer='adam',
loss='categorical_crossentropy',
learning_rate=0.001)
model_ref = tflearn.DNN(network, tensorboard_verbose=0)
model_ref.load('cifar10_cnn.tfl')
# Train using classifier
model_ref.fit(CX, CY, n_epoch=1, shuffle=True, validation_set=(CX_test, CY_test),
show_metric=True, batch_size=100, run_id='cifar10_cnn_refine')
In [9]:
# now load that model and run on the fish data. choose to not load weights using restore=False
with tf.Graph().as_default():
# Real-time data preprocessing
img_prep = ImagePreprocessing()
img_prep.add_featurewise_zero_center()
img_prep.add_featurewise_stdnorm()
# Real-time data augmentation
img_aug = ImageAugmentation()
img_aug.add_random_flip_leftright()
img_aug.add_random_rotation(max_angle=25.)
# Convolutional network building
network = input_data(shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
data_preprocessing=img_prep,
data_augmentation=img_aug)
network = conv_2d(network, 32, 3, activation='relu')
network = max_pool_2d(network, 2)
network = conv_2d(network, 64, 3, activation='relu')
network = conv_2d(network, 64, 3, activation='relu')
network = max_pool_2d(network, 2)
network = fully_connected(network, 512, activation='relu',restore=False)
network = dropout(network, 0.5)
network = fully_connected(network, NUM_CATEGORIES, activation='softmax',restore=False)
network = regression(network, optimizer='adam',
loss='categorical_crossentropy',
learning_rate=0.001)
model_ref = tflearn.DNN(network, tensorboard_verbose=0)
model_ref.load('cifar10_cnn.tfl')
# Train using classifier
model_ref.fit(X, Y, n_epoch=1, shuffle=True, validation_set=(X_valid, Y_valid),
show_metric=True, batch_size=5, run_id='fish_cnn_refine')
In [ ]: