Dogs versus Cats Redux Competition on Kaggle

Setup



In [1]:

    
#reset python environment
%reset -f

from pathlib import Path

import numpy as np
import tensorflow as tf
import time
import os

current_dir = os.getcwd()
home_directory = Path(os.getcwd())

dataset_directory = home_directory / "datasets" / "dogs-vs-cats-redux-kernels-edition"

training_dataset_dir = dataset_directory / "train"
validation_dataset_dir = dataset_directory / "valid"
test_dataset_dir = dataset_directory / "test1"

sample_dataset_directory = home_directory / "datasets" / "dogs-vs-cats-redux-kernels-edition" / "sample"
sample_training_dataset_dir = sample_dataset_directory / "train"
sample_validation_dataset_dir = sample_dataset_directory / "valid"
sample_test_dataset_dir = sample_dataset_directory / "test1"

dogs_dir = "dog"
cats_dir = "cat"

default_device = "/gpu:0"
# default_device = v/cpu:0"

Prepare Data

Note: This only needs to run if features haven't been already extracted

Pick random files from training set and use as validation set
Pick a subset of files for experimentation (sample)

Extract zip files



In [7]:

    
from zipfile import ZipFile

# Create base directory
dataset_directory.mkdir(parents=True)
    
# Kaggle's train.zip and test1.zip have to be present in ./zips/dogs-vs-cats-redux-kernels-edition/
zips_directory = Path("zips") / "dogs-vs-cats-redux-kernels-edition"

with ZipFile(str(zips_directory / "train.zip")) as train_zip:
    train_zip.extractall(dataset_directory)
    
with ZipFile(str(zips_directory / "test1.zip")) as test_zip:
    test_zip.extractall(dataset_directory)

Create validation and test sets



In [8]:

    
import os
import shutil
from glob import glob

valid_percentage = 0.1
sample_percentage = 0.1

def pick_random(files, percentage, target_dir, move=False):
    shuffled = np.random.permutation(files)
    num_files = int(len(shuffled) * percentage)

    for f in shuffled[:num_files]:
        if move:
            f.rename(target_dir / f.name)
        else:
            shutil.copy(str(f), str(target_dir / f.name))
                
try:    
    # Create directory for training and validation images
    cats_training_dataset_dir = training_dataset_dir / cats_dir
    dogs_training_dataset_dir = training_dataset_dir / dogs_dir
    
    cats_training_dataset_dir.mkdir()
    dogs_training_dataset_dir.mkdir()
    
    cats_validation_dataset_dir = validation_dataset_dir / cats_dir
    dogs_validation_dataset_dir = validation_dataset_dir / dogs_dir
    
    cats_validation_dataset_dir.mkdir(parents=True)
    dogs_validation_dataset_dir.mkdir(parents=True)
    
    # Move classes to their respective directories
    for f in training_dataset_dir.glob("cat.*.jpg"):
        f.rename(cats_training_dataset_dir / f.name)

    for f in training_dataset_dir.glob("dog.*.jpg"):
        f.rename(dogs_training_dataset_dir / f.name)
    
    # Move randomly picked validation files
    pick_random(
        list(cats_training_dataset_dir.glob("*.jpg")), valid_percentage,
        cats_validation_dataset_dir, move=True)
    
    pick_random(
        list(dogs_training_dataset_dir.glob("*.jpg")), valid_percentage,
        dogs_validation_dataset_dir, move=True)
    
    # Create directories for sample data
    cats_sample_training_dataset_dir = (sample_training_dataset_dir / cats_dir)
    dogs_sample_training_dataset_dir = (sample_training_dataset_dir / dogs_dir)
    
    cats_sample_training_dataset_dir.mkdir(parents=True)
    dogs_sample_training_dataset_dir.mkdir(parents=True)
    
    cats_sample_validation_dataset_dir = sample_validation_dataset_dir / cats_dir
    dogs_sample_validation_dataset_dir = sample_validation_dataset_dir / dogs_dir
    
    cats_sample_validation_dataset_dir.mkdir(parents=True)
    dogs_sample_validation_dataset_dir.mkdir(parents=True)
    
    sample_test_dataset_dir.mkdir(parents=True)
    
    # Copy randomly picked training and test files to samples
    pick_random(
        list(cats_training_dataset_dir.glob("*.jpg")), sample_percentage,
        cats_sample_training_dataset_dir, move=False)
    
    pick_random(
        list(dogs_training_dataset_dir.glob("*.jpg")), sample_percentage,
        dogs_sample_training_dataset_dir, move=False)
    
    pick_random(
        list(test_dataset_dir.glob("*.jpg")), sample_percentage,
        sample_test_dataset_dir, move=False)
        
    # Move randomly picked validation files
    pick_random(
        list(cats_sample_training_dataset_dir.glob("*.jpg")), valid_percentage,
        cats_sample_validation_dataset_dir, move=False)
    
    pick_random(
        list(dogs_sample_training_dataset_dir.glob("*.jpg")), valid_percentage,
        dogs_sample_validation_dataset_dir, move=False)
    
    print("Done. Validation and sample sets created.")
except FileExistsError as e:
    print("Error: Looks like data has already been prepared. Delete everything except the zip files to recreate.")









    



Done. Validation and sample sets created.

Checkpoint - Extract Features



In [16]:

    
from glob import glob
                    
def filenames_and_labels(path):
    cat_filenames = np.array(glob("{}/cat/*.jpg".format(path)))
    cat_labels = np.tile([1, 0], (len(cat_filenames), 1))
    dog_filenames = np.array(glob("{}/dog/*.jpg".format(path)))
    dog_labels = np.tile([0, 1], (len(dog_filenames), 1))
    
    return np.concatenate([cat_filenames, dog_filenames]), np.concatenate([cat_labels, dog_labels])



In [150]:

    
import time
import tensorflow as tf

import tensorflow_image_utils as tiu
from vgg16 import Vgg16Model

def extract_features(*, sess, directory, output_filename, batch_size=32, augment=False, input_epochs=1):
    filenames, labels = filenames_and_labels(directory)

    filename_queue, label_queue = tf.train.slice_input_producer(
                        [
                            tf.convert_to_tensor(filenames, dtype=tf.string),
                            tf.convert_to_tensor(labels, dtype=tf.float32)
                        ], num_epochs=input_epochs, shuffle=False)
    
    image = tiu.load_image(filename_queue, size=(224, 224))
    
    if augment:
        image = tiu.distort_image(image)    
        
    image = tiu.vgg16_preprocess(image, shape=(224, 224, 3))
    
    batched_data = tf.train.batch(
        [image, label_queue, filename_queue],
        batch_size=batch_size,
        num_threads=4,
        enqueue_many=False,
        allow_smaller_final_batch=True,
        capacity=3 * batch_size, )
    
    inputs = tf.placeholder(tf.float32, shape=(None, 224, 224, 3), name="input")
    model = Vgg16Model()
    model.build(inputs)

    sess.run([
        tf.local_variables_initializer(),
        tf.global_variables_initializer()
    ])
    
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess,coord=coord)
    
    codes = []
    
    num_unique_files = len(filenames)
    num_files_to_process = num_unique_files * input_epochs
    num_iterations = num_files_to_process // batch_size
    
    if num_files_to_process % batch_size != 0:
        num_iterations = num_iterations + 1
    
    current_iteration = 0
    
    tstart = time.perf_counter()
    try:
        while not coord.should_stop():
            t0 = time.perf_counter()
            batch_images, batch_labels, batch_filenames = sess.run(batched_data)
            t1 = time.perf_counter()
            
            print("\nIteration {}/{}:".format(current_iteration + 1, num_iterations))
            print("\tFetching batch took {:.3f} seconds".format(t1-t0))
            
            # flatten shape of maxpool5: (7, 7, 512) -> 7 * 7 * 512
            flattened = tf.reshape(model.max_pool5, shape=(-1, 7 * 7 * 512))
            
            features = sess.run(flattened, feed_dict={inputs: batch_images})
            t2 = time.perf_counter()
            print("\tExtracting features took {:.3f} seconds".format(t2-t1))
            
            for i, batch_filename in enumerate(batch_filenames):
                codes.append((batch_labels[i], batch_filename, features[i]))
            
            t3 = time.perf_counter()
            current_iteration = current_iteration + 1
            print("\tProcessing {} images took {:.3f} seconds".format(len(batch_filenames), t3-t0))
    except tf.errors.OutOfRangeError:
            print("\nDone -- epoch limit reached")
    finally:
        coord.request_stop()
        coord.join(threads)
        
    np.save(output_filename, np.array(codes, dtype="object"))

    print("Extracted to '{}' in {:.3f} seconds\n\n".format(output_filename ,time.perf_counter() - tstart))



In [151]:

    
with tf.Session(graph=tf.Graph()) as sess:
    extract_features(sess=sess, directory=sample_validation_dataset_dir,
                     output_filename="sample_validation_codes.npy", input_epochs=2)
    
with tf.Session(graph=tf.Graph()) as sess:
    extract_features(sess=sess, directory=sample_training_dataset_dir,
                     output_filename="sample_training_codes.npy",
                     augment=True, input_epochs=4)









    



Iteration 1/14:
	Fetching batch took 0.021 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.363 seconds

Iteration 2/14:
	Fetching batch took 0.006 seconds
	Extracting features took 0.335 seconds
	Processing 32 images took 0.341 seconds

Iteration 3/14:
	Fetching batch took 0.008 seconds
	Extracting features took 0.333 seconds
	Processing 32 images took 0.342 seconds

Iteration 4/14:
	Fetching batch took 0.006 seconds
	Extracting features took 0.335 seconds
	Processing 32 images took 0.342 seconds

Iteration 5/14:
	Fetching batch took 0.007 seconds
	Extracting features took 0.335 seconds
	Processing 32 images took 0.343 seconds

Iteration 6/14:
	Fetching batch took 0.007 seconds
	Extracting features took 0.335 seconds
	Processing 32 images took 0.342 seconds

Iteration 7/14:
	Fetching batch took 0.008 seconds
	Extracting features took 0.335 seconds
	Processing 32 images took 0.343 seconds

Iteration 8/14:
	Fetching batch took 0.007 seconds
	Extracting features took 0.345 seconds
	Processing 32 images took 0.352 seconds

Iteration 9/14:
	Fetching batch took 0.007 seconds
	Extracting features took 0.350 seconds
	Processing 32 images took 0.357 seconds

Iteration 10/14:
	Fetching batch took 0.007 seconds
	Extracting features took 0.345 seconds
	Processing 32 images took 0.352 seconds

Iteration 11/14:
	Fetching batch took 0.009 seconds
	Extracting features took 0.335 seconds
	Processing 32 images took 0.344 seconds

Iteration 12/14:
	Fetching batch took 0.007 seconds
	Extracting features took 0.337 seconds
	Processing 32 images took 0.344 seconds

Iteration 13/14:
	Fetching batch took 0.007 seconds
	Extracting features took 0.349 seconds
	Processing 32 images took 0.357 seconds

Iteration 14/14:
	Fetching batch took 0.007 seconds
	Extracting features took 0.329 seconds
	Processing 32 images took 0.336 seconds

Done -- epoch limit reached
Extracted to 'sample_validation_codes.npy' in 4.957 seconds



Iteration 1/282:
	Fetching batch took 0.052 seconds
	Extracting features took 0.358 seconds
	Processing 32 images took 0.410 seconds

Iteration 2/282:
	Fetching batch took 0.026 seconds
	Extracting features took 0.352 seconds
	Processing 32 images took 0.378 seconds

Iteration 3/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.403 seconds
	Processing 32 images took 0.427 seconds

Iteration 4/282:
	Fetching batch took 0.027 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.373 seconds

Iteration 5/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.340 seconds
	Processing 32 images took 0.360 seconds

Iteration 6/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.369 seconds

Iteration 7/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.365 seconds

Iteration 8/282:
	Fetching batch took 0.028 seconds
	Extracting features took 0.333 seconds
	Processing 32 images took 0.361 seconds

Iteration 9/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.360 seconds

Iteration 10/282:
	Fetching batch took 0.018 seconds
	Extracting features took 0.334 seconds
	Processing 32 images took 0.353 seconds

Iteration 11/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.352 seconds
	Processing 32 images took 0.373 seconds

Iteration 12/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.361 seconds

Iteration 13/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.333 seconds
	Processing 32 images took 0.353 seconds

Iteration 14/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.360 seconds

Iteration 15/282:
	Fetching batch took 0.019 seconds
	Extracting features took 0.333 seconds
	Processing 32 images took 0.352 seconds

Iteration 16/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.393 seconds
	Processing 32 images took 0.413 seconds

Iteration 17/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.335 seconds
	Processing 32 images took 0.356 seconds

Iteration 18/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.329 seconds
	Processing 32 images took 0.354 seconds

Iteration 19/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.333 seconds
	Processing 32 images took 0.359 seconds

Iteration 20/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.366 seconds

Iteration 21/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.391 seconds
	Processing 32 images took 0.413 seconds

Iteration 22/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.340 seconds
	Processing 32 images took 0.362 seconds

Iteration 23/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.335 seconds
	Processing 32 images took 0.354 seconds

Iteration 24/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.332 seconds
	Processing 32 images took 0.354 seconds

Iteration 25/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.333 seconds
	Processing 32 images took 0.358 seconds

Iteration 26/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.335 seconds
	Processing 32 images took 0.360 seconds

Iteration 27/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.337 seconds
	Processing 32 images took 0.358 seconds

Iteration 28/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.360 seconds

Iteration 29/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.332 seconds
	Processing 32 images took 0.355 seconds

Iteration 30/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.359 seconds

Iteration 31/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.361 seconds

Iteration 32/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.353 seconds
	Processing 32 images took 0.375 seconds

Iteration 33/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.360 seconds

Iteration 34/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.359 seconds

Iteration 35/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.335 seconds
	Processing 32 images took 0.358 seconds

Iteration 36/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.358 seconds

Iteration 37/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.330 seconds
	Processing 32 images took 0.353 seconds

Iteration 38/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.361 seconds

Iteration 39/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.360 seconds

Iteration 40/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.355 seconds
	Processing 32 images took 0.378 seconds

Iteration 41/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.361 seconds

Iteration 42/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.361 seconds

Iteration 43/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.330 seconds
	Processing 32 images took 0.353 seconds

Iteration 44/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.367 seconds

Iteration 45/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.337 seconds
	Processing 32 images took 0.360 seconds

Iteration 46/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.337 seconds
	Processing 32 images took 0.360 seconds

Iteration 47/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.360 seconds

Iteration 48/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.358 seconds

Iteration 49/282:
	Fetching batch took 0.019 seconds
	Extracting features took 0.391 seconds
	Processing 32 images took 0.410 seconds

Iteration 50/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.331 seconds
	Processing 32 images took 0.353 seconds

Iteration 51/282:
	Fetching batch took 0.026 seconds
	Extracting features took 0.335 seconds
	Processing 32 images took 0.361 seconds

Iteration 52/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.358 seconds

Iteration 53/282:
	Fetching batch took 0.026 seconds
	Extracting features took 0.396 seconds
	Processing 32 images took 0.422 seconds

Iteration 54/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.335 seconds
	Processing 32 images took 0.358 seconds

Iteration 55/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.358 seconds

Iteration 56/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.337 seconds
	Processing 32 images took 0.358 seconds

Iteration 57/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.330 seconds
	Processing 32 images took 0.352 seconds

Iteration 58/282:
	Fetching batch took 0.026 seconds
	Extracting features took 0.337 seconds
	Processing 32 images took 0.364 seconds

Iteration 59/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.363 seconds

Iteration 60/282:
	Fetching batch took 0.019 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.358 seconds

Iteration 61/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.333 seconds
	Processing 32 images took 0.356 seconds

Iteration 62/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.362 seconds

Iteration 63/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.334 seconds
	Processing 32 images took 0.358 seconds

Iteration 64/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.366 seconds

Iteration 65/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.335 seconds
	Processing 32 images took 0.359 seconds

Iteration 66/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.363 seconds

Iteration 67/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.357 seconds

Iteration 68/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.337 seconds
	Processing 32 images took 0.358 seconds

Iteration 69/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.363 seconds

Iteration 70/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.358 seconds

Iteration 71/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.337 seconds
	Processing 32 images took 0.359 seconds

Iteration 72/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.363 seconds

Iteration 73/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.361 seconds

Iteration 74/282:
	Fetching batch took 0.019 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.358 seconds

Iteration 75/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.357 seconds
	Processing 32 images took 0.378 seconds

Iteration 76/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.355 seconds
	Processing 32 images took 0.376 seconds

Iteration 77/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.360 seconds

Iteration 78/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.362 seconds

Iteration 79/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.395 seconds
	Processing 32 images took 0.415 seconds

Iteration 80/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.337 seconds
	Processing 32 images took 0.357 seconds

Iteration 81/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.340 seconds
	Processing 32 images took 0.361 seconds

Iteration 82/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.358 seconds

Iteration 83/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.340 seconds
	Processing 32 images took 0.360 seconds

Iteration 84/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.358 seconds

Iteration 85/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.360 seconds

Iteration 86/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.337 seconds
	Processing 32 images took 0.359 seconds

Iteration 87/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.334 seconds
	Processing 32 images took 0.358 seconds

Iteration 88/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.364 seconds

Iteration 89/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.364 seconds

Iteration 90/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.340 seconds
	Processing 32 images took 0.363 seconds

Iteration 91/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.337 seconds
	Processing 32 images took 0.358 seconds

Iteration 92/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.368 seconds

Iteration 93/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.393 seconds
	Processing 32 images took 0.415 seconds

Iteration 94/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.340 seconds
	Processing 32 images took 0.363 seconds

Iteration 95/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.364 seconds

Iteration 96/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.366 seconds

Iteration 97/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.356 seconds

Iteration 98/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.363 seconds

Iteration 99/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.333 seconds
	Processing 32 images took 0.357 seconds

Iteration 100/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.362 seconds

Iteration 101/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.365 seconds

Iteration 102/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.360 seconds

Iteration 103/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.364 seconds

Iteration 104/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.360 seconds

Iteration 105/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.362 seconds

Iteration 106/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.340 seconds
	Processing 32 images took 0.362 seconds

Iteration 107/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.362 seconds

Iteration 108/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.351 seconds
	Processing 32 images took 0.372 seconds

Iteration 109/282:
	Fetching batch took 0.027 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.369 seconds

Iteration 110/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.365 seconds

Iteration 111/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.397 seconds
	Processing 32 images took 0.419 seconds

Iteration 112/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.337 seconds
	Processing 32 images took 0.359 seconds

Iteration 113/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.366 seconds

Iteration 114/282:
	Fetching batch took 0.027 seconds
	Extracting features took 0.337 seconds
	Processing 32 images took 0.364 seconds

Iteration 115/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.366 seconds

Iteration 116/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.340 seconds
	Processing 32 images took 0.360 seconds

Iteration 117/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.333 seconds
	Processing 32 images took 0.355 seconds

Iteration 118/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.332 seconds
	Processing 32 images took 0.353 seconds

Iteration 119/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.332 seconds
	Processing 32 images took 0.356 seconds

Iteration 120/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.366 seconds

Iteration 121/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.366 seconds

Iteration 122/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.363 seconds

Iteration 123/282:
	Fetching batch took 0.026 seconds
	Extracting features took 0.361 seconds
	Processing 32 images took 0.387 seconds

Iteration 124/282:
	Fetching batch took 0.019 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.358 seconds

Iteration 125/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.362 seconds

Iteration 126/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.367 seconds

Iteration 127/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.363 seconds

Iteration 128/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.362 seconds

Iteration 129/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.340 seconds
	Processing 32 images took 0.363 seconds

Iteration 130/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.399 seconds
	Processing 32 images took 0.422 seconds

Iteration 131/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.398 seconds
	Processing 32 images took 0.418 seconds

Iteration 132/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.370 seconds

Iteration 133/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.367 seconds

Iteration 134/282:
	Fetching batch took 0.019 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.361 seconds

Iteration 135/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.334 seconds
	Processing 32 images took 0.356 seconds

Iteration 136/282:
	Fetching batch took 0.029 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.367 seconds

Iteration 137/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.364 seconds

Iteration 138/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.367 seconds

Iteration 139/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.367 seconds

Iteration 140/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.389 seconds
	Processing 32 images took 0.413 seconds

Iteration 141/282:
	Fetching batch took 0.026 seconds
	Extracting features took 0.358 seconds
	Processing 32 images took 0.385 seconds

Iteration 142/282:
	Fetching batch took 0.019 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.361 seconds

Iteration 143/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.340 seconds
	Processing 32 images took 0.362 seconds

Iteration 144/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.340 seconds
	Processing 32 images took 0.362 seconds

Iteration 145/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.345 seconds
	Processing 32 images took 0.366 seconds

Iteration 146/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.365 seconds

Iteration 147/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.367 seconds

Iteration 148/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.365 seconds

Iteration 149/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.367 seconds

Iteration 150/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.340 seconds
	Processing 32 images took 0.361 seconds

Iteration 151/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.365 seconds

Iteration 152/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.398 seconds
	Processing 32 images took 0.421 seconds

Iteration 153/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.403 seconds
	Processing 32 images took 0.426 seconds

Iteration 154/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.346 seconds
	Processing 32 images took 0.366 seconds

Iteration 155/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.399 seconds
	Processing 32 images took 0.421 seconds

Iteration 156/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.366 seconds

Iteration 157/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.333 seconds
	Processing 32 images took 0.357 seconds

Iteration 158/282:
	Fetching batch took 0.027 seconds
	Extracting features took 0.345 seconds
	Processing 32 images took 0.372 seconds

Iteration 159/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.397 seconds
	Processing 32 images took 0.418 seconds

Iteration 160/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.398 seconds
	Processing 32 images took 0.421 seconds

Iteration 161/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.357 seconds
	Processing 32 images took 0.378 seconds

Iteration 162/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.368 seconds

Iteration 163/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.372 seconds

Iteration 164/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.362 seconds

Iteration 165/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.365 seconds

Iteration 166/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.366 seconds

Iteration 167/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.364 seconds

Iteration 168/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.364 seconds

Iteration 169/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.374 seconds

Iteration 170/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.366 seconds

Iteration 171/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.363 seconds

Iteration 172/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.345 seconds
	Processing 32 images took 0.367 seconds

Iteration 173/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.363 seconds

Iteration 174/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.369 seconds
	Processing 32 images took 0.394 seconds

Iteration 175/282:
	Fetching batch took 0.018 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.360 seconds

Iteration 176/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.366 seconds

Iteration 177/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.341 seconds
	Processing 32 images took 0.361 seconds

Iteration 178/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.372 seconds

Iteration 179/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.365 seconds

Iteration 180/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.345 seconds
	Processing 32 images took 0.368 seconds

Iteration 181/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.400 seconds
	Processing 32 images took 0.425 seconds

Iteration 182/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.367 seconds
	Processing 32 images took 0.389 seconds

Iteration 183/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.365 seconds

Iteration 184/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.365 seconds

Iteration 185/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.367 seconds

Iteration 186/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.369 seconds

Iteration 187/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.364 seconds
	Processing 32 images took 0.387 seconds

Iteration 188/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.365 seconds

Iteration 189/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.339 seconds
	Processing 32 images took 0.364 seconds

Iteration 190/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.346 seconds
	Processing 32 images took 0.371 seconds

Iteration 191/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.365 seconds

Iteration 192/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.368 seconds

Iteration 193/282:
	Fetching batch took 0.026 seconds
	Extracting features took 0.402 seconds
	Processing 32 images took 0.428 seconds

Iteration 194/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.372 seconds

Iteration 195/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.367 seconds

Iteration 196/282:
	Fetching batch took 0.026 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.371 seconds

Iteration 197/282:
	Fetching batch took 0.028 seconds
	Extracting features took 0.332 seconds
	Processing 32 images took 0.360 seconds

Iteration 198/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.406 seconds
	Processing 32 images took 0.430 seconds

Iteration 199/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.371 seconds

Iteration 200/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.365 seconds

Iteration 201/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.364 seconds

Iteration 202/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.368 seconds

Iteration 203/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.351 seconds
	Processing 32 images took 0.372 seconds

Iteration 204/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.372 seconds

Iteration 205/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.369 seconds

Iteration 206/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.346 seconds
	Processing 32 images took 0.369 seconds

Iteration 207/282:
	Fetching batch took 0.019 seconds
	Extracting features took 0.342 seconds
	Processing 32 images took 0.361 seconds

Iteration 208/282:
	Fetching batch took 0.026 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.374 seconds

Iteration 209/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.403 seconds
	Processing 32 images took 0.425 seconds

Iteration 210/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.365 seconds

Iteration 211/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.349 seconds
	Processing 32 images took 0.372 seconds

Iteration 212/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.346 seconds
	Processing 32 images took 0.368 seconds

Iteration 213/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.401 seconds
	Processing 32 images took 0.425 seconds

Iteration 214/282:
	Fetching batch took 0.019 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.367 seconds

Iteration 215/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.370 seconds

Iteration 216/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.407 seconds
	Processing 32 images took 0.429 seconds

Iteration 217/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.369 seconds

Iteration 218/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.346 seconds
	Processing 32 images took 0.370 seconds

Iteration 219/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.368 seconds

Iteration 220/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.398 seconds
	Processing 32 images took 0.420 seconds

Iteration 221/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.373 seconds

Iteration 222/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.368 seconds

Iteration 223/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.346 seconds
	Processing 32 images took 0.369 seconds

Iteration 224/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.370 seconds

Iteration 225/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.370 seconds

Iteration 226/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.358 seconds

Iteration 227/282:
	Fetching batch took 0.028 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.376 seconds

Iteration 228/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.345 seconds
	Processing 32 images took 0.369 seconds

Iteration 229/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.351 seconds
	Processing 32 images took 0.372 seconds

Iteration 230/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.349 seconds
	Processing 32 images took 0.372 seconds

Iteration 231/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.345 seconds
	Processing 32 images took 0.369 seconds

Iteration 232/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.343 seconds
	Processing 32 images took 0.368 seconds

Iteration 233/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.373 seconds

Iteration 234/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.346 seconds
	Processing 32 images took 0.366 seconds

Iteration 235/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.338 seconds
	Processing 32 images took 0.357 seconds

Iteration 236/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.346 seconds
	Processing 32 images took 0.371 seconds

Iteration 237/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.345 seconds
	Processing 32 images took 0.367 seconds

Iteration 238/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.344 seconds
	Processing 32 images took 0.365 seconds

Iteration 239/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.340 seconds
	Processing 32 images took 0.362 seconds

Iteration 240/282:
	Fetching batch took 0.028 seconds
	Extracting features took 0.351 seconds
	Processing 32 images took 0.379 seconds

Iteration 241/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.369 seconds

Iteration 242/282:
	Fetching batch took 0.026 seconds
	Extracting features took 0.368 seconds
	Processing 32 images took 0.394 seconds

Iteration 243/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.373 seconds

Iteration 244/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.373 seconds

Iteration 245/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.373 seconds

Iteration 246/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.346 seconds
	Processing 32 images took 0.369 seconds

Iteration 247/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.354 seconds
	Processing 32 images took 0.376 seconds

Iteration 248/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.345 seconds
	Processing 32 images took 0.368 seconds

Iteration 249/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.369 seconds

Iteration 250/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.371 seconds

Iteration 251/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.349 seconds
	Processing 32 images took 0.372 seconds

Iteration 252/282:
	Fetching batch took 0.026 seconds
	Extracting features took 0.336 seconds
	Processing 32 images took 0.363 seconds

Iteration 253/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.350 seconds
	Processing 32 images took 0.373 seconds

Iteration 254/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.351 seconds
	Processing 32 images took 0.375 seconds

Iteration 255/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.371 seconds

Iteration 256/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.370 seconds

Iteration 257/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.409 seconds
	Processing 32 images took 0.430 seconds

Iteration 258/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.371 seconds

Iteration 259/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.352 seconds
	Processing 32 images took 0.374 seconds

Iteration 260/282:
	Fetching batch took 0.024 seconds
	Extracting features took 0.406 seconds
	Processing 32 images took 0.430 seconds

Iteration 261/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.366 seconds
	Processing 32 images took 0.388 seconds

Iteration 262/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.350 seconds
	Processing 32 images took 0.374 seconds

Iteration 263/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.351 seconds
	Processing 32 images took 0.371 seconds

Iteration 264/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.403 seconds
	Processing 32 images took 0.425 seconds

Iteration 265/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.334 seconds
	Processing 32 images took 0.357 seconds

Iteration 266/282:
	Fetching batch took 0.027 seconds
	Extracting features took 0.352 seconds
	Processing 32 images took 0.379 seconds

Iteration 267/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.368 seconds

Iteration 268/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.368 seconds

Iteration 269/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.346 seconds
	Processing 32 images took 0.369 seconds

Iteration 270/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.350 seconds
	Processing 32 images took 0.373 seconds

Iteration 271/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.366 seconds
	Processing 32 images took 0.391 seconds

Iteration 272/282:
	Fetching batch took 0.025 seconds
	Extracting features took 0.346 seconds
	Processing 32 images took 0.371 seconds

Iteration 273/282:
	Fetching batch took 0.020 seconds
	Extracting features took 0.365 seconds
	Processing 32 images took 0.386 seconds

Iteration 274/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.370 seconds

Iteration 275/282:
	Fetching batch took 0.026 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.374 seconds

Iteration 276/282:
	Fetching batch took 0.023 seconds
	Extracting features took 0.407 seconds
	Processing 32 images took 0.431 seconds

Iteration 277/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.370 seconds
	Processing 32 images took 0.392 seconds

Iteration 278/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.347 seconds
	Processing 32 images took 0.369 seconds

Iteration 279/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.367 seconds
	Processing 32 images took 0.389 seconds

Iteration 280/282:
	Fetching batch took 0.021 seconds
	Extracting features took 0.348 seconds
	Processing 32 images took 0.369 seconds

Iteration 281/282:
	Fetching batch took 0.022 seconds
	Extracting features took 0.346 seconds
	Processing 32 images took 0.368 seconds

Iteration 282/282:
	Fetching batch took 0.001 seconds
	Extracting features took 0.104 seconds
	Processing 8 images took 0.105 seconds

Done -- epoch limit reached
Extracted to 'sample_training_codes.npy' in 106.617 seconds

Checkpoint - Transfer Learning Model



In [5]:

    
from tensorflow.python.saved_model import builder as saved_model_builder
from tensorflow.python.saved_model.signature_def_utils import predict_signature_def

from tensorflow.python.saved_model.tag_constants import SERVING
from tensorflow.python.saved_model.signature_constants import DEFAULT_SERVING_SIGNATURE_DEF_KEY
from tensorflow.python.saved_model.signature_constants import PREDICT_INPUTS
from tensorflow.python.saved_model.signature_constants import PREDICT_OUTPUTS


class TransferModel:
    
    def build(self, *, input_size, num_hidden=1, hidden_layer_size=256, use_batchnorm=True, use_dropout=True):
        with tf.name_scope("inputs"):
            self.input = tf.placeholder(tf.float32, shape=(None, input_size), name="input")
            self.is_training = tf.placeholder(tf.bool, name="is_training")
            self.keep_prob = tf.placeholder(tf.float32, name="keep_probability")
            self.learning_rate = tf.placeholder(tf.float32, name="learning_rate")

        with tf.name_scope("targets"):
            self.labels = tf.placeholder(tf.float32, shape=(None, 2), name="labels")

        prev_size = input_size
        next_input = self.input

        for i in range(num_hidden):
            with tf.variable_scope("hidden_layer_{}".format(i)):
                hidden_weights = tf.Variable(
                    initial_value = tf.truncated_normal([prev_size, hidden_layer_size], mean=0.0, stddev=0.01),
                    dtype=tf.float32, name="hidden_weights"
                )

                hidden_bias = tf.Variable(
                    initial_value = tf.zeros(hidden_layer_size), 
                    dtype=tf.float32,
                    name="hidden_bias"
                )

                hidden = tf.matmul(next_input, hidden_weights) + hidden_bias
                
                if use_batchnorm:
                    hidden = tf.layers.batch_normalization(hidden, training=self.is_training)
                hidden = tf.nn.relu(hidden, name="hidden_relu")
                
                if use_dropout:
                    hidden = tf.nn.dropout(hidden, keep_prob=self.keep_prob, name="hidden_dropout")

                tf.summary.histogram("hidden_weights_{}".format(i), hidden_weights)
                tf.summary.histogram("hidden_bias_{}".format(i), hidden_bias)

                next_input = hidden
                prev_size = hidden_layer_size


        with tf.name_scope("outputs"):
            output_weights = tf.Variable(
                initial_value=tf.truncated_normal(shape=(hidden_layer_size, 2), mean=0.0, stddev=0.01),
                dtype=tf.float32, name="output_weights"
            )

            output_bias = tf.Variable(initial_value=tf.zeros(2), dtype=tf.float32, name="output_bias")

            self.logits = tf.matmul(next_input, output_weights) + output_bias
            self.predictions = tf.nn.softmax(self.logits, name="predictions")

            tf.summary.histogram("output_weights", output_weights)
            tf.summary.histogram("output_bias", output_bias)
            tf.summary.histogram("predictions", self.predictions)

        with tf.name_scope("cost"):
            cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=self.logits, labels=self.labels, name="cross_entropy")
            self.cost = tf.reduce_mean(cross_entropy, name="cost")
            tf.summary.scalar("cost", self.cost)

        with tf.name_scope("train"):
            with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):                
                correct_predictions = tf.equal(tf.argmax(self.predictions, 1), tf.argmax(self.labels, 1), name="correct_predictions")
                self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, tf.float32), name="accuracy")
                self.optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(self.cost)
                
        self.merged_summaries = tf.summary.merge_all()
        
    def run_training(self, *, sess, fn_get_batches, num_epochs,
                     validation_images, validation_labels,
                     writer=None, keep_prob=0.5, batch_size=64, 
                     learning_rate=0.01, accuracy_print_steps=100):
        
        sess.run(tf.global_variables_initializer())

        iteration = 0
        for epoch in range(num_epochs):
                for batch_train_images, batch_train_labels in fn_get_batches(batch_size):
                    
                    train_acc, train_loss, _, p, summary = sess.run(
                        [self.accuracy, self.cost, self.optimizer, self.logits, self.merged_summaries], 
                        feed_dict = { 
                            self.input: batch_train_images,
                            self.labels: batch_train_labels,
                            self.keep_prob: keep_prob,
                            self.learning_rate: learning_rate,
                            self.is_training: True})

                    iteration = iteration + 1

                    if iteration % accuracy_print_steps == 0:
                        if not writer == None:
                            writer.add_summary(summary, iteration)

                        if iteration % accuracy_print_steps == 0:
                            val_acc = sess.run(self.accuracy, feed_dict ={
                                self.input: validation_images,
                                self.labels: validation_labels,
                                self.keep_prob: 1.,
                                self.is_training: False})

                            print("\tEpoch {}/{} Iteration {}, trainacc: {}, valacc: {}, loss: {}".format(epoch + 1, num_epochs, iteration, train_acc, val_acc, train_loss))
                            
    def save_model(self, *, sess, saved_model_path):
        builder = saved_model_builder.SavedModelBuilder(saved_model_path)

        builder.add_meta_graph_and_variables(
            sess, [SERVING],
            signature_def_map = {
                DEFAULT_SERVING_SIGNATURE_DEF_KEY: predict_signature_def(
                    inputs = { PREDICT_INPUTS: self.images },
                    outputs = { PREDICT_OUTPUTS: self.predictions }
                )
            }
        )

        builder.save()

Load labels and features



In [6]:

    
training_features = np.load("sample_training_codes.npy")
validation_features = np.load("sample_validation_codes.npy")

np.random.shuffle(training_features)

training_x = np.array(list(map(lambda row: row[2], training_features)))
training_y = np.array(list(map(lambda row: row[0], training_features)))

validation_x = np.array(list(map(lambda row: row[2], validation_features)))
validation_y = np.array(list(map(lambda row: row[0], validation_features)))



In [7]:

    
def get_batches(x, y, batch_size=32):
    num_rows = y.shape[0]
    
    num_batches = num_rows // batch_size
    
    if num_rows % batch_size != 0:
        num_batches = num_batches + 1

    for batch in range(num_batches):
        yield x[batch_size * batch: batch_size * (batch + 1)], y[batch_size * batch: batch_size * (batch + 1)]

Run model with dropout and batchnorm



In [12]:

    
tf.reset_default_graph()

with tf.Session() as sess:
    m = TransferModel()
    
    m.build(input_size=7 * 7 * 512, num_hidden=1, hidden_layer_size=256, use_batchnorm=True, use_dropout=True)
    
    m.run_training(
        sess=sess, num_epochs=5, learning_rate=0.01, keep_prob=0.8, batch_size=64,
        fn_get_batches=lambda batch_size: get_batches(training_x, training_y),
        validation_images=validation_x, validation_labels=validation_y)









    



	Epoch 1/5 Iteration 100, trainacc: 0.96875, valacc: 0.9241071939468384, loss: 0.06337632238864899
	Epoch 1/5 Iteration 200, trainacc: 1.0, valacc: 0.9598215222358704, loss: 0.07641230523586273
	Epoch 2/5 Iteration 300, trainacc: 0.96875, valacc: 0.9375001192092896, loss: 0.04248032346367836
	Epoch 2/5 Iteration 400, trainacc: 1.0, valacc: 0.9375, loss: 0.01440589502453804
	Epoch 2/5 Iteration 500, trainacc: 1.0, valacc: 0.9508928060531616, loss: 0.024288026615977287
	Epoch 3/5 Iteration 600, trainacc: 1.0, valacc: 0.9241071939468384, loss: 0.0008406672277487814
	Epoch 3/5 Iteration 700, trainacc: 1.0, valacc: 0.9196428656578064, loss: 0.005353865679353476
	Epoch 3/5 Iteration 800, trainacc: 1.0, valacc: 0.9375001192092896, loss: 0.0006719963275827467
	Epoch 4/5 Iteration 900, trainacc: 1.0, valacc: 0.9196428656578064, loss: 0.034090228378772736
	Epoch 4/5 Iteration 1000, trainacc: 1.0, valacc: 0.9151785969734192, loss: 0.0016528507694602013
	Epoch 4/5 Iteration 1100, trainacc: 1.0, valacc: 0.9375, loss: 0.0006564409704878926
	Epoch 5/5 Iteration 1200, trainacc: 1.0, valacc: 0.9241070747375488, loss: 0.007378122303634882
	Epoch 5/5 Iteration 1300, trainacc: 1.0, valacc: 0.9196428060531616, loss: 0.002801208756864071
	Epoch 5/5 Iteration 1400, trainacc: 1.0, valacc: 0.901785671710968, loss: 0.00813545472919941

Looks like we can stop after 2 epochs. Doesn't get much better afterwards.

Run model with dropout only



In [13]:

    
tf.reset_default_graph()

with tf.Session() as sess:
    m = TransferModel()
    
    m.build(input_size=7 * 7 * 512, num_hidden=1, hidden_layer_size=256, use_batchnorm=False, use_dropout=True)
    
    m.run_training(
        sess=sess, num_epochs=5, learning_rate=0.01, keep_prob=0.8, batch_size=64,
        fn_get_batches=lambda batch_size: get_batches(training_x, training_y),
        validation_images=validation_x, validation_labels=validation_y)









    



	Epoch 1/5 Iteration 100, trainacc: 0.6875, valacc: 0.7678571939468384, loss: 0.8892062902450562
	Epoch 1/5 Iteration 200, trainacc: 0.78125, valacc: 0.7946428656578064, loss: 0.47087350487709045
	Epoch 2/5 Iteration 300, trainacc: 0.625, valacc: 0.6473214626312256, loss: 0.5318276882171631
	Epoch 2/5 Iteration 400, trainacc: 0.625, valacc: 0.7901785969734192, loss: 0.5235291719436646
	Epoch 2/5 Iteration 500, trainacc: 0.5, valacc: 0.8616071939468384, loss: 0.6341253519058228
	Epoch 3/5 Iteration 600, trainacc: 0.75, valacc: 0.8660714030265808, loss: 0.49052342772483826
	Epoch 3/5 Iteration 700, trainacc: 0.65625, valacc: 0.6651785969734192, loss: 0.6229223012924194
	Epoch 3/5 Iteration 800, trainacc: 0.71875, valacc: 0.785714328289032, loss: 0.5308653116226196
	Epoch 4/5 Iteration 900, trainacc: 0.4375, valacc: 0.754464328289032, loss: 0.6473531723022461
	Epoch 4/5 Iteration 1000, trainacc: 0.625, valacc: 0.7991071939468384, loss: 0.581628680229187
	Epoch 4/5 Iteration 1100, trainacc: 0.65625, valacc: 0.745535671710968, loss: 0.5172590017318726
	Epoch 5/5 Iteration 1200, trainacc: 0.5625, valacc: 0.8526785373687744, loss: 0.6065104603767395
	Epoch 5/5 Iteration 1300, trainacc: 0.53125, valacc: 0.7901785373687744, loss: 0.5416250228881836
	Epoch 5/5 Iteration 1400, trainacc: 0.5, valacc: 0.8080357313156128, loss: 0.6116245985031128

Training accuracy jumps around a lot and is way lower than validation accuracy.

Either the learning rate it too low, or we're underfitting either because of regularization (dropout), or our model is not complex enough.

Jermeny@fast.ai:

Yes your assumption is true - although if you're underfitting due to reasons other than dropout (or other regularization techniques), you won't see this.

The key technique to avoiding underfitting is using a model with plenty of layers and parameters, and picking an appropriate architecture (e.g. CNN with batchnorm for images). Also picking appropriate learning rates.

Picking the output with the highest validation accuracy is generally a good approach.

Lower Learning Rate



In [15]:

    
tf.reset_default_graph()

with tf.Session() as sess:
    m = TransferModel()
    
    m.build(input_size=7 * 7 * 512, num_hidden=1, hidden_layer_size=256, use_batchnorm=False, use_dropout=True)
    
    m.run_training(
        sess=sess, num_epochs=5, learning_rate=0.001, keep_prob=0.8, batch_size=64,
        fn_get_batches=lambda batch_size: get_batches(training_x, training_y),
        validation_images=validation_x, validation_labels=validation_y)









    



	Epoch 1/5 Iteration 100, trainacc: 1.0, valacc: 0.9374999403953552, loss: 0.04282510653138161
	Epoch 1/5 Iteration 200, trainacc: 0.96875, valacc: 0.9598214626312256, loss: 0.06804436445236206
	Epoch 2/5 Iteration 300, trainacc: 1.0, valacc: 0.946428656578064, loss: 0.0420147143304348
	Epoch 2/5 Iteration 400, trainacc: 0.96875, valacc: 0.9553571939468384, loss: 0.0671163946390152
	Epoch 2/5 Iteration 500, trainacc: 1.0, valacc: 0.9553571939468384, loss: 0.03134910389780998
	Epoch 3/5 Iteration 600, trainacc: 1.0, valacc: 0.9508929252624512, loss: 0.0004375204152893275
	Epoch 3/5 Iteration 700, trainacc: 1.0, valacc: 0.9598215222358704, loss: 0.009630686603486538
	Epoch 3/5 Iteration 800, trainacc: 0.96875, valacc: 0.9196428656578064, loss: 0.06697984784841537
	Epoch 4/5 Iteration 900, trainacc: 0.96875, valacc: 0.941964328289032, loss: 0.10326146334409714
	Epoch 4/5 Iteration 1000, trainacc: 0.96875, valacc: 0.9375, loss: 0.08831929415464401
	Epoch 4/5 Iteration 1100, trainacc: 0.96875, valacc: 0.9285714030265808, loss: 0.17388474941253662
	Epoch 5/5 Iteration 1200, trainacc: 1.0, valacc: 0.9508929252624512, loss: 0.0161976870149374
	Epoch 5/5 Iteration 1300, trainacc: 0.96875, valacc: 0.946428656578064, loss: 0.14780263602733612
	Epoch 5/5 Iteration 1400, trainacc: 1.0, valacc: 0.9553571939468384, loss: 0.014909764751791954

Lowering the learning rate works wonders, looks like the model wasn't underfitting before.

Result look better than with batchnorm.

Run model with batchnorm only



In [14]:

    
tf.reset_default_graph()

with tf.Session() as sess:
    m = TransferModel()
    
    m.build(input_size=7 * 7 * 512, num_hidden=1, hidden_layer_size=256, use_batchnorm=True, use_dropout=False)
    
    m.run_training(
        sess=sess, num_epochs=5, learning_rate=0.01, keep_prob=1, batch_size=64,
        fn_get_batches=lambda batch_size: get_batches(training_x, training_y),
        validation_images=validation_x, validation_labels=validation_y)









    



	Epoch 1/5 Iteration 100, trainacc: 1.0, valacc: 0.9196428656578064, loss: 0.04793165251612663
	Epoch 1/5 Iteration 200, trainacc: 1.0, valacc: 0.9464285969734192, loss: 0.08547316491603851
	Epoch 2/5 Iteration 300, trainacc: 0.96875, valacc: 0.910714328289032, loss: 0.03894037380814552
	Epoch 2/5 Iteration 400, trainacc: 1.0, valacc: 0.9330357313156128, loss: 0.011972760781645775
	Epoch 2/5 Iteration 500, trainacc: 0.9375, valacc: 0.9196428656578064, loss: 0.0832422599196434
	Epoch 3/5 Iteration 600, trainacc: 1.0, valacc: 0.9107142686843872, loss: 0.00015778496162965894
	Epoch 3/5 Iteration 700, trainacc: 1.0, valacc: 0.9375, loss: 0.0016327091725543141
	Epoch 3/5 Iteration 800, trainacc: 1.0, valacc: 0.9598215222358704, loss: 0.002206772565841675
	Epoch 4/5 Iteration 900, trainacc: 1.0, valacc: 0.941964328289032, loss: 0.023371178656816483
	Epoch 4/5 Iteration 1000, trainacc: 1.0, valacc: 0.9375, loss: 0.0020532466005533934
	Epoch 4/5 Iteration 1100, trainacc: 1.0, valacc: 0.941964328289032, loss: 0.000919074285775423
	Epoch 5/5 Iteration 1200, trainacc: 1.0, valacc: 0.9285714626312256, loss: 0.00040314358193427324
	Epoch 5/5 Iteration 1300, trainacc: 1.0, valacc: 0.9017857909202576, loss: 0.0025369408540427685
	Epoch 5/5 Iteration 1400, trainacc: 1.0, valacc: 0.9151785373687744, loss: 0.005326876416802406

Works nearly as good as batchnorm + dropout