Evolving Convnets to Classify Labeled Depths


In [ ]:
%matplotlib inline
from __future__ import print_function
import gc
import ipywidgets
import math
import os
import random
import sys

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt

from IPython.display import Image
from scipy import ndimage
from scipy.misc import imsave
from six.moves import cPickle as pickle

import outputer
import improc
import convnet
import mutate
import convevo
import darwin

In [ ]:
# For use during development
from imp import reload

reload (improc)
reload (convnet)
reload (mutate)
reload (convevo)
reload (darwin)

Enumerate Images

Image names are sequential, so add every tenth image to the validation set based on filename.


In [ ]:
training, test = improc.enumerate_images("captures")

print("Training:", len(training), "Test:", len(test))
print(training[:2])
print(test[:2])

Image Processing

Each image file contains a color image (top half), and an encoded depth image (bottom half)

  • Note: The image may also contain the orientation data. If so it is encoded in the first two pixels of the depth image. If the first pixel of the depth image is red, the second has the x, y, z, w quaternion components encoded in the r,g,b,a values.

The improc module contains functions for splitting the image, decoding the depth back into floating point millimeters, and for filling in gaps.

Image processing examples:


In [ ]:
example_image, example_depth, example_attitude = improc.load_image("testing/IMG_2114.PNG")
plt.imshow(example_image)
print(example_image.shape, example_image.dtype)

In [ ]:
plt.imshow(example_depth)
print(example_depth.shape, example_depth.dtype)
print(example_attitude)

In [ ]:
example_lab = improc.rgb2lab_normalized(example_image)
plt.imshow(example_lab[:,:,0], cmap='Greys_r')

In [ ]:
plt.imshow(example_lab[:,:,1], cmap='Greys_r')

Depth Labels and Batching

Covert depth to classification labels.

Want more precision for nearby things, so use progressively expanding buckets for labels, so if smallest bucket has size s and each succesive bucket is larger by a factor F then:

improc.MAX_DEPTH == sF0 + sF1 + sF2 + ... + sFlabel count - 1

So, plug into sum of geometric series formula:

improc.MAX_DEPTH == s * (1 - Flabel count) / (1 - F)

Since there are two unknowns we can choose either the factor or the bucket size. A factor of 1.3 resulted in buckets that seemed about right.


In [ ]:
def size_for_factor(factor, buckets):
    return improc.MAX_DEPTH * (1 - factor) / (1 - factor ** buckets)

def depth_label_boundaries(factor, buckets):
    boundaries = []
    size_sum = 0
    bucket_size = size_for_factor(factor, buckets)
    for i in range(buckets):
        size_sum += bucket_size
        boundaries.append(size_sum)
        bucket_size *= factor
    return boundaries

def boundary_midpoints(boundaries):
    midpoints = np.zeros(shape=[len(boundaries)], dtype=np.float32)
    depth = 0
    prev_boundary = 0
    for i, boundary in enumerate(DEPTH_BOUNDARIES):
        midpoints[i] = (boundary + prev_boundary) / 2
        prev_boundary = boundary
    return midpoints

DEPTH_LABEL_COUNT = 40
DEPTH_BUCKET_SCALE_FACTOR = 1.2
DEPTH_BOUNDARIES = depth_label_boundaries(DEPTH_BUCKET_SCALE_FACTOR, DEPTH_LABEL_COUNT)
DEPTH_BOUNDARY_MIDPOINTS = boundary_midpoints(DEPTH_BOUNDARIES)

def depth_label_index(depth):
    for i, boundary in enumerate(DEPTH_BOUNDARIES):
        if depth < boundary:
            return i
    return DEPTH_LABEL_COUNT - 1

def depth_label(depth, labels=None):
    if labels is None:
        labels = np.zeros(shape=(DEPTH_LABEL_COUNT + 1), dtype=np.float32)
    labels[depth_label_index(depth)] = 1
    labels[DEPTH_LABEL_COUNT] = depth / improc.MAX_DEPTH
    return labels

def depth_for_label(labels):
    depth = 0
    prev_boundary = 0
    for label, boundary in zip(labels, DEPTH_BOUNDARIES):
        boundary_midpoint = (boundary + prev_boundary) / 2
        depth += boundary_midpoint * label
        prev_boundary = boundary
    return depth

def depth_for_label_normalized(labels):
    return depth_for_label(labels) / improc.MAX_DEPTH

def depths_for_labels(labels):
    return labels * DEPTH_BOUNDARY_MIDPOINTS

def depths_for_labels_normalized(labels):
    return np.sum(depths_for_labels(labels) / improc.MAX_DEPTH, axis=1)

def depth_label_image(depths):
    labeled = depths.copy()
    for y in range(depths.shape[0]):
        for x in range(depths.shape[1]):
            labeled[y,x] = depth_label_index(depths[y,x])
    return labeled

# Precomputed via improc.compute_mean_depth(training)
# Actually it should 1680.24, value below is actually the mean of the image means.
# Keeping this value as it was what was used in the experiments to date,
# and it is close to the correct value.
MEAN_DEPTH = np.float32(1688.97)

In [ ]:
print(DEPTH_BOUNDARIES[:5])
print("Mean depth label:", depth_label(MEAN_DEPTH), np.argmax(depth_label(MEAN_DEPTH)))
print("Zero depth label:", depth_label(0)[0], depth_label(0)[-1])
print("Max depth label:", depth_label(improc.MAX_DEPTH)[-2:])
roundtrip_mean = depth_for_label(depth_label(MEAN_DEPTH))
print("Roundtrip mean depth:", roundtrip_mean, np.argmax(depth_label(roundtrip_mean)))

In [ ]:
# Set up cache directory.
depth_image_cache_path = outputer.setup_directory("temp", "cache")

def linear_order(height_span, width_span):
    pixel_indices = []
    for y in range(height_span):
        for x in range(width_span):
            pixel_indices.append((y, x))
    return pixel_indices

class ImageSampler(object):
    """Wrap an image for sampling."""
    def __init__(self, image_file,
                 sample_height, sample_width,
                 half_valid_check=2, tolerance=0):
        # Process the image or grab it from the cache.
        # image is normalized CIELAB, depth is not normalized.
        self.image, self.depth = improc.process_cached(depth_image_cache_path, image_file)
        self.index = 0
        self.pixel_index = (0, 0)
        self.sample_height = sample_height
        self.sample_width = sample_width
        self.depth_offset_y = (sample_height + 1) // 2
        self.depth_offset_x = (sample_width + 1) // 2
        self.height = self.image.shape[0]        
        self.width = self.image.shape[1]
        self.half_valid_check = half_valid_check
        self.tolerance = tolerance
        
    def depth_value(self, y, x):
        return self.depth[y + self.depth_offset_y, x + self.depth_offset_x]
        
    def sample(self, inputs, labels, index):
        self.sample_at(self.pixel_index, inputs, labels, index)
        self.advance()
        
    def sample_at(self, pixel, inputs, labels, index):
        y, x = pixel
        patch = self.image[y : y + self.sample_height, x : x + self.sample_width]
        inputs[index] = patch
        depth = self.depth_value(y, x)
        if np.isnan(depth):
            return False
        depth_label(depth, labels[index])
        return True
    
    def setup_sample_order(self, sample_orders, entropy):
        height_span = self.height - self.sample_height
        width_span = self.width - self.sample_width
        cached = sample_orders.get((height_span, width_span))
        if cached:
            return cached

        pixel_indices = linear_order(height_span, width_span)
        mutate.fisher_yates_shuffle(pixel_indices, entropy)
        sample_orders[(height_span, width_span)] = pixel_indices
        return pixel_indices
        
    def advance(self):
        self.index += 1
    
    def next_sample(self, sample_orders, entropy):
        c = self.half_valid_check
        order = self.setup_sample_order(sample_orders, entropy)
        while self.index < len(order):
            self.pixel_index = order[self.index]
            depth_y = self.pixel_index[0] + self.depth_offset_y
            depth_x = self.pixel_index[1] + self.depth_offset_x
            # Check that the sample is from a clean part of the image.
            sum = np.sum(np.isnan(self.depth[depth_y - c : depth_y + c,
                                             depth_x - c: depth_x + c]))
            if sum <= self.tolerance:
                return True
            self.advance()
        return False

In [ ]:
class BatchSampler(object):
    """Created sample batches for a set of image files"""
    def __init__(self, image_files, sample_height, sample_width, samplers_count=100):
        self.files = image_files
        self.samplers_count = samplers_count
        self.sample_height = sample_height
        self.sample_width = sample_width
        self.sample_orders = {}
        self.reset()
    
    # Access or initialize the specified sampler.
    def sampler(self, index, entropy):
        sampler = self.samplers[index]
        if sampler and not sampler.next_sample(self.sample_orders, entropy):
            sampler = None

        while sampler is None:
            path = self.files[self.file_index]
            sampler = ImageSampler(path, self.sample_height, self.sample_width)
            self.file_index = (self.file_index + 1) % len(self.files)
            if not sampler.next_sample(self.sample_orders, entropy):
                sampler = None
                print ("No samples in", path)
            else:
                self.samplers[index] = sampler
        return sampler
    
    # Get the next single sample.
    def sample(self, inputs, labels, index, entropy):
        sampler = self.sampler(self.sample_index, entropy)

        self.sample_index = (self.sample_index + 1) % len(self.samplers)
        sampler.sample(inputs, labels, index)
    
    # Get the next batch of samples.
    def sample_batch(self, inputs, labels, batch_size, entropy):
        labels.fill(0)
        for b in range(batch_size):
            self.sample(inputs, labels, b, entropy)
            
    def reset(self):
        self.sample_index = 0
        self.file_index = 0
        self.samplers = [None] * self.samplers_count
    
    # Force load all the samplers.
    def fill_and_pickle(self, path, entropy):
        for i in range(self.samplers_count):
            sampler = self.sampler(i, entropy)

        try:
            with open(path, 'wb') as f:
                pickle.dump(self, f, pickle.HIGHEST_PROTOCOL)
        except Exception as e:
            print('Unable to save data to', path, ':', e)
            raise

Depth label and batching examples


In [ ]:
plt.imshow(depth_label_image(example_depth))

In [ ]:
del example_image
del example_depth
del example_lab
gc.collect()

In [ ]:
SAMPLE_SIZE = 101
batcher = BatchSampler(["testing/IMG_2114.PNG", "testing/IMG_3410.PNG"],
                       SAMPLE_SIZE, SAMPLE_SIZE, 2)

In [ ]:
BATCH_SIZE = 100

inputs = np.ones(shape=(BATCH_SIZE, SAMPLE_SIZE, SAMPLE_SIZE, improc.COLOR_CHANNELS),
                 dtype=np.float32)
labels = np.zeros(shape=(BATCH_SIZE, DEPTH_LABEL_COUNT + 1), dtype=np.float32)

for _ in range(100):
    batcher.sample_batch(inputs, labels, BATCH_SIZE, random.Random(42))

In [ ]:
plt.imshow(inputs[1,:,:,0], cmap='Greys_r')
print(inputs[1].shape)
print(labels[1])

Data Management


In [ ]:
data_files = {
    "image_size": (101, 101, improc.COLOR_CHANNELS),
    "depth_labels": DEPTH_LABEL_COUNT,
    "train_files": np.array(training),
    "test_files": np.array(sorted(test))
}

del training
del test

In [ ]:
def setup_cross_validation(
    data,
    train_count, valid_count, test_count=None,
    label_count=None, entropy=random
):
    """Shuffle the data and split off training, validation and test sets."""
    cross_data = data.copy()
    
    if label_count:
        cross_data["depth_labels"] = label_count

    paths = cross_data["train_files"][:]
    mutate.fisher_yates_shuffle(paths, entropy)

    cross_data["train_files"] = paths[:train_count]
    cross_data["valid_files"] = paths[train_count:train_count + valid_count]

    if test_count is not None:
        cross_data["test_files"] = data["test_files"][:test_count]

    return cross_data

Batcher Caching

The evolutionary process will involve running many graphs with the same data. To make this as efficent as possible, these are used cache and restore the processed batch data.


In [ ]:
def pickle_batch(data, set_name, samplers, entropy):
    path = os.path.join("temp", set_name + ".pickle")
    files = data[set_name + "_files"]
    image_size = data["image_size"]
    batcher = BatchSampler(files, image_size[0], image_size[1], samplers)
    batcher.fill_and_pickle(path, entropy)
    del batcher
    gc.collect()
    return path

def load_batcher(pickle_batches, set_name):
    if pickle_batches:
        path = pickle_batches.get(set_name)
        if path:
            with open(path, 'rb') as f:
                return pickle.load(f)
    return None

Data Management examples


In [ ]:
pickle_data = setup_cross_validation(
    data_files, 0, 100, None,
    label_count=DEPTH_LABEL_COUNT, entropy=random.Random(24601)
)
pickle_size = pickle_data["image_size"]
pickle_files = pickle_data["valid_files"]
pickle_sampler = BatchSampler(pickle_files,pickle_size[0],pickle_size[1],len(pickle_files))

In [ ]:
pickle_sampler.fill_and_pickle("temp/depth_valid.pickle", random)

In [ ]:
with open("temp/depth_valid.pickle", 'rb') as f:
    loaded_sampler = pickle.load(f)

In [ ]:
BATCH_SIZE = 100

inputs = np.ones(shape=(BATCH_SIZE, pickle_size[0], pickle_size[1], improc.COLOR_CHANNELS),
                 dtype=np.float32)
labels = np.zeros(shape=(BATCH_SIZE, DEPTH_LABEL_COUNT + 1), dtype=np.float32)

for _ in range(500):
    loaded_sampler.sample_batch(inputs, labels, BATCH_SIZE, random.Random(42))

In [ ]:
del pickle_data
del pickle_files
del pickle_sampler
del loaded_sampler
gc.collect()

Graph Setup


In [ ]:
def batch_input_shape(batch_size, image_shape):
    return (batch_size,) + image_shape

def batch_output_shape(batch_size, label_count):
    return (batch_size, label_count + 1)

def setup_graph(
    batch_size,
    image_shape,
    label_count,
    regress_factor,
    stack
):
    graph = tf.Graph()
    with graph.as_default():
        input_shape = batch_input_shape(batch_size, image_shape)
        output_shape = batch_output_shape(batch_size, label_count)
        train   = tf.placeholder(tf.float32, shape=input_shape)
        targets = tf.placeholder(tf.float32, shape=output_shape)
        verify  = tf.placeholder(tf.float32, shape=input_shape)

        operations = stack.construct(input_shape)
        l2_loss = convnet.setup(operations)

        result = convnet.connect_model(train, operations, True)[-1]
        
        depth_label = tf.slice(targets, [0, label_count], [batch_size, 1])
        depths      = tf.slice(result, [0, label_count], [batch_size, 1])
        labels = tf.slice(targets, [0, 0], [batch_size, label_count])
        logits = tf.slice(result, [0, 0], [batch_size, label_count])

        loss = l2_loss
        if regress_factor >= 0:
            loss += tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, labels))
        else:
            regress_factor = -regress_factor

        if regress_factor > 0:
            loss += regress_factor * tf.reduce_mean(
                tf.squared_difference(depths, depth_label)
            )

        verify_result = convnet.connect_model(verify,  operations, False)[-1]
        verify_logits = tf.slice(verify_result, [0, 0], [batch_size, label_count])
        verify_depths = tf.slice(verify_result, [0, label_count], [batch_size, 1])

        verify_depths = tf.maximum(verify_depths, 0)
        verify_depths = tf.minimum(verify_depths, 1)
        
        info = {
            "graph": graph,
            "batch_size": batch_size,
            "train": train,
            "targets": targets,
            "depths": depths,
            "loss": loss,
            "optimizer": stack.construct_optimizer(loss),
            "predictions": tf.nn.softmax(logits),
            "verify": verify,
            "verify_predictions": tf.nn.softmax(verify_logits),
            "verify_depths": verify_depths,
            "saver": tf.train.Saver()
        }
    return info

Graph Execution


In [ ]:
def accuracy(predictions, labels):
    correct_predictions = np.argmax(predictions, 1) == np.argmax(labels, 1)
    return (100.0 * np.sum(correct_predictions) / predictions.shape[0])

In [ ]:
def mean_depth_error(depths, labels):
    return np.mean(np.absolute(depths[:,0] - labels[:,-1]))

In [ ]:
def score_result(loss, predictions, depths, labels):
    return (loss, accuracy(predictions, labels[:,0:-1]), mean_depth_error(depths, labels))

In [ ]:
def print_batch_info(
    context, score, predictions, depths, labels, verbose, print_count=20, depth_print=10
):
    print(context, "accuracy: %.1f%%" % score[1])
    if verbose:
        print(np.argmax(predictions[0:print_count],1))
        print(np.argmax(labels[0:print_count,0:-1],1))
        print(context, "average depth error:", score[2])
        print(depths[0:depth_print,0])
        print(labels[0:depth_print,-1])

In [ ]:
def batch_accuracy(
    context, session, graph, batcher, entropy, inputs, labels, batch_size, count, verbose
):
    total_accuracy = 0
    total_depth = 0
    for b in range(count):
        batcher.sample_batch(inputs, labels, batch_size, entropy)
        targets = [graph["verify_predictions"], graph["verify_depths"]]
        predictions, depths = session.run(targets, feed_dict={graph["verify"] : inputs})
        total_accuracy += accuracy(predictions, labels) / float(count)
        total_depth += mean_depth_error(depths, labels) / float(count)
    
    score = (0, total_accuracy, total_depth)
    print_batch_info(context, score, predictions, depths, labels, verbose)
    return score

In [ ]:
def run_graph(
    graph_info,
    data,
    step_count,
    valid_count,
    test_count=0,
    batch_sampler_count=1000,
    report_every=50,
    verbose=True,
    accuracy_minimum=None, # Minimimum validation percent accuracy for early abort
    pickle_batches=None, # pickle files for training and validation batchers
    tracker=None,
    entropy=random
):
    with tf.Session(graph=graph_info["graph"]) as session:
        tf.initialize_all_variables().run()
        print("Initialized")

        # Optionally restore graph parameters from disk.
        convnet.restore_model(graph_info, session)

        # Set up space for graph inputs / feed values
        batch_size = graph_info["batch_size"]
        depth_labels = data["depth_labels"]
        height, width, _ = data["image_size"]
        inputs = np.zeros(shape=batch_input_shape(batch_size, data["image_size"]),
                          dtype=np.float32)
        labels = np.zeros(shape=batch_output_shape(batch_size, depth_labels),
                          dtype=np.float32)

        # Construct or unpickle training batcher.
        train_batcher = load_batcher(pickle_batches, "train")
        if not train_batcher:
            train_batcher = BatchSampler(
                data["train_files"], height, width, batch_sampler_count
            )

        score = (0,1)

        try:
            for step in range(step_count + 1):
                if tracker:
                    tracker.update_progress(step)

                # Generate a batch
                train_batcher.sample_batch(inputs, labels, batch_size, entropy)
                
                # Graph targets
                run_targets = [
                    graph_info["optimizer"],
                    graph_info["loss"],
                    graph_info["predictions"],
                    graph_info["depths"]
                ]
                
                # Graph inputs:
                feed_dict = {graph_info["train"] : inputs, graph_info["targets"] : labels}
                _, loss, predictions, depths = session.run(run_targets,feed_dict=feed_dict)
                
                # Keep track of and possibly display score.
                batch_score = score_result(loss, predictions, depths, labels)
                if tracker:
                    tracker.record_score(batch_score)

                if np.isnan(loss):
                    print("Error computing loss at step", step)
                    print_batch_info("Minibatch", batch_score, predictions,
                                     depths, labels, True)
                    return 0
                if (step % report_every == 0):
                    if verbose:
                        print("Minibatch loss at step", step, ":", loss)
                        print_batch_info("Minibatch", batch_score, predictions,
                                         depths, labels, True)

                    # Evaluate the validation data.
                    valid_batcher = load_batcher(pickle_batches, "valid")
                    if not valid_batcher:
                        valid_files = data["valid_files"]
                        valid_batcher = BatchSampler(
                            valid_files, height, width, len(valid_files)
                        )
                    valid_score = batch_accuracy(
                        "Validation", session, graph_info, valid_batcher, entropy,
                        inputs, labels, batch_size, valid_count, verbose
                    )
                    del valid_batcher
                    score = valid_score[1:]
                    if accuracy_minimum and step > 0 and valid_score[1] < accuracy_minimum:
                        print("Early out.")
                        break

            # Evaluate the test data, if any.
            if test_count > 0:
                test_batcher = BatchSampler(data["test_files"], height, width)
                valid_accuracy = batch_accuracy(
                    "Test", session, graph_info, test_batcher, entropy,
                    inputs, labels, batch_size, test_count, verbose
                )

            return score
        finally:
            # Optionally save out graph parameters to disk.
            convnet.save_model(graph_info, session)

In [ ]:
def valid_accuracy_metric(valid_accuracy, valid_depth_error, train_results):
    return valid_accuracy

def valid_error_metric(valid_accuracy, valid_depth_error, train_results):
    return valid_depth_error

def train_accuracy_metric(valid_accuracy, valid_depth_error, train_results):
    result_count = min(len(train_results), 1000)
    return sum(accuracy for _, accuracy, _ in train_results[-result_count:]) / result_count

def train_depth_error_metric(valid_accuracy, valid_depth_error, train_results):
    result_count = min(len(train_results), 1000)
    error = sum(error for _, _, error in train_results[-result_count:]) / result_count
    return max(0, 1 - error)

In [ ]:
results_path = outputer.setup_directory("temp", "classy_results")

In [ ]:
def make_eval(
    batch_size=20,
    eval_steps=10000,
    valid_steps=500,
    regress_factor=1.0,
    report_every=None,
    reuse_cross=False,
    metric=valid_accuracy_metric,
    entropy=random
):
    pickle_batches = {}
    train_count = 9700
    valid_count = 400
    batch_sampler_count = min(801, eval_steps * batch_size)
    test_count = None
    
    #if reusing data, set up training and test data, and pickle batchers for efficiency.
    if reuse_cross:
        redata = setup_cross_validation(
            data_files, train_count, valid_count, test_count,
            label_count=DEPTH_LABEL_COUNT, entropy=entropy
        )
        pickle_batches["valid"] = pickle_batch(
            redata, "valid", len(redata["valid_files"]), entropy
        )
        print("Pickled Validation")
        pickle_batches["train"] = pickle_batch(
            redata, "train", batch_sampler_count, entropy
        )
        print("Pickled Training")
        
    progress_tracker = outputer.ProgressTracker(
        ["Loss", "Accuracy", "Error"], eval_steps, results_path, convevo.serialize
    )
  
    def evaluate(stack, eval_entropy):
        # If not reusing data, generate training and validation sets
        if not reuse_cross:
            data = setup_cross_validation(
                data_files, train_count, valid_count, test_count,
                label_count=DEPTH_LABEL_COUNT, entropy=eval_entropy
            )
            pickle_batches["valid"] = pickle_batch(
                data, "valid", len(data["valid_files"]), eval_entropy
            )
            print("Pickled Validation")
        else:
            data = redata
            
        progress_tracker.setup_eval(stack)

        # Set up the Tensorflow graph
        try:
            graph_info = setup_graph(
                batch_size,
                data["image_size"],
                data["depth_labels"],
                regress_factor,
                stack
            )
        except KeyboardInterrupt:
            raise
        except:
            progress_tracker.error(sys.exc_info())
            return -10
       
        progress_tracker.start_eval(graph_info)
    
        # Run the graph
        try:
            valid_accuracy, valid_depth_error = run_graph(
                graph_info,
                data,
                eval_steps,
                valid_count=valid_steps,
                batch_sampler_count=batch_sampler_count,
                report_every=report_every if report_every else eval_steps//4,
                verbose=True,
                accuracy_minimum=None,
                pickle_batches=pickle_batches,
                tracker=progress_tracker,
                entropy=eval_entropy
            )
            if metric:
                return metric(valid_accuracy, valid_depth_error, progress_tracker.results)
            return valid_accuracy
        except KeyboardInterrupt:
            raise
        except:
            progress_tracker.error(sys.exc_info())
            return -1
        finally:
            progress_tracker.output()
    return evaluate

Test of components in isoloation


In [ ]:
cross_data = setup_cross_validation(data_files,9700,400,1000,label_count=DEPTH_LABEL_COUNT)

In [ ]:
batch_size = 20
conv_layers = [
    ("conv_bias", 20, 2, 10, "SAME", True),
    ("conv_bias", 10, 5, 20, "SAME", True),
    ("conv_bias",  5, 2, 40, "SAME", True)
]
hidden_sizes = [400, 100, cross_data["depth_labels"] + 1]
optimizer = convevo.Optimizer("GradientDescent", 0.01)
optimizer.default_parameters()
prototype = convevo.create_stack(conv_layers,[],True,hidden_sizes,0.0, 0.05, 0.0,optimizer)
prototype.reseed(random.Random(42))

In [ ]:
prototype_graph = setup_graph(
    batch_size,
    cross_data["image_size"],
    cross_data["depth_labels"],
    1.0,
    prototype
)

In [ ]:
run_graph(
    prototype_graph, cross_data, 1000,
    valid_count=200, report_every=500, verbose=True, entropy=random.Random(42)
)

In [ ]:
print(convevo.serialize(prototype))
prototype_entropy = random.Random(42)
prototype_eval = make_eval(
    batch_size=100,
    eval_steps=100,
    valid_steps=20,
    regress_factor=1.0,
    reuse_cross=True,
    entropy=prototype_entropy
)
prototype_eval(prototype, prototype_entropy)

In [ ]:
del cross_data
del conv_layers
del hidden_sizes
del prototype_graph
del prototype_eval
gc.collect()

Evolving Convnets


In [ ]:
prototypes = [prototype]

In [ ]:
population,_,_ = convevo.load_population("testing/color_quad_run.xml", False)
prototypes = population[:5]
print(len(prototypes))

In [ ]:
prototypes = [
    convevo.load_stack("testing/candidate1.xml"),
    convevo.load_stack("testing/candidate2.xml"),
    convevo.load_stack("testing/candidate3.xml"),
    convevo.load_stack("testing/candidate4.xml"),
    convevo.load_stack("testing/candidate5.xml")
]

In [ ]:
with outputer.TeeOutput(os.path.join("temp", outputer.timestamp("Depth_Evolve_", "txt"))):
    mutate_seed = random.randint(1, 100000)
    print("Mutate Seed:", mutate_seed)
    mutate_entropy = random.Random(mutate_seed)
    eval_seed = random.randint(1, 100000)
    print("Eval Seed:", eval_seed)
    eval_entropy = random.Random(eval_seed)

    population_size = 10
    generations = 5
    batch_size = 100

    breed_options = {
        "input_shape": batch_input_shape(batch_size, data_files["image_size"]),
        "output_shape": batch_output_shape(batch_size, data_files["depth_labels"])
    }

    for stack in prototypes:
        stack.make_safe(breed_options["input_shape"], breed_options["output_shape"])

    evaluator = make_eval(
        batch_size=batch_size, eval_steps=40000, valid_steps=1000, regress_factor=1.0,
        reuse_cross=True, metric=None, entropy=eval_entropy
    )
    charles = darwin.Darwin(convevo.serialize, evaluator, convevo.breed)
    charles.init_population(prototypes, population_size, False,
                            breed_options, mutate_entropy)

    for g in range(generations):
        print("Generation", g)
        results = charles.evaluate(eval_entropy)
        convevo.output_results(results, "temp", outputer.timestamp() + ".xml",
                               mutate_seed, eval_seed)
        charles.repopulate(population_size, 0.3, 3, results, breed_options, mutate_entropy)

In [ ]:
results = darwin.descending_score(charles.history.values())
convevo.output_results(results, "testing", "candidates_evolve_run.xml",
                       mutate_seed, eval_seed)
len(results)

Candidate Evaluation

Do a long training run for the best graph to date. Note: on my GPU accelerated machine, this takes 5 days to run.


In [ ]:
BATCH_SIZE = 100
candidate = convevo.load_stack("testing/candidate6.xml")
candidate.make_safe(
    batch_input_shape(BATCH_SIZE, data_files["image_size"]),
    batch_output_shape(BATCH_SIZE, data_files["depth_labels"])
)
print(convevo.serialize(candidate))

In [ ]:
candidate_evaluator = make_eval(
    batch_size=BATCH_SIZE,
    eval_steps=10000000,
    valid_steps=100000,
    regress_factor=1.0,
    report_every=500000,
    reuse_cross=False,
    metric=None,
    entropy=random.Random(42)
)

In [ ]:
with outputer.TeeOutput(os.path.join("temp", "candidate6_results.txt")):
    candidate_evaluator(candidate, random.Random(57))

Test reloading the resulting graph for additional training/validation.


In [ ]:
with outputer.TeeOutput(os.path.join("temp", "candidate6_retest.txt")):
    candidate = convevo.load_stack("testing/candidate6.xml")
    candidate_reevaluator = make_eval(
        batch_size=100, eval_steps=10000, valid_steps=10000, regress_factor=1.0,
        reuse_cross=False, metric=None, entropy=random.Random(42)
    )
    candidate.checkpoint_path("testing/candidate6/full/2016-06-11~15_23_44_712.ckpt")
    candidate_reevaluator(candidate, random.Random(42))

Candidate Testing

Calculates for non-NaN pixels:

  • accuracy score,
  • mean depth error for the predicted depth,
  • mean depth error for the softmax predicted label converted to a depth via:
    • sum(midpoint of bucket * softmax value for bucket).
  • mean depth error for the bucket midpoint corresponding to the argmax of the predicted label

In [ ]:
def test_score(labels, predictions, depths, count):
    is_finite = np.isfinite(labels[:count,-1])
    where_valid = np.where(is_finite)
    count = np.count_nonzero(is_finite)
    if count:
        score = accuracy(predictions[where_valid], labels[where_valid])
        error = mean_depth_error(depths[where_valid], labels[where_valid])
        
        valid_predictions = predictions[where_valid]
        label_depths = depths_for_labels_normalized(valid_predictions)
        label_error = mean_depth_error(label_depths[:,np.newaxis], labels[where_valid])

        argmax_predictions = np.argmax(valid_predictions, axis=1)
        argmax_depths = DEPTH_BOUNDARY_MIDPOINTS[argmax_predictions] / improc.MAX_DEPTH
        argmax_error = mean_depth_error(argmax_depths[:,np.newaxis], labels[where_valid])
        
        return score*count, error*count, label_error*count, argmax_error*count, count
    return 0, 0, 0, 0, 0

In [ ]:
# Validate the test_score function.
def check_test_score():
    test_batch_size = 10
    test_labels = np.zeros(shape=batch_output_shape(test_batch_size, DEPTH_LABEL_COUNT),
                           dtype=np.float32)
    test_depths = np.zeros(shape=(test_batch_size,1), dtype=np.float32)
    for l in range(test_batch_size):
        test_depth = improc.MAX_DEPTH * l / float(test_batch_size)
        depth_label(test_depth, test_labels[l])
        test_depths[l, 0] = test_labels[l,-1]

    test_predictions = np.copy(test_labels)[:,:-1]
    test_predictions[0, 10] = 0.5

    test_labels[2] = np.nan

    score = test_score(test_labels, test_predictions, test_depths, 7)
    print(score)
    print([s / score[-1] for s in score[:-1]])
check_test_score()

For all the test images in the provided data set, compute metrics for full images, and generate the corresponding image for the output depth and either the linear combination of the labeled softmax depth output, or the argmax labeled depth output.


In [ ]:
def compute_test_images(graph_info, data, output_path):
    with tf.Session(graph=graph_info["graph"]) as session:
        tf.initialize_all_variables().run()
        print("Initialized")

        # restore graph parameters from disk.
        convnet.restore_model(graph_info, session)
        
        # Set up space for graph inputs / feed values
        batch_size = graph_info["batch_size"]
        depth_labels = data["depth_labels"]
        image_size = data["image_size"]
        inputs = np.zeros(shape=batch_input_shape(batch_size, image_size),
                          dtype=np.float32)
        labels = np.zeros(shape=batch_output_shape(batch_size, depth_labels),
                          dtype=np.float32)
        nan_label = np.array([np.nan]*labels.shape[-1], dtype=np.float32)
        
        source_image_size = (480, 640)
        height_span = source_image_size[0] - image_size[0]
        width_span = source_image_size[1] - image_size[1]
        pixel_order = np.array(linear_order(height_span, width_span))
        
        files = data["test_files"]
        
        eval_count = len(files) * len(pixel_order) // batch_size
        progress = outputer.show_progress("Evaluation Steps:", eval_count)
        
        eval_count = 0;
        all_scores = {}

        for image_path in files:
            sampler = ImageSampler(image_path, image_size[0], image_size[1])
            if output_path:
                raw_depths = ndimage.imread(image_path)
                label_depths = np.copy(raw_depths)
                argmax_depths = np.copy(raw_depths)
            image_scores = np.zeros(shape=(5,), dtype=np.float32)
            gc.collect()
            
            for row in range(height_span):
                # Update progress
                eval_count += 1
                progress.value = eval_count
                
                # Generate a batch and run the graph
                batch_pixels = pixel_order[row * width_span : (row + 1) * width_span, :]
                labels.fill(0)
                for i, pixel in enumerate(batch_pixels):
                    if not sampler.sample_at(pixel, inputs, labels, i):
                        labels[i] = nan_label

                targets = [graph_info["verify_predictions"],
                           graph_info["verify_depths"]]
                predictions, depths = session.run(
                    targets, feed_dict={graph_info["verify"] : inputs}
                )
                
                if output_path:
                    iy = ((raw_depths.shape[0] + image_size[0]) // 2) + row
                    sx = (image_size[0] // 2)
                    ex =  sx + width_span
                    raw_depths[iy, sx : ex] = improc.encode_normalized_depths(depths)
                    label_depths[iy, sx : ex] = improc.encode_normalized_depths(
                        depths_for_labels_normalized(predictions)[:, np.newaxis]
                    )
                    argmax_predictions = np.argmax(predictions, axis=1)
                    argmax_depth_values = DEPTH_BOUNDARY_MIDPOINTS[argmax_predictions]
                    argmax_depths[iy, sx : ex] = improc.encode_normalized_depths(
                        (argmax_depth_values / improc.MAX_DEPTH)[:, np.newaxis]
                    )
                        
                image_scores += test_score(labels, predictions, depths, len(batch_pixels))
            
            image_name, ext = os.path.splitext(os.path.basename(image_path))
            all_scores[image_name] = image_scores
            print("Image scores for", image_name, image_scores[:-1] / image_scores[-1])
            if output_path:
                outputs = [
                    (raw_depths, "_depth"),
                    (label_depths, "_softmax"),
                    (argmax_depths, "_argmax")
                ]
                for image, postfix in outputs:
                    imsave(os.path.join(output_path, image_name + postfix + ".png"), image)
        return all_scores

Format the test results as a CSV file, and find the min/max for each metric.


In [ ]:
def output_test_scores(test_scores, test_data, path):
    with outputer.TeeOutput(path):
        titles = ["Name", "Accuracy", "Error", "Label Error", "Argmax Error", "Count"]
        total = np.zeros(shape=(5,), dtype=np.float32)
        lines = [titles]
        for image_path in test_data["test_files"]:
            image_name, ext = os.path.splitext(os.path.basename(image_path))
            scores = test_scores[image_name]
            total += scores
            line = [image_name]
            line.extend(scores[:-1] / scores[-1])
            line.append(scores[-1])
            lines.append(line)
        line = ["Total"]
        line.extend(total[:-1] / total[-1])
        line.append(total[-1])
        lines.append(line)

        text = "\n".join(",".join(str(v) for v in line) for line in lines)
        print(text)

        for i in range(1, 5):
            sorted_lines = sorted(lines[1:-1], key=lambda l: l[i])
            print(titles[i] + " high")
            print(",".join([str(v) for v in sorted_lines[-1]]))
            print(titles[i] + " low")
            print(",".join([str(v) for v in sorted_lines[0]]))

Constructs a simple graph that just computes the output label and depth corresponding to a constant depth value.


In [ ]:
def predict_constant_depth(batch_size, image_shape, label_count, value):
    graph = tf.Graph()
    with graph.as_default():
        verify  = tf.placeholder(tf.float32,
                                 shape=batch_input_shape(batch_size, image_shape))
        mean_label = tf.one_hot(depth_label_index(value), label_count, np.float32(1), 0)
        mean_label = tf.reshape(mean_label, (1, DEPTH_LABEL_COUNT))
        return {
            "graph": graph,
            "batch_size": batch_size,
            "verify": verify,
            "verify_predictions": tf.tile(mean_label, [batch_size, 1]),
            "verify_depths": tf.fill([batch_size, 1], value / improc.MAX_DEPTH)
        }

Compute the test scores resulting from just predicting the mean for every pixel.


In [ ]:
with outputer.TeeOutput(os.path.join("temp", "guess_mean_test.txt")):
    mean_graph = predict_constant_depth(
        100, data_files["image_size"], DEPTH_LABEL_COUNT, MEAN_DEPTH
    )
    test_data = setup_cross_validation(
        data_files, 0, 0, 1123, label_count=DEPTH_LABEL_COUNT
    )
    mean_test_scores = compute_test_images(
        mean_graph, test_data, False, None
    )

output_test_scores(mean_test_scores, test_data, "temp/mean_test_scores.csv")

Run and score the candidate graph for the full test set.


In [ ]:
def test_candidate_stack(stack_path, output_path, output_images):
    batch_size = 640 - data_files["image_size"][1]

    with outputer.TeeOutput(os.path.join(output_path, "full_test.txt")):
        candidate = convevo.load_stack(stack_path)
        test_data = setup_cross_validation(
            data_files, 0, 0, 1123, label_count=DEPTH_LABEL_COUNT
        )
        candidate_graph = setup_graph(
            batch_size, test_data["image_size"], test_data["depth_labels"], 1.0, candidate
        )
        convnet.setup_restore_model(
            candidate_graph, candidate.checkpoint_path()
        )
        test_scores = compute_test_images(
            candidate_graph, test_data, output_path if output_images else None
        )
    output_test_scores(
        test_scores, test_data, os.path.join(output_path, "full_test_scores.csv")
    )

In [ ]:
candidate6_results_path = outputer.setup_directory("temp/candidate6")
test_candidate_stack("testing/candidate6.xml", candidate6_results_path, True)

In [ ]: