In [ ]:
%matplotlib inline
from __future__ import print_function
import gc
import ipywidgets
import math
import os
import random
import sys
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
from IPython.display import Image
from scipy import ndimage
from scipy.misc import imsave
from six.moves import cPickle as pickle
import outputer
import improc
import convnet
import mutate
import convevo
import darwin
In [ ]:
# For use during development
from imp import reload
reload (improc)
reload (convnet)
reload (mutate)
reload (convevo)
reload (darwin)
In [ ]:
training, test = improc.enumerate_images("captures")
print("Training:", len(training), "Test:", len(test))
print(training[:2])
print(test[:2])
Each image file contains a color image (top half), and an encoded depth image (bottom half)
The improc module contains functions for splitting the image, decoding the depth back into floating point millimeters, and for filling in gaps.
In [ ]:
example_image, example_depth, example_attitude = improc.load_image("testing/IMG_2114.PNG")
plt.imshow(example_image)
print(example_image.shape, example_image.dtype)
In [ ]:
plt.imshow(example_depth)
print(example_depth.shape, example_depth.dtype)
print(example_attitude)
In [ ]:
example_lab = improc.rgb2lab_normalized(example_image)
plt.imshow(example_lab[:,:,0], cmap='Greys_r')
In [ ]:
plt.imshow(example_lab[:,:,1], cmap='Greys_r')
Want more precision for nearby things, so use progressively expanding buckets for labels, so if smallest bucket has size s and each succesive bucket is larger by a factor F then:
improc.MAX_DEPTH == sF0 + sF1 + sF2 + ... + sFlabel count - 1
So, plug into sum of geometric series formula:
improc.MAX_DEPTH == s * (1 - Flabel count) / (1 - F)
Since there are two unknowns we can choose either the factor or the bucket size. A factor of 1.3 resulted in buckets that seemed about right.
In [ ]:
def size_for_factor(factor, buckets):
return improc.MAX_DEPTH * (1 - factor) / (1 - factor ** buckets)
def depth_label_boundaries(factor, buckets):
boundaries = []
size_sum = 0
bucket_size = size_for_factor(factor, buckets)
for i in range(buckets):
size_sum += bucket_size
boundaries.append(size_sum)
bucket_size *= factor
return boundaries
def boundary_midpoints(boundaries):
midpoints = np.zeros(shape=[len(boundaries)], dtype=np.float32)
depth = 0
prev_boundary = 0
for i, boundary in enumerate(DEPTH_BOUNDARIES):
midpoints[i] = (boundary + prev_boundary) / 2
prev_boundary = boundary
return midpoints
DEPTH_LABEL_COUNT = 40
DEPTH_BUCKET_SCALE_FACTOR = 1.2
DEPTH_BOUNDARIES = depth_label_boundaries(DEPTH_BUCKET_SCALE_FACTOR, DEPTH_LABEL_COUNT)
DEPTH_BOUNDARY_MIDPOINTS = boundary_midpoints(DEPTH_BOUNDARIES)
def depth_label_index(depth):
for i, boundary in enumerate(DEPTH_BOUNDARIES):
if depth < boundary:
return i
return DEPTH_LABEL_COUNT - 1
def depth_label(depth, labels=None):
if labels is None:
labels = np.zeros(shape=(DEPTH_LABEL_COUNT + 1), dtype=np.float32)
labels[depth_label_index(depth)] = 1
labels[DEPTH_LABEL_COUNT] = depth / improc.MAX_DEPTH
return labels
def depth_for_label(labels):
depth = 0
prev_boundary = 0
for label, boundary in zip(labels, DEPTH_BOUNDARIES):
boundary_midpoint = (boundary + prev_boundary) / 2
depth += boundary_midpoint * label
prev_boundary = boundary
return depth
def depth_for_label_normalized(labels):
return depth_for_label(labels) / improc.MAX_DEPTH
def depths_for_labels(labels):
return labels * DEPTH_BOUNDARY_MIDPOINTS
def depths_for_labels_normalized(labels):
return np.sum(depths_for_labels(labels) / improc.MAX_DEPTH, axis=1)
def depth_label_image(depths):
labeled = depths.copy()
for y in range(depths.shape[0]):
for x in range(depths.shape[1]):
labeled[y,x] = depth_label_index(depths[y,x])
return labeled
# Precomputed via improc.compute_mean_depth(training)
# Actually it should 1680.24, value below is actually the mean of the image means.
# Keeping this value as it was what was used in the experiments to date,
# and it is close to the correct value.
MEAN_DEPTH = np.float32(1688.97)
In [ ]:
print(DEPTH_BOUNDARIES[:5])
print("Mean depth label:", depth_label(MEAN_DEPTH), np.argmax(depth_label(MEAN_DEPTH)))
print("Zero depth label:", depth_label(0)[0], depth_label(0)[-1])
print("Max depth label:", depth_label(improc.MAX_DEPTH)[-2:])
roundtrip_mean = depth_for_label(depth_label(MEAN_DEPTH))
print("Roundtrip mean depth:", roundtrip_mean, np.argmax(depth_label(roundtrip_mean)))
In [ ]:
# Set up cache directory.
depth_image_cache_path = outputer.setup_directory("temp", "cache")
def linear_order(height_span, width_span):
pixel_indices = []
for y in range(height_span):
for x in range(width_span):
pixel_indices.append((y, x))
return pixel_indices
class ImageSampler(object):
"""Wrap an image for sampling."""
def __init__(self, image_file,
sample_height, sample_width,
half_valid_check=2, tolerance=0):
# Process the image or grab it from the cache.
# image is normalized CIELAB, depth is not normalized.
self.image, self.depth = improc.process_cached(depth_image_cache_path, image_file)
self.index = 0
self.pixel_index = (0, 0)
self.sample_height = sample_height
self.sample_width = sample_width
self.depth_offset_y = (sample_height + 1) // 2
self.depth_offset_x = (sample_width + 1) // 2
self.height = self.image.shape[0]
self.width = self.image.shape[1]
self.half_valid_check = half_valid_check
self.tolerance = tolerance
def depth_value(self, y, x):
return self.depth[y + self.depth_offset_y, x + self.depth_offset_x]
def sample(self, inputs, labels, index):
self.sample_at(self.pixel_index, inputs, labels, index)
self.advance()
def sample_at(self, pixel, inputs, labels, index):
y, x = pixel
patch = self.image[y : y + self.sample_height, x : x + self.sample_width]
inputs[index] = patch
depth = self.depth_value(y, x)
if np.isnan(depth):
return False
depth_label(depth, labels[index])
return True
def setup_sample_order(self, sample_orders, entropy):
height_span = self.height - self.sample_height
width_span = self.width - self.sample_width
cached = sample_orders.get((height_span, width_span))
if cached:
return cached
pixel_indices = linear_order(height_span, width_span)
mutate.fisher_yates_shuffle(pixel_indices, entropy)
sample_orders[(height_span, width_span)] = pixel_indices
return pixel_indices
def advance(self):
self.index += 1
def next_sample(self, sample_orders, entropy):
c = self.half_valid_check
order = self.setup_sample_order(sample_orders, entropy)
while self.index < len(order):
self.pixel_index = order[self.index]
depth_y = self.pixel_index[0] + self.depth_offset_y
depth_x = self.pixel_index[1] + self.depth_offset_x
# Check that the sample is from a clean part of the image.
sum = np.sum(np.isnan(self.depth[depth_y - c : depth_y + c,
depth_x - c: depth_x + c]))
if sum <= self.tolerance:
return True
self.advance()
return False
In [ ]:
class BatchSampler(object):
"""Created sample batches for a set of image files"""
def __init__(self, image_files, sample_height, sample_width, samplers_count=100):
self.files = image_files
self.samplers_count = samplers_count
self.sample_height = sample_height
self.sample_width = sample_width
self.sample_orders = {}
self.reset()
# Access or initialize the specified sampler.
def sampler(self, index, entropy):
sampler = self.samplers[index]
if sampler and not sampler.next_sample(self.sample_orders, entropy):
sampler = None
while sampler is None:
path = self.files[self.file_index]
sampler = ImageSampler(path, self.sample_height, self.sample_width)
self.file_index = (self.file_index + 1) % len(self.files)
if not sampler.next_sample(self.sample_orders, entropy):
sampler = None
print ("No samples in", path)
else:
self.samplers[index] = sampler
return sampler
# Get the next single sample.
def sample(self, inputs, labels, index, entropy):
sampler = self.sampler(self.sample_index, entropy)
self.sample_index = (self.sample_index + 1) % len(self.samplers)
sampler.sample(inputs, labels, index)
# Get the next batch of samples.
def sample_batch(self, inputs, labels, batch_size, entropy):
labels.fill(0)
for b in range(batch_size):
self.sample(inputs, labels, b, entropy)
def reset(self):
self.sample_index = 0
self.file_index = 0
self.samplers = [None] * self.samplers_count
# Force load all the samplers.
def fill_and_pickle(self, path, entropy):
for i in range(self.samplers_count):
sampler = self.sampler(i, entropy)
try:
with open(path, 'wb') as f:
pickle.dump(self, f, pickle.HIGHEST_PROTOCOL)
except Exception as e:
print('Unable to save data to', path, ':', e)
raise
In [ ]:
plt.imshow(depth_label_image(example_depth))
In [ ]:
del example_image
del example_depth
del example_lab
gc.collect()
In [ ]:
SAMPLE_SIZE = 101
batcher = BatchSampler(["testing/IMG_2114.PNG", "testing/IMG_3410.PNG"],
SAMPLE_SIZE, SAMPLE_SIZE, 2)
In [ ]:
BATCH_SIZE = 100
inputs = np.ones(shape=(BATCH_SIZE, SAMPLE_SIZE, SAMPLE_SIZE, improc.COLOR_CHANNELS),
dtype=np.float32)
labels = np.zeros(shape=(BATCH_SIZE, DEPTH_LABEL_COUNT + 1), dtype=np.float32)
for _ in range(100):
batcher.sample_batch(inputs, labels, BATCH_SIZE, random.Random(42))
In [ ]:
plt.imshow(inputs[1,:,:,0], cmap='Greys_r')
print(inputs[1].shape)
print(labels[1])
In [ ]:
data_files = {
"image_size": (101, 101, improc.COLOR_CHANNELS),
"depth_labels": DEPTH_LABEL_COUNT,
"train_files": np.array(training),
"test_files": np.array(sorted(test))
}
del training
del test
In [ ]:
def setup_cross_validation(
data,
train_count, valid_count, test_count=None,
label_count=None, entropy=random
):
"""Shuffle the data and split off training, validation and test sets."""
cross_data = data.copy()
if label_count:
cross_data["depth_labels"] = label_count
paths = cross_data["train_files"][:]
mutate.fisher_yates_shuffle(paths, entropy)
cross_data["train_files"] = paths[:train_count]
cross_data["valid_files"] = paths[train_count:train_count + valid_count]
if test_count is not None:
cross_data["test_files"] = data["test_files"][:test_count]
return cross_data
In [ ]:
def pickle_batch(data, set_name, samplers, entropy):
path = os.path.join("temp", set_name + ".pickle")
files = data[set_name + "_files"]
image_size = data["image_size"]
batcher = BatchSampler(files, image_size[0], image_size[1], samplers)
batcher.fill_and_pickle(path, entropy)
del batcher
gc.collect()
return path
def load_batcher(pickle_batches, set_name):
if pickle_batches:
path = pickle_batches.get(set_name)
if path:
with open(path, 'rb') as f:
return pickle.load(f)
return None
In [ ]:
pickle_data = setup_cross_validation(
data_files, 0, 100, None,
label_count=DEPTH_LABEL_COUNT, entropy=random.Random(24601)
)
pickle_size = pickle_data["image_size"]
pickle_files = pickle_data["valid_files"]
pickle_sampler = BatchSampler(pickle_files,pickle_size[0],pickle_size[1],len(pickle_files))
In [ ]:
pickle_sampler.fill_and_pickle("temp/depth_valid.pickle", random)
In [ ]:
with open("temp/depth_valid.pickle", 'rb') as f:
loaded_sampler = pickle.load(f)
In [ ]:
BATCH_SIZE = 100
inputs = np.ones(shape=(BATCH_SIZE, pickle_size[0], pickle_size[1], improc.COLOR_CHANNELS),
dtype=np.float32)
labels = np.zeros(shape=(BATCH_SIZE, DEPTH_LABEL_COUNT + 1), dtype=np.float32)
for _ in range(500):
loaded_sampler.sample_batch(inputs, labels, BATCH_SIZE, random.Random(42))
In [ ]:
del pickle_data
del pickle_files
del pickle_sampler
del loaded_sampler
gc.collect()
In [ ]:
def batch_input_shape(batch_size, image_shape):
return (batch_size,) + image_shape
def batch_output_shape(batch_size, label_count):
return (batch_size, label_count + 1)
def setup_graph(
batch_size,
image_shape,
label_count,
regress_factor,
stack
):
graph = tf.Graph()
with graph.as_default():
input_shape = batch_input_shape(batch_size, image_shape)
output_shape = batch_output_shape(batch_size, label_count)
train = tf.placeholder(tf.float32, shape=input_shape)
targets = tf.placeholder(tf.float32, shape=output_shape)
verify = tf.placeholder(tf.float32, shape=input_shape)
operations = stack.construct(input_shape)
l2_loss = convnet.setup(operations)
result = convnet.connect_model(train, operations, True)[-1]
depth_label = tf.slice(targets, [0, label_count], [batch_size, 1])
depths = tf.slice(result, [0, label_count], [batch_size, 1])
labels = tf.slice(targets, [0, 0], [batch_size, label_count])
logits = tf.slice(result, [0, 0], [batch_size, label_count])
loss = l2_loss
if regress_factor >= 0:
loss += tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, labels))
else:
regress_factor = -regress_factor
if regress_factor > 0:
loss += regress_factor * tf.reduce_mean(
tf.squared_difference(depths, depth_label)
)
verify_result = convnet.connect_model(verify, operations, False)[-1]
verify_logits = tf.slice(verify_result, [0, 0], [batch_size, label_count])
verify_depths = tf.slice(verify_result, [0, label_count], [batch_size, 1])
verify_depths = tf.maximum(verify_depths, 0)
verify_depths = tf.minimum(verify_depths, 1)
info = {
"graph": graph,
"batch_size": batch_size,
"train": train,
"targets": targets,
"depths": depths,
"loss": loss,
"optimizer": stack.construct_optimizer(loss),
"predictions": tf.nn.softmax(logits),
"verify": verify,
"verify_predictions": tf.nn.softmax(verify_logits),
"verify_depths": verify_depths,
"saver": tf.train.Saver()
}
return info
In [ ]:
def accuracy(predictions, labels):
correct_predictions = np.argmax(predictions, 1) == np.argmax(labels, 1)
return (100.0 * np.sum(correct_predictions) / predictions.shape[0])
In [ ]:
def mean_depth_error(depths, labels):
return np.mean(np.absolute(depths[:,0] - labels[:,-1]))
In [ ]:
def score_result(loss, predictions, depths, labels):
return (loss, accuracy(predictions, labels[:,0:-1]), mean_depth_error(depths, labels))
In [ ]:
def print_batch_info(
context, score, predictions, depths, labels, verbose, print_count=20, depth_print=10
):
print(context, "accuracy: %.1f%%" % score[1])
if verbose:
print(np.argmax(predictions[0:print_count],1))
print(np.argmax(labels[0:print_count,0:-1],1))
print(context, "average depth error:", score[2])
print(depths[0:depth_print,0])
print(labels[0:depth_print,-1])
In [ ]:
def batch_accuracy(
context, session, graph, batcher, entropy, inputs, labels, batch_size, count, verbose
):
total_accuracy = 0
total_depth = 0
for b in range(count):
batcher.sample_batch(inputs, labels, batch_size, entropy)
targets = [graph["verify_predictions"], graph["verify_depths"]]
predictions, depths = session.run(targets, feed_dict={graph["verify"] : inputs})
total_accuracy += accuracy(predictions, labels) / float(count)
total_depth += mean_depth_error(depths, labels) / float(count)
score = (0, total_accuracy, total_depth)
print_batch_info(context, score, predictions, depths, labels, verbose)
return score
In [ ]:
def run_graph(
graph_info,
data,
step_count,
valid_count,
test_count=0,
batch_sampler_count=1000,
report_every=50,
verbose=True,
accuracy_minimum=None, # Minimimum validation percent accuracy for early abort
pickle_batches=None, # pickle files for training and validation batchers
tracker=None,
entropy=random
):
with tf.Session(graph=graph_info["graph"]) as session:
tf.initialize_all_variables().run()
print("Initialized")
# Optionally restore graph parameters from disk.
convnet.restore_model(graph_info, session)
# Set up space for graph inputs / feed values
batch_size = graph_info["batch_size"]
depth_labels = data["depth_labels"]
height, width, _ = data["image_size"]
inputs = np.zeros(shape=batch_input_shape(batch_size, data["image_size"]),
dtype=np.float32)
labels = np.zeros(shape=batch_output_shape(batch_size, depth_labels),
dtype=np.float32)
# Construct or unpickle training batcher.
train_batcher = load_batcher(pickle_batches, "train")
if not train_batcher:
train_batcher = BatchSampler(
data["train_files"], height, width, batch_sampler_count
)
score = (0,1)
try:
for step in range(step_count + 1):
if tracker:
tracker.update_progress(step)
# Generate a batch
train_batcher.sample_batch(inputs, labels, batch_size, entropy)
# Graph targets
run_targets = [
graph_info["optimizer"],
graph_info["loss"],
graph_info["predictions"],
graph_info["depths"]
]
# Graph inputs:
feed_dict = {graph_info["train"] : inputs, graph_info["targets"] : labels}
_, loss, predictions, depths = session.run(run_targets,feed_dict=feed_dict)
# Keep track of and possibly display score.
batch_score = score_result(loss, predictions, depths, labels)
if tracker:
tracker.record_score(batch_score)
if np.isnan(loss):
print("Error computing loss at step", step)
print_batch_info("Minibatch", batch_score, predictions,
depths, labels, True)
return 0
if (step % report_every == 0):
if verbose:
print("Minibatch loss at step", step, ":", loss)
print_batch_info("Minibatch", batch_score, predictions,
depths, labels, True)
# Evaluate the validation data.
valid_batcher = load_batcher(pickle_batches, "valid")
if not valid_batcher:
valid_files = data["valid_files"]
valid_batcher = BatchSampler(
valid_files, height, width, len(valid_files)
)
valid_score = batch_accuracy(
"Validation", session, graph_info, valid_batcher, entropy,
inputs, labels, batch_size, valid_count, verbose
)
del valid_batcher
score = valid_score[1:]
if accuracy_minimum and step > 0 and valid_score[1] < accuracy_minimum:
print("Early out.")
break
# Evaluate the test data, if any.
if test_count > 0:
test_batcher = BatchSampler(data["test_files"], height, width)
valid_accuracy = batch_accuracy(
"Test", session, graph_info, test_batcher, entropy,
inputs, labels, batch_size, test_count, verbose
)
return score
finally:
# Optionally save out graph parameters to disk.
convnet.save_model(graph_info, session)
In [ ]:
def valid_accuracy_metric(valid_accuracy, valid_depth_error, train_results):
return valid_accuracy
def valid_error_metric(valid_accuracy, valid_depth_error, train_results):
return valid_depth_error
def train_accuracy_metric(valid_accuracy, valid_depth_error, train_results):
result_count = min(len(train_results), 1000)
return sum(accuracy for _, accuracy, _ in train_results[-result_count:]) / result_count
def train_depth_error_metric(valid_accuracy, valid_depth_error, train_results):
result_count = min(len(train_results), 1000)
error = sum(error for _, _, error in train_results[-result_count:]) / result_count
return max(0, 1 - error)
In [ ]:
results_path = outputer.setup_directory("temp", "classy_results")
In [ ]:
def make_eval(
batch_size=20,
eval_steps=10000,
valid_steps=500,
regress_factor=1.0,
report_every=None,
reuse_cross=False,
metric=valid_accuracy_metric,
entropy=random
):
pickle_batches = {}
train_count = 9700
valid_count = 400
batch_sampler_count = min(801, eval_steps * batch_size)
test_count = None
#if reusing data, set up training and test data, and pickle batchers for efficiency.
if reuse_cross:
redata = setup_cross_validation(
data_files, train_count, valid_count, test_count,
label_count=DEPTH_LABEL_COUNT, entropy=entropy
)
pickle_batches["valid"] = pickle_batch(
redata, "valid", len(redata["valid_files"]), entropy
)
print("Pickled Validation")
pickle_batches["train"] = pickle_batch(
redata, "train", batch_sampler_count, entropy
)
print("Pickled Training")
progress_tracker = outputer.ProgressTracker(
["Loss", "Accuracy", "Error"], eval_steps, results_path, convevo.serialize
)
def evaluate(stack, eval_entropy):
# If not reusing data, generate training and validation sets
if not reuse_cross:
data = setup_cross_validation(
data_files, train_count, valid_count, test_count,
label_count=DEPTH_LABEL_COUNT, entropy=eval_entropy
)
pickle_batches["valid"] = pickle_batch(
data, "valid", len(data["valid_files"]), eval_entropy
)
print("Pickled Validation")
else:
data = redata
progress_tracker.setup_eval(stack)
# Set up the Tensorflow graph
try:
graph_info = setup_graph(
batch_size,
data["image_size"],
data["depth_labels"],
regress_factor,
stack
)
except KeyboardInterrupt:
raise
except:
progress_tracker.error(sys.exc_info())
return -10
progress_tracker.start_eval(graph_info)
# Run the graph
try:
valid_accuracy, valid_depth_error = run_graph(
graph_info,
data,
eval_steps,
valid_count=valid_steps,
batch_sampler_count=batch_sampler_count,
report_every=report_every if report_every else eval_steps//4,
verbose=True,
accuracy_minimum=None,
pickle_batches=pickle_batches,
tracker=progress_tracker,
entropy=eval_entropy
)
if metric:
return metric(valid_accuracy, valid_depth_error, progress_tracker.results)
return valid_accuracy
except KeyboardInterrupt:
raise
except:
progress_tracker.error(sys.exc_info())
return -1
finally:
progress_tracker.output()
return evaluate
In [ ]:
cross_data = setup_cross_validation(data_files,9700,400,1000,label_count=DEPTH_LABEL_COUNT)
In [ ]:
batch_size = 20
conv_layers = [
("conv_bias", 20, 2, 10, "SAME", True),
("conv_bias", 10, 5, 20, "SAME", True),
("conv_bias", 5, 2, 40, "SAME", True)
]
hidden_sizes = [400, 100, cross_data["depth_labels"] + 1]
optimizer = convevo.Optimizer("GradientDescent", 0.01)
optimizer.default_parameters()
prototype = convevo.create_stack(conv_layers,[],True,hidden_sizes,0.0, 0.05, 0.0,optimizer)
prototype.reseed(random.Random(42))
In [ ]:
prototype_graph = setup_graph(
batch_size,
cross_data["image_size"],
cross_data["depth_labels"],
1.0,
prototype
)
In [ ]:
run_graph(
prototype_graph, cross_data, 1000,
valid_count=200, report_every=500, verbose=True, entropy=random.Random(42)
)
In [ ]:
print(convevo.serialize(prototype))
prototype_entropy = random.Random(42)
prototype_eval = make_eval(
batch_size=100,
eval_steps=100,
valid_steps=20,
regress_factor=1.0,
reuse_cross=True,
entropy=prototype_entropy
)
prototype_eval(prototype, prototype_entropy)
In [ ]:
del cross_data
del conv_layers
del hidden_sizes
del prototype_graph
del prototype_eval
gc.collect()
In [ ]:
prototypes = [prototype]
In [ ]:
population,_,_ = convevo.load_population("testing/color_quad_run.xml", False)
prototypes = population[:5]
print(len(prototypes))
In [ ]:
prototypes = [
convevo.load_stack("testing/candidate1.xml"),
convevo.load_stack("testing/candidate2.xml"),
convevo.load_stack("testing/candidate3.xml"),
convevo.load_stack("testing/candidate4.xml"),
convevo.load_stack("testing/candidate5.xml")
]
In [ ]:
with outputer.TeeOutput(os.path.join("temp", outputer.timestamp("Depth_Evolve_", "txt"))):
mutate_seed = random.randint(1, 100000)
print("Mutate Seed:", mutate_seed)
mutate_entropy = random.Random(mutate_seed)
eval_seed = random.randint(1, 100000)
print("Eval Seed:", eval_seed)
eval_entropy = random.Random(eval_seed)
population_size = 10
generations = 5
batch_size = 100
breed_options = {
"input_shape": batch_input_shape(batch_size, data_files["image_size"]),
"output_shape": batch_output_shape(batch_size, data_files["depth_labels"])
}
for stack in prototypes:
stack.make_safe(breed_options["input_shape"], breed_options["output_shape"])
evaluator = make_eval(
batch_size=batch_size, eval_steps=40000, valid_steps=1000, regress_factor=1.0,
reuse_cross=True, metric=None, entropy=eval_entropy
)
charles = darwin.Darwin(convevo.serialize, evaluator, convevo.breed)
charles.init_population(prototypes, population_size, False,
breed_options, mutate_entropy)
for g in range(generations):
print("Generation", g)
results = charles.evaluate(eval_entropy)
convevo.output_results(results, "temp", outputer.timestamp() + ".xml",
mutate_seed, eval_seed)
charles.repopulate(population_size, 0.3, 3, results, breed_options, mutate_entropy)
In [ ]:
results = darwin.descending_score(charles.history.values())
convevo.output_results(results, "testing", "candidates_evolve_run.xml",
mutate_seed, eval_seed)
len(results)
In [ ]:
BATCH_SIZE = 100
candidate = convevo.load_stack("testing/candidate6.xml")
candidate.make_safe(
batch_input_shape(BATCH_SIZE, data_files["image_size"]),
batch_output_shape(BATCH_SIZE, data_files["depth_labels"])
)
print(convevo.serialize(candidate))
In [ ]:
candidate_evaluator = make_eval(
batch_size=BATCH_SIZE,
eval_steps=10000000,
valid_steps=100000,
regress_factor=1.0,
report_every=500000,
reuse_cross=False,
metric=None,
entropy=random.Random(42)
)
In [ ]:
with outputer.TeeOutput(os.path.join("temp", "candidate6_results.txt")):
candidate_evaluator(candidate, random.Random(57))
Test reloading the resulting graph for additional training/validation.
In [ ]:
with outputer.TeeOutput(os.path.join("temp", "candidate6_retest.txt")):
candidate = convevo.load_stack("testing/candidate6.xml")
candidate_reevaluator = make_eval(
batch_size=100, eval_steps=10000, valid_steps=10000, regress_factor=1.0,
reuse_cross=False, metric=None, entropy=random.Random(42)
)
candidate.checkpoint_path("testing/candidate6/full/2016-06-11~15_23_44_712.ckpt")
candidate_reevaluator(candidate, random.Random(42))
Calculates for non-NaN pixels:
In [ ]:
def test_score(labels, predictions, depths, count):
is_finite = np.isfinite(labels[:count,-1])
where_valid = np.where(is_finite)
count = np.count_nonzero(is_finite)
if count:
score = accuracy(predictions[where_valid], labels[where_valid])
error = mean_depth_error(depths[where_valid], labels[where_valid])
valid_predictions = predictions[where_valid]
label_depths = depths_for_labels_normalized(valid_predictions)
label_error = mean_depth_error(label_depths[:,np.newaxis], labels[where_valid])
argmax_predictions = np.argmax(valid_predictions, axis=1)
argmax_depths = DEPTH_BOUNDARY_MIDPOINTS[argmax_predictions] / improc.MAX_DEPTH
argmax_error = mean_depth_error(argmax_depths[:,np.newaxis], labels[where_valid])
return score*count, error*count, label_error*count, argmax_error*count, count
return 0, 0, 0, 0, 0
In [ ]:
# Validate the test_score function.
def check_test_score():
test_batch_size = 10
test_labels = np.zeros(shape=batch_output_shape(test_batch_size, DEPTH_LABEL_COUNT),
dtype=np.float32)
test_depths = np.zeros(shape=(test_batch_size,1), dtype=np.float32)
for l in range(test_batch_size):
test_depth = improc.MAX_DEPTH * l / float(test_batch_size)
depth_label(test_depth, test_labels[l])
test_depths[l, 0] = test_labels[l,-1]
test_predictions = np.copy(test_labels)[:,:-1]
test_predictions[0, 10] = 0.5
test_labels[2] = np.nan
score = test_score(test_labels, test_predictions, test_depths, 7)
print(score)
print([s / score[-1] for s in score[:-1]])
check_test_score()
For all the test images in the provided data set, compute metrics for full images, and generate the corresponding image for the output depth and either the linear combination of the labeled softmax depth output, or the argmax labeled depth output.
In [ ]:
def compute_test_images(graph_info, data, output_path):
with tf.Session(graph=graph_info["graph"]) as session:
tf.initialize_all_variables().run()
print("Initialized")
# restore graph parameters from disk.
convnet.restore_model(graph_info, session)
# Set up space for graph inputs / feed values
batch_size = graph_info["batch_size"]
depth_labels = data["depth_labels"]
image_size = data["image_size"]
inputs = np.zeros(shape=batch_input_shape(batch_size, image_size),
dtype=np.float32)
labels = np.zeros(shape=batch_output_shape(batch_size, depth_labels),
dtype=np.float32)
nan_label = np.array([np.nan]*labels.shape[-1], dtype=np.float32)
source_image_size = (480, 640)
height_span = source_image_size[0] - image_size[0]
width_span = source_image_size[1] - image_size[1]
pixel_order = np.array(linear_order(height_span, width_span))
files = data["test_files"]
eval_count = len(files) * len(pixel_order) // batch_size
progress = outputer.show_progress("Evaluation Steps:", eval_count)
eval_count = 0;
all_scores = {}
for image_path in files:
sampler = ImageSampler(image_path, image_size[0], image_size[1])
if output_path:
raw_depths = ndimage.imread(image_path)
label_depths = np.copy(raw_depths)
argmax_depths = np.copy(raw_depths)
image_scores = np.zeros(shape=(5,), dtype=np.float32)
gc.collect()
for row in range(height_span):
# Update progress
eval_count += 1
progress.value = eval_count
# Generate a batch and run the graph
batch_pixels = pixel_order[row * width_span : (row + 1) * width_span, :]
labels.fill(0)
for i, pixel in enumerate(batch_pixels):
if not sampler.sample_at(pixel, inputs, labels, i):
labels[i] = nan_label
targets = [graph_info["verify_predictions"],
graph_info["verify_depths"]]
predictions, depths = session.run(
targets, feed_dict={graph_info["verify"] : inputs}
)
if output_path:
iy = ((raw_depths.shape[0] + image_size[0]) // 2) + row
sx = (image_size[0] // 2)
ex = sx + width_span
raw_depths[iy, sx : ex] = improc.encode_normalized_depths(depths)
label_depths[iy, sx : ex] = improc.encode_normalized_depths(
depths_for_labels_normalized(predictions)[:, np.newaxis]
)
argmax_predictions = np.argmax(predictions, axis=1)
argmax_depth_values = DEPTH_BOUNDARY_MIDPOINTS[argmax_predictions]
argmax_depths[iy, sx : ex] = improc.encode_normalized_depths(
(argmax_depth_values / improc.MAX_DEPTH)[:, np.newaxis]
)
image_scores += test_score(labels, predictions, depths, len(batch_pixels))
image_name, ext = os.path.splitext(os.path.basename(image_path))
all_scores[image_name] = image_scores
print("Image scores for", image_name, image_scores[:-1] / image_scores[-1])
if output_path:
outputs = [
(raw_depths, "_depth"),
(label_depths, "_softmax"),
(argmax_depths, "_argmax")
]
for image, postfix in outputs:
imsave(os.path.join(output_path, image_name + postfix + ".png"), image)
return all_scores
Format the test results as a CSV file, and find the min/max for each metric.
In [ ]:
def output_test_scores(test_scores, test_data, path):
with outputer.TeeOutput(path):
titles = ["Name", "Accuracy", "Error", "Label Error", "Argmax Error", "Count"]
total = np.zeros(shape=(5,), dtype=np.float32)
lines = [titles]
for image_path in test_data["test_files"]:
image_name, ext = os.path.splitext(os.path.basename(image_path))
scores = test_scores[image_name]
total += scores
line = [image_name]
line.extend(scores[:-1] / scores[-1])
line.append(scores[-1])
lines.append(line)
line = ["Total"]
line.extend(total[:-1] / total[-1])
line.append(total[-1])
lines.append(line)
text = "\n".join(",".join(str(v) for v in line) for line in lines)
print(text)
for i in range(1, 5):
sorted_lines = sorted(lines[1:-1], key=lambda l: l[i])
print(titles[i] + " high")
print(",".join([str(v) for v in sorted_lines[-1]]))
print(titles[i] + " low")
print(",".join([str(v) for v in sorted_lines[0]]))
Constructs a simple graph that just computes the output label and depth corresponding to a constant depth value.
In [ ]:
def predict_constant_depth(batch_size, image_shape, label_count, value):
graph = tf.Graph()
with graph.as_default():
verify = tf.placeholder(tf.float32,
shape=batch_input_shape(batch_size, image_shape))
mean_label = tf.one_hot(depth_label_index(value), label_count, np.float32(1), 0)
mean_label = tf.reshape(mean_label, (1, DEPTH_LABEL_COUNT))
return {
"graph": graph,
"batch_size": batch_size,
"verify": verify,
"verify_predictions": tf.tile(mean_label, [batch_size, 1]),
"verify_depths": tf.fill([batch_size, 1], value / improc.MAX_DEPTH)
}
Compute the test scores resulting from just predicting the mean for every pixel.
In [ ]:
with outputer.TeeOutput(os.path.join("temp", "guess_mean_test.txt")):
mean_graph = predict_constant_depth(
100, data_files["image_size"], DEPTH_LABEL_COUNT, MEAN_DEPTH
)
test_data = setup_cross_validation(
data_files, 0, 0, 1123, label_count=DEPTH_LABEL_COUNT
)
mean_test_scores = compute_test_images(
mean_graph, test_data, False, None
)
output_test_scores(mean_test_scores, test_data, "temp/mean_test_scores.csv")
Run and score the candidate graph for the full test set.
In [ ]:
def test_candidate_stack(stack_path, output_path, output_images):
batch_size = 640 - data_files["image_size"][1]
with outputer.TeeOutput(os.path.join(output_path, "full_test.txt")):
candidate = convevo.load_stack(stack_path)
test_data = setup_cross_validation(
data_files, 0, 0, 1123, label_count=DEPTH_LABEL_COUNT
)
candidate_graph = setup_graph(
batch_size, test_data["image_size"], test_data["depth_labels"], 1.0, candidate
)
convnet.setup_restore_model(
candidate_graph, candidate.checkpoint_path()
)
test_scores = compute_test_images(
candidate_graph, test_data, output_path if output_images else None
)
output_test_scores(
test_scores, test_data, os.path.join(output_path, "full_test_scores.csv")
)
In [ ]:
candidate6_results_path = outputer.setup_directory("temp/candidate6")
test_candidate_stack("testing/candidate6.xml", candidate6_results_path, True)
In [ ]: