In [ ]:
%matplotlib inline
from __future__ import print_function
import gc
import ipywidgets
import math
import os
import random
import sys
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
from IPython.display import Image
from scipy import ndimage
from scipy.misc import imsave
import outputer
import improc
import convnet
import mutate
import convevo
import darwin
In [ ]:
reload (improc)
reload (convnet)
reload (mutate)
reload (convevo)
reload (darwin)
In [ ]:
training, test = improc.enumerate_images("captures")
print("Training:", len(training), "Test:", len(test))
print(training[:2])
print(test[:2])
Each image file contains a color image (top half), and an encoded depth image (bottom half)
The improc module contains functions for splitting the image, decoding the depth back into floating point millimeters, and for filling in gaps.
In [ ]:
# Precomputed via compute_average_depth()
# Actually it should 1680.24, value below is actually the mean of the image means.
# Keeping this value as it was what was used in the experiments to date,
# and it is close to the correct value.
MEAN_DEPTH = np.float32(1688.97)
NORMALIZED_MEAN_DEPTH = MEAN_DEPTH / improc.MAX_DEPTH
print(MEAN_DEPTH)
In [ ]:
depth_image_cache_path = outputer.setup_directory("temp", "cache")
class ImageSampler(object):
"""Wrap an image for sampling."""
def __init__(self, image_file, make_holes, zero_mean=False):
# Process the image or grab it from the cache.
# image is normalized CIELAB, depth is not normalized.
self.image, self.depth = improc.process_cached(depth_image_cache_path, image_file)
self.depth /= improc.MAX_DEPTH
self.make_holes = make_holes
self.mean_offset = 0
if zero_mean:
self.mean_offset = NORMALIZED_MEAN_DEPTH
self.depth -= self.mean_offset
def scale_size(self, size, pixel_count):
return int(size * pixel_count / 100.0)
def make_hole_mask(self, image_slot, entropy, bias=-0.15, noise_scales=None):
mask_height = image_slot.shape[0]
mask_width = image_slot.shape[1]
if not noise_scales:
noise_scales = [
(self.scale_size(2, mask_height), self.scale_size(2, mask_width), 0.6),
(self.scale_size(40, mask_height), self.scale_size(20, mask_width), .35),
(mask_height // 2, mask_width // 2, 0.05)
]
noise_image = np.zeros(shape=(image_slot.shape[:2]))
for y_scale, x_scale, amplitude in noise_scales:
noise_image += improc.make_noise(
mask_height, mask_width, y_scale, x_scale, entropy
) * amplitude
return noise_image, noise_image > bias
def sample(self, image_slot, depth_slot,
mask_slot=None, h_offset=None, w_offset=None, entropy=np.random):
height = image_slot.shape[0]
spare_height = self.image.shape[0] - height
y = spare_height // 2 if h_offset is None else h_offset
width = image_slot.shape[1]
spare_width = self.image.shape[1] - width
x = spare_width // 2 if w_offset is None else w_offset
if self.make_holes:
input_depth = image_slot[:,:,-1]
input_depth[:,:] = improc.mipmap_imputer(
self.depth[y : y + height, x : x + width], smooth=True
)
_, mask = self.make_hole_mask(input_depth, entropy)
input_depth[np.where(mask)] = -1
image_slot = image_slot[:,:,:-1]
if mask_slot is not None:
mask_slot[:,:,0] = mask
else:
image_slot = image_slot[:,:,:]
image_slot[:,:,:] = self.image[y : y+height, x : x+width, : image_slot.shape[-1]]
depth_slot[:,:,0] = self.depth[y : y+height, x : x+width]
In [ ]:
def prepare_images(paths, inputs, targets, masks=None, entropy=np.random):
for i, sampler in enumerate([ImageSampler(path, masks is not None) for path in paths]):
sampler.sample(inputs[i], targets[i],
None if masks is None else masks[i], entropy=entropy)
In [ ]:
example_image, example_depth, example_attitude = improc.load_image("testing/IMG_2114.PNG")
plt.imshow(example_image)
print(example_image.shape, example_image.dtype)
In [ ]:
plt.imshow(example_depth)
print(example_depth.shape, example_depth.dtype)
print(example_attitude)
In [ ]:
sampler = ImageSampler("testing/IMG_2114.PNG", False)
sample_size = 100
image_sample = np.zeros(shape=(sample_size, sample_size, 3))
depth_sample = np.zeros(shape=(sample_size, sample_size, 1))
sampler.sample(image_sample, depth_sample)
print(image_sample.shape, image_sample.dtype)
plt.imshow(image_sample)
In [ ]:
plt.imshow(depth_sample.reshape(sample_size, sample_size))
print(depth_sample.shape, depth_sample.dtype)
print(np.min(depth_sample), np.max(depth_sample))
In [ ]:
sampler = ImageSampler("testing/IMG_2114.PNG", True)
sample_size = 100
image_sample = np.zeros(shape=(sample_size, sample_size, 4))
depth_sample = np.zeros(shape=(sample_size, sample_size, 1))
sample_mask = np.zeros(shape=(sample_size, sample_size, 1))
sampler.sample(image_sample, depth_sample, sample_mask, entropy=np.random.RandomState(11))
print(image_sample.shape, image_sample.dtype)
plt.imshow(image_sample[:,:,-1])
In [ ]:
plt.imshow(sample_mask[:,:,0])
print(np.sum(sample_mask) / np.sum(np.ones_like(sample_mask)))
In [ ]:
COLOR_CHANNELS = 3
image_height = 480
image_width = 640
data_files = {
"image_size": (image_height, image_width, COLOR_CHANNELS),
"depth_size": (image_height, image_width, 1),
"train_files": np.array(sorted(training)),
"test_files": np.array(sorted(test))
}
del training
del test
In [ ]:
def setup_cross_validation(
data, valid_count, test_count=None, chunk_size=None, entropy=random
):
cross_data = data.copy()
if chunk_size:
cross_data["image_size"] = chunk_size
cross_data["depth_size"] = chunk_size[:-1] + (1,)
paths = cross_data["train_files"][:]
mutate.fisher_yates_shuffle(paths, entropy)
cross_data["train_files"] = paths[:-valid_count]
cross_data["valid_files"] = paths[-valid_count:]
if test_count is None:
del cross_data["test_files"]
else:
cross_data["test_files"] = data["test_files"][:test_count]
return cross_data
In [ ]:
def setup_graph(
batch_size,
image_shape,
target_shape,
stack
):
graph = tf.Graph()
with graph.as_default():
input_shape = (batch_size,) + image_shape
output_shape = (batch_size,) + target_shape
train = tf.placeholder(tf.float32, shape=input_shape)
targets = tf.placeholder(tf.float32, shape=output_shape)
verify = tf.placeholder(tf.float32, shape=input_shape)
operations = stack.construct(input_shape, output_shape)
l2_loss = convnet.setup(operations)
results = convnet.connect_model(train, operations, True)[-1]
# Fill NaNs in target with values from results to
# eliminate any contribution to the gradient
valid_targets = tf.where(tf.is_nan(targets), results, targets)
loss = tf.reduce_mean(tf.squared_difference(results, valid_targets)) + l2_loss
verify_predictions = convnet.connect_model(verify, operations, False)[-1]
verify_predictions = tf.maximum(verify_predictions, 0)
verify_predictions = tf.minimum(verify_predictions, 1)
info = {
"graph": graph,
"batch_size": batch_size,
"train": train,
"targets": targets,
"loss": loss,
"optimizer": stack.construct_optimizer(loss),
"predictions": results,
"verify": verify,
"verify_predictions": verify_predictions,
"saver": tf.train.Saver()
}
return info
In [ ]:
def prediction_error(predictions, targets):
is_finite = np.isfinite(targets)
where_valid = np.where(is_finite)
error = np.mean(np.absolute(predictions[where_valid] - targets[where_valid]))
return error, np.count_nonzero(is_finite)
In [ ]:
def make_predictor(session, graph_info):
def predict(inputs, targets):
feed_dict = {graph_info["verify"]: inputs}
return session.run([graph_info["verify_predictions"]], feed_dict=feed_dict)[0]
return predict
In [ ]:
def mask_targets(targets, masks):
if masks is not None:
targets[np.where(np.logical_not(masks))] = np.nan
In [ ]:
def batch_prediction_error(
predictor, files, inputs, targets, masks, batch_size, entropy=np.random
):
total_error = 0
total_count = 0
batch_count = len(files) // batch_size
for b in range(batch_count):
offset = b * batch_size
end = offset + batch_size
prepare_images(files[offset:end], inputs, targets, masks, entropy)
predictions = predictor(inputs, targets)
mask_targets(targets, masks)
error, count = prediction_error(predictions, targets)
total_error += error * count
total_count += count
return (total_error / np.float32(total_count)), predictions, targets
In [ ]:
def depth_mean_like(depths):
return np.ones_like(depths) * (MEAN_DEPTH / improc.MAX_DEPTH)
def always_guess_mean_error(files, inputs, targets, masks, batch_size, entropy):
def predict_mean(images, depths):
return depth_mean_like(depths)
return batch_prediction_error(
predict_mean,
files, inputs,
targets, masks,
batch_size, entropy
)[0]
In [ ]:
def batch_input_shape(batch_size, data, make_holes):
shape = (batch_size,) + data["image_size"]
if make_holes:
shape = shape[:-1] + (shape[-1] + 1,)
return shape
def batch_output_shape(batch_size, data):
return (batch_size,) + data["depth_size"]
def score_run(guess_mean_error, valid_error):
return guess_mean_error - min(valid_error, 1)
def run_graph(
graph_info,
data,
make_holes,
step_count,
report_every=50,
verbose=True,
tracker=None,
track_minibatch_error=False,
mean_error_cache=None,
error_maximum=None,
prepare_seeds=None
):
with tf.Session(graph=graph_info["graph"]) as session:
tf.global_variables_initializer().run()
print("Initialized")
# Optionally restore graph parameters from disk.
convnet.restore_model(graph_info, session)
batch_size = graph_info["batch_size"]
batch_inputs = np.empty(shape=batch_input_shape(batch_size, data, make_holes),
dtype=np.float32)
batch_targets = np.empty(shape=batch_output_shape(batch_size, data),
dtype=np.float32)
if make_holes:
batch_masks = np.empty(shape=batch_targets.shape, dtype=np.bool)
else:
batch_masks = None
# Validation and scoring bits.
valid_error = 1
if prepare_seeds is None:
prepare_seeds = [random.randint(1, 12345), random.randint(1, 12345)]
guess_mean_error = None
if mean_error_cache is not None:
guess_mean_error = mean_error_cache.get("cached")
if not guess_mean_error:
guess_mean_error = always_guess_mean_error(
data["valid_files"], batch_inputs, batch_targets, batch_masks, batch_size,
np.random.RandomState(prepare_seeds[1])
)
print("Error if just guess mean:", guess_mean_error)
if mean_error_cache is not None:
mean_error_cache["cached"] = guess_mean_error
predictor = make_predictor(session, graph_info)
prepare_entropy = np.random.RandomState(prepare_seeds[0])
training_files = data["train_files"]
try:
for step in range(step_count + 1):
if tracker:
tracker.update_progress(step)
# Generate a minibatch.
offset = (step * batch_size) % (training_files.shape[0] - batch_size)
batch_files = training_files[offset:(offset + batch_size)]
prepare_images(
batch_files, batch_inputs, batch_targets, batch_masks, prepare_entropy
)
# Graph evaluation targets:
targets = [
graph_info["optimizer"],
graph_info["loss"],
graph_info["predictions"]
]
# Graph inputs:
feed_dict = {
graph_info["train"] : batch_inputs,
graph_info["targets"] : batch_targets
}
# Run the graph
_, loss, predictions = session.run(targets, feed_dict=feed_dict)
# Capture last prediction
results = (predictions[-1], batch_targets[-1], batch_inputs[-1])
# Update stats:
reporting = step % report_every == 0
if reporting or track_minibatch_error:
batch_error, _ = prediction_error(predictions, batch_targets)
else:
batch_error = None
if tracker:
tracker.record_score((loss, batch_error))
if not np.isfinite(loss):
print("Error computing loss:", loss)
print(np.sum(np.isnan(predictions)))
return score_run(guess_mean_error, valid_error), results
if reporting:
if verbose:
print("Minibatch loss at step", step, ":", loss)
print("Minibatch error:", batch_error)
valid_error, _, _ = batch_prediction_error(
predictor, data["valid_files"],
batch_inputs, batch_targets, batch_masks, batch_size,
np.random.RandomState(prepare_seeds[1])
)
print("Validation error:", valid_error)
if error_maximum and step > 0 and valid_error > error_maximum:
print("Early out.")
break
test_files = data.get("test_files")
if test_files is not None:
test_results = batch_prediction_error(
predictor, test_files,
batch_inputs, batch_targets, batch_masks, batch_size,
np.random.RandomState(prepare_seeds[1])
)
print("Test error:", test_results[0])
results = results + test_results
return score_run(guess_mean_error, valid_error), results
finally:
# Optionally save out graph parameters to disk.
convnet.save_model(graph_info, session)
In [ ]:
TEST_BATCH = 1
test_inputs = np.empty(shape=(TEST_BATCH, 480, 640, COLOR_CHANNELS + 1), dtype=np.float32)
test_depths = np.empty_like(test_inputs[:,:,:,:1])
test_mask = np.empty_like(test_depths)
prepare_images(data_files["test_files"][:TEST_BATCH], test_inputs, test_depths, test_mask,
np.random.RandomState(1))
In [ ]:
plt.imshow(test_inputs[0,:,:,0], cmap='Greys_r')
print(np.min(test_inputs[0,:,:,0]),np.max(test_inputs[0,:,:,0]))
print(np.min(test_inputs[0,:,:,1]),np.max(test_inputs[0,:,:,1]))
print(np.min(test_inputs[0,:,:,2]),np.max(test_inputs[0,:,:,2]))
In [ ]:
plt.imshow(test_depths[0,:,:,0])
depths_valid = np.where(np.isfinite(test_depths[0]))
print(np.min(test_depths[0][depths_valid]),np.max(test_depths[0][depths_valid]))
In [ ]:
print(test_inputs.shape)
print(test_depths.shape)
prediction_error(test_inputs[:,:,:,0:1], test_depths)
In [ ]:
prediction_error(np.zeros_like(test_depths), test_depths)
In [ ]:
prediction_error(depth_mean_like(test_depths), test_depths)
In [ ]:
prediction_error(test_depths, test_depths)
In [ ]:
mask_targets(test_depths, test_mask)
prediction_error(depth_mean_like(test_depths), test_depths)
In [ ]:
plt.imshow(test_depths[0,:,:,0])
In [ ]:
TEST_BATCH = 1
MAKE_HOLES = True
conv_layers = [
("conv", 5, 2, 10, "SAME", False),
("conv", 10, 2, 20, "SAME", False),
("conv_bias", 15, 5, 25, "SAME", False)
]
expand_layers = [
(5, 5, "SAME", True, False),
(2, 5, "SAME", True, False),
(2, 5, "SAME", True, False)
]
test_input_shape = batch_input_shape(TEST_BATCH, data_files, MAKE_HOLES)
test_output_shape = batch_output_shape(TEST_BATCH, data_files)
test_stack = convevo.create_stack(conv_layers, expand_layers, False, [], 0.0, 0.01, 0.0)
test_stack.make_safe(test_input_shape, test_output_shape)
test_stack.reseed(random.Random(24601))
In [ ]:
sample_size = data_files["image_size"]
depth_size = data_files["depth_size"]
test_graph = setup_graph(TEST_BATCH, test_input_shape[1:], depth_size, test_stack)
In [ ]:
test_cross = setup_cross_validation(data_files, 200, 200, sample_size)
test_score, test_results = run_graph(
test_graph, test_cross, MAKE_HOLES, 8, 4, True, prepare_seeds=[654, 321]
)
print(test_score)
In [ ]:
plt.imshow(test_results[0][:,:,0])
print(np.min(test_results[0]),np.max(test_results[0]))
In [ ]:
plt.imshow(test_results[1].reshape(sample_size[0],sample_size[1]))
In [ ]:
plt.imshow(test_results[2][:,:,0], cmap='Greys_r')
In [ ]:
del test_stack
del test_graph
del test_cross
del test_results
gc.collect()
In [ ]:
results_path = outputer.setup_directory("temp", "pyndent_results")
In [ ]:
def make_eval(batch_size, eval_steps, valid_size, reuse_cross, make_holes, entropy=random):
mean_error_cache = None
if reuse_cross:
redata = setup_cross_validation(
data_files, valid_size, entropy=entropy
)
mean_error_cache = {}
progress_tracker = outputer.ProgressTracker(
["Loss", "Error"], eval_steps, results_path, convevo.serialize
)
prepare_seeds = [random.randint(1, 12345), random.randint(1, 12345)]
def evaluate(stack, eval_entropy):
# If not reusing data, generate training and validation sets
if not reuse_cross:
data = setup_cross_validation(
data_files, valid_size, entropy=eval_entropy
)
else:
data = redata
progress_tracker.setup_eval(stack)
# Set up the graph
try:
graph_info = setup_graph(
batch_size,
batch_input_shape(batch_size, data, make_holes)[1:],
data["depth_size"],
stack
)
except KeyboardInterrupt:
raise
except:
progress_tracker.error(sys.exc_info())
return -10
progress_tracker.start_eval(graph_info)
# Run the graph
try:
valid_error, _ = run_graph(
graph_info,
data,
make_holes,
eval_steps,
report_every=eval_steps//10,
verbose=True,
tracker=progress_tracker,
mean_error_cache=mean_error_cache,
error_maximum=None,
prepare_seeds=prepare_seeds
)
return valid_error
except KeyboardInterrupt:
raise
except:
progress_tracker.error(sys.exc_info())
return -1
finally:
progress_tracker.output()
return evaluate
In [ ]:
# Basic convolutional network.
conv_layers = [
("conv", 5, 2, 10, "SAME", False),
("conv", 10, 2, 20, "SAME", False),
("conv_bias", 15, 5, 25, "SAME", False)
]
expand_layers = [
(5, 5, "SAME", True, False),
(2, 5, "SAME", True, False),
(2, 5, "SAME", True, False)
]
prototype = convevo.create_stack(conv_layers, expand_layers, False, [], 0.0, 0.01, 0.0)
prototypes = [prototype]
In [ ]:
# Past evolutionary experiment
population,_,_ = convevo.load_population("testing/pyndent_evolve_run.xml", False)
prototypes = population[:5]
print(len(prototypes))
In [ ]:
# Hand selected/tuned protototypes
prototypes = [
convevo.load_stack("testing/pyndent1.xml"),
convevo.load_stack("testing/pyndent2.xml"),
convevo.load_stack("testing/pyndent3.xml"),
convevo.load_stack("testing/pyndent4.xml"),
convevo.load_stack("testing/pyndent5.xml")
]
In [ ]:
# Evoloved prototype
prototypes = [
convevo.load_stack("testing/pyndent6/2016-06-22~17_51_16_872.xml")
]
In [ ]:
# Hole filling variant experiments
prototypes = [
convevo.load_stack("testing/passthrough.xml"),
convevo.load_stack("testing/holes1.xml")
]
In [ ]:
with outputer.TeeOutput(os.path.join("temp", outputer.timestamp("Pyndent_evo", "txt"))):
mutate_seed = random.randint(1, 100000)
print("Mutate Seed:", mutate_seed)
mutate_entropy = random.Random(mutate_seed)
eval_seed = random.randint(1, 100000)
print("Eval Seed:", eval_seed)
eval_entropy = random.Random(eval_seed)
population_size = 10
generations = 5
batch_size = 1
make_holes = False
breed_options = {
"input_shape": batch_input_shape(batch_size, data_files, make_holes),
"output_shape": batch_output_shape(batch_size, data_files),
"fixed_stride": make_holes,
"fixed_padding": make_holes
}
# Ensure loaded networks match input/output shapes for this run
for stack in prototypes:
stack.make_safe(breed_options["input_shape"], breed_options["output_shape"])
evaluator = make_eval(
batch_size=batch_size, eval_steps=80000, valid_size=400,
reuse_cross=True, make_holes=make_holes, entropy=eval_entropy
)
charles = darwin.Darwin(convevo.serialize, evaluator, convevo.breed)
charles.init_population(
prototypes, population_size, True, breed_options, mutate_entropy
)
try:
for g in range(generations):
print("Generation", g)
results = charles.evaluate(eval_entropy)
convevo.output_results(
results, "temp", outputer.timestamp() + ".xml", mutate_seed, eval_seed
)
charles.repopulate(
population_size, 0.3, 3, results, breed_options, mutate_entropy
)
finally:
results = darwin.descending_score(charles.history.values())
convevo.output_results(results, "temp", "pyndent_evo.xml", mutate_seed,eval_seed)
print("Evaluated", len(results))
In [ ]:
def compute_test_images(graph_info, data, make_holes, output_path, entropy):
with tf.Session(graph=graph_info["graph"]) as session:
tf.global_variables_initializer().run()
print("Initialized")
# restore graph parameters from disk.
convnet.restore_model(graph_info, session)
# Set up space for graph inputs
batch_size = graph_info["batch_size"]
batch_inputs = np.empty(shape=batch_input_shape(batch_size, data, make_holes),
dtype=np.float32)
batch_targets = np.empty(shape=batch_output_shape(batch_size, data),
dtype=np.float32)
batch_masks = None
if make_holes:
batch_masks = np.empty(shape=batch_targets.shape, dtype=np.bool)
predictor = make_predictor(session, graph_info)
# Set up progress bar
test_files = data["test_files"]
eval_count = len(test_files) // batch_size
progress = outputer.show_progress("Evaluation Steps:", eval_count)
# Only score error on same portion of the image as classy so we can compare.
input_image_size = batch_inputs.shape[1:]
classy_sample = (101, 101)
sample_start = tuple(cs // 2 for cs in classy_sample)
sample_end = tuple(
ss+iis-cs for ss,iis,cs in zip(sample_start, input_image_size, classy_sample)
)
classy_depth_start = (input_image_size[0] + sample_start[0], sample_start[1])
classy_depth_end = tuple(
cds+se-ss for cds,se,ss in zip(classy_depth_start, sample_end, sample_start)
)
def sample_patch(image, index):
return image[
index,
sample_start[0] : sample_end[0],
sample_start[1] : sample_end[1],
:
]
titles = ["Name", "Error", "Count"]
print(",".join(titles))
all_scores = []
error_sum = 0
pixel_count = 0
for step in range(eval_count):
progress.value = step # Update progress bar
# Load the test data and calculate predicted depths.
offset = step * batch_size
end = offset + batch_size
batch_files = test_files[offset:end]
prepare_images(batch_files, batch_inputs, batch_targets, batch_masks)
predictions = predictor(batch_inputs, batch_targets)
mask_targets(batch_targets, batch_masks)
for i in range(batch_size):
image_path = batch_files[i]
image_name, ext = os.path.splitext(os.path.basename(image_path))
# Calculate metrics
error, count = prediction_error(
sample_patch(predictions, i), sample_patch(batch_targets, i)
)
results = [image_name, error, count]
error_sum += error * count
pixel_count += count
all_scores.append(results)
# Encode result images
image = ndimage.imread(image_path)
encoded_depths = improc.encode_normalized_depths(predictions)
# Classy patch
image[
classy_depth_start[0] : classy_depth_end[0],
classy_depth_start[1] : classy_depth_end[1],
:
] = sample_patch(encoded_depths, i)
imsave(os.path.join(output_path, image_name + "_framed.png"), image)
# Full image
image[input_image_size[0]:,:,:] = encoded_depths[i]
imsave(os.path.join(output_path, image_name + "_full.png"), image)
print(",".join(str(v) for v in results))
# Aggregate results
print(",".join(["Total", str(error_sum / pixel_count), str(pixel_count)]))
sorted_scores = sorted(all_scores, key=lambda l: l[1])
print(titles[1] + " high")
print(",".join([str(v) for v in sorted_scores[-1]]))
print(titles[1] + " low")
print(",".join([str(v) for v in sorted_scores[0]]))
return all_scores
In [ ]:
test_results_path = outputer.setup_directory("temp/pyndent6")
BATCH_SIZE = 1
MAKE_HOLES = False
with outputer.TeeOutput(os.path.join(test_results_path, "test_results.txt")):
candidate = convevo.load_stack(
"testing/pyndent6/2016-07-15~20_30_58_640.xml"
)
test_data = setup_cross_validation(
data_files, 0, 1123, entropy=random.Random(121)
)
test_input_shape = batch_input_shape(BATCH_SIZE, test_data, MAKE_HOLES)
candidate_graph = setup_graph(
BATCH_SIZE, test_input_shape[1:], test_data["depth_size"], candidate
)
convnet.setup_restore_model(
candidate_graph, candidate.checkpoint_path()
)
candidate_test_scores = compute_test_images(
candidate_graph, test_data, MAKE_HOLES, test_results_path, random.Random(57)
)
In [ ]: