In [1]:

    

import sys
niftynet_path = '/home/tom/phd/NiftyNet-Generator-PR/NiftyNet'
sys.path.append(niftynet_path)
import pandas as pd
import os
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
from niftynet.io.image_reader import ImageReader
from collections import namedtuple

from niftynet.contrib.preprocessors.preprocessing import Preprocessing
from niftynet.contrib.csv_reader.sampler_csv_rows import ImageWindowDatasetCSV
from niftynet.contrib.csv_reader.sampler_resize_v2_csv import ResizeSamplerCSV as ResizeSampler
from niftynet.contrib.csv_reader.csv_reader import CSVReader


    

    




INFO:tensorflow:TensorFlow version 1.6.0
CRITICAL:tensorflow:Optional Python module cv2 not found, please install cv2 and retry if the application fails.
INFO:tensorflow:Available Image Loaders:
['nibabel', 'skimage', 'pillow', 'simpleitk', 'dummy'].
INFO:niftynet: Optional Python module yaml not found, please install yaml and retry if the application fails.
INFO:niftynet: Optional Python module yaml version None not found, please install yaml-None and retry if the application fails.

In [2]:

    

from niftynet.utilities.download import download
download('mr_ct_regression_model_zoo_data')
labels_location = 'ct.csv'
files = [file for file in os.listdir('/home/tom/niftynet/data/mr_ct_regression/CT_zero_mean') if file.endswith('.nii.gz')]
pd.DataFrame(data=[(file, file.replace('.nii.gz', '')) for file in files]).to_csv('ct.csv', index=None)


    

    




Accessing: https://github.com/NifTK/NiftyNetModelZoo
mr_ct_regression_model_zoo_data: OK. 
Already downloaded. Use the -r option to download again.

In [3]:

    

NetParam = namedtuple('NetParam', 'normalise_foreground_only foreground_type multimod_foreground_type histogram_ref_file norm_type cutoff normalisation whitening')
ActionParam = namedtuple('ActionParam', 'random_flipping_axes scaling_percentage rotation_angle rotation_angle_x rotation_angle_y rotation_angle_z do_elastic_deformation num_ctrl_points deformation_sigma proportion_to_deform')

        
class TaskParam:
    def __init__(self, classes):
        self.image = classes
net_param = NetParam(normalise_foreground_only=False,
                     foreground_type='threshold_plus',
                     multimod_foreground_type = 'and',
                     histogram_ref_file='mapping.txt',
                     norm_type='percentile',
                     cutoff=(0.05, 0.95),
                     normalisation=False,
                     whitening=True
                    )
action_param = ActionParam(random_flipping_axes=[],
                           scaling_percentage=[],
                           rotation_angle=None,
                           rotation_angle_x=None,
                           rotation_angle_y=None,
                           rotation_angle_z=None,
                           do_elastic_deformation=False,
                           num_ctrl_points=6,
                           deformation_sigma=50,
                           proportion_to_deform=0.9)

task_param = {'image': {'image':True}}
task_param = TaskParam(['image'])
print(vars(task_param).get('image'))
# creating an image reader.
data_param = {'CT': {'path_to_search': '~/niftynet/data/mr_ct_regression/CT_zero_mean',
            'filename_contains': 'nii'}}
grouping_param = {'image': (['CT'])}

image_reader = ImageReader().initialise(data_param, grouping_param)
preprocessing = Preprocessing(net_param, action_param, task_param)
normalisation_layers = preprocessing.prepare_normalisation_layers()
augmentation_layers = preprocessing.prepare_augmentation_layers()
image_reader.add_preprocessing_layers(normalisation_layers + augmentation_layers)
csv_reader = CSVReader().initialise(labels_location)


    

    




['image']
INFO:niftynet: 

Number of subjects 15, input section names: ['subject_id', 'CT']
-- using all subjects (without data partitioning).

INFO:niftynet: Image reader: loading 15 subjects from sections ['CT'] as input [image]

In [4]:

    

import time

num_parallel_calls = [2, 4, 8, 16]
print(num_parallel_calls)
total_times_dict = {}
batches = 10
batch_size = 100
for num_parallel_call in num_parallel_calls:
    window_sizes = {'image': (100, 100, 100), 'label': (1, 1, 1)}
    sampler = ResizeSampler(reader=image_reader,
                            csv_reader=csv_reader,
                            window_sizes=window_sizes,
                            num_threads=num_parallel_call,
                            smaller_final_batch_mode='drop',
                            batch_size=batch_size,
                            queue_length=num_parallel_call)
    next_window = sampler.pop_batch_op()
    with tf.Session() as sess:
        print('Num Parallel Calls: {}'.format(num_parallel_call))
        t0 = time.time()
        batch_times = []
        sess.run(sampler.iterator.make_initializer(sampler.dataset))
        for i in range(batches):
            try:
                value = sess.run(next_window)
                print(value['image'].shape, value['label'].shape)
            except Exception as e:
                print(e)
            batch_time = time.time() - t0
            batch_times.append(batch_time)
            print('Batch {} / {}'.format(i+1, batches))
            print('Time per batch: {}'.format(batch_time))
            t0 = time.time()
        total_times_dict[num_parallel_call] = batch_times
        print('Mean batch time: {}'.format(sum(batch_times[1:])/len(batch_times[1:])))
    if sampler.enqueuer is not None:
        sampler.enqueuer.stop()


    

    




[2, 4, 8, 16]
INFO:niftynet: reading size of preprocessed images
WARNING:niftynet: queue_length should be larger than batch_size, defaulting to batch_size * 5.0 (500).
INFO:niftynet: Initialising dataset from generator...






    




---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~/.local/lib/python3.6/site-packages/tensorflow/python/eager/execute.py in make_type(v, arg_name)
    122   try:
--> 123     v = dtypes.as_dtype(v).base_dtype
    124   except TypeError:

~/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py in as_dtype(type_value)
    670   try:
--> 671     return _INTERN_TABLE[type_value]
    672   except KeyError:

TypeError: unhashable type: 'TensorShape'

During handling of the above exception, another exception occurred:

TypeError                                 Traceback (most recent call last)
<ipython-input-4-3a863674afe1> in <module>()
     15                             batch_size=batch_size,
     16                             queue_length=num_parallel_call)
---> 17     next_window = sampler.pop_batch_op()
     18     with tf.Session() as sess:
     19         print('Num Parallel Calls: {}'.format(num_parallel_call))

~/phd/NiftyNet-Generator-PR/NiftyNet/niftynet/engine/image_window_dataset.py in pop_batch_op(self)
    212             # in case `run_threads` is not called,
    213             # here we initialise the dataset and iterator
--> 214             self.init_dataset()
    215             self.iterator = self.dataset.make_one_shot_iterator()
    216             # self.iterator = tf.data.Iterator.from_structure(

~/phd/NiftyNet-Generator-PR/NiftyNet/niftynet/engine/image_window_dataset.py in init_dataset(self)
    233             dataset = self._dataset_from_range()
    234         else:
--> 235             dataset = self._dataset_from_generator()
    236         self.dataset = self.dataset_preprocessing(dataset)
    237 

~/phd/NiftyNet-Generator-PR/NiftyNet/niftynet/engine/image_window_dataset.py in _dataset_from_generator(self)
    345             generator=window_generator,
    346             output_types=self.tf_dtypes,
--> 347             output_shapes=element_shapes)
    348         return dataset
    349 

~/.local/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py in from_generator(generator, output_types, output_shapes)
    435     # versions of the returned dataset to be created, because it forces
    436     # the generation of a new ID for each version.
--> 437     return id_dataset.flat_map(flat_map_fn)
    438 
    439   @staticmethod

~/.local/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py in flat_map(self, map_func)
    803       Dataset: A `Dataset`.
    804     """
--> 805     return FlatMapDataset(self, map_func)
    806 
    807   def interleave(self, map_func, cycle_length, block_length=1):

~/.local/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py in __init__(self, input_dataset, map_func)
   1684 
   1685     self._map_func = tf_map_func
-> 1686     self._map_func.add_to_graph(ops.get_default_graph())
   1687 
   1688   def _as_variant_tensor(self):

~/.local/lib/python3.6/site-packages/tensorflow/python/framework/function.py in add_to_graph(self, g)
    484   def add_to_graph(self, g):
    485     """Adds this function into the graph g."""
--> 486     self._create_definition_if_needed()
    487 
    488     # Adds this function into 'g'.

~/.local/lib/python3.6/site-packages/tensorflow/python/framework/function.py in _create_definition_if_needed(self)
    319     """Creates the function definition if it's not created yet."""
    320     with context.graph_mode():
--> 321       self._create_definition_if_needed_impl()
    322 
    323   def _create_definition_if_needed_impl(self):

~/.local/lib/python3.6/site-packages/tensorflow/python/framework/function.py in _create_definition_if_needed_impl(self)
    336       # Call func and gather the output tensors.
    337       with vs.variable_scope("", custom_getter=temp_graph.getvar):
--> 338         outputs = self._func(*inputs)
    339 
    340       # There is no way of distinguishing between a function not returning

~/.local/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py in tf_map_func(*args)
   1672         dataset = map_func(*nested_args)
   1673       else:
-> 1674         dataset = map_func(nested_args)
   1675 
   1676       if not isinstance(dataset, Dataset):

~/.local/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py in flat_map_fn(iterator_id_t)
    419       # relevant ID, and raises StopIteration when that iterator contains no
    420       # more elements.
--> 421       return repeated_id.map(generator_map_fn)
    422 
    423     # A single-element dataset that, each time it is evaluated, contains a

~/.local/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py in map(self, map_func, num_parallel_calls)
    788     """
    789     if num_parallel_calls is None:
--> 790       return MapDataset(self, map_func)
    791     else:
    792       return ParallelMapDataset(self, map_func, num_parallel_calls)

~/.local/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py in __init__(self, input_dataset, map_func)
   1595 
   1596     self._map_func = tf_map_func
-> 1597     self._map_func.add_to_graph(ops.get_default_graph())
   1598 
   1599   def _as_variant_tensor(self):

~/.local/lib/python3.6/site-packages/tensorflow/python/framework/function.py in add_to_graph(self, g)
    484   def add_to_graph(self, g):
    485     """Adds this function into the graph g."""
--> 486     self._create_definition_if_needed()
    487 
    488     # Adds this function into 'g'.

~/.local/lib/python3.6/site-packages/tensorflow/python/framework/function.py in _create_definition_if_needed(self)
    319     """Creates the function definition if it's not created yet."""
    320     with context.graph_mode():
--> 321       self._create_definition_if_needed_impl()
    322 
    323   def _create_definition_if_needed_impl(self):

~/.local/lib/python3.6/site-packages/tensorflow/python/framework/function.py in _create_definition_if_needed_impl(self)
    336       # Call func and gather the output tensors.
    337       with vs.variable_scope("", custom_getter=temp_graph.getvar):
--> 338         outputs = self._func(*inputs)
    339 
    340       # There is no way of distinguishing between a function not returning

~/.local/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py in tf_map_func(*args)
   1560         ret = map_func(*nested_args)
   1561       else:
-> 1562         ret = map_func(nested_args)
   1563 
   1564       # If `map_func` returns a list of tensors, `nest.flatten()` and

~/.local/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py in generator_map_fn(iterator_id_t)
    399 
    400       flat_values = script_ops.py_func(
--> 401           generator_py_func, [iterator_id_t], flattened_types, stateful=True)
    402 
    403       # The `py_func()` op drops the inferred shapes, so we add them back in

~/.local/lib/python3.6/site-packages/tensorflow/python/ops/script_ops.py in py_func(func, inp, Tout, stateful, name)
    315   """
    316   return _internal_py_func(
--> 317       func=func, inp=inp, Tout=Tout, stateful=stateful, eager=False, name=name)
    318 
    319 

~/.local/lib/python3.6/site-packages/tensorflow/python/ops/script_ops.py in _internal_py_func(func, inp, Tout, stateful, eager, name)
    223     if stateful:
    224       result = gen_script_ops._py_func(
--> 225           input=inp, token=token, Tout=Tout, name=name)
    226     else:
    227       result = gen_script_ops._py_func_stateless(

~/.local/lib/python3.6/site-packages/tensorflow/python/ops/gen_script_ops.py in _py_func(input, token, Tout, name)
     87         "Expected list for 'Tout' argument to "
     88         "'py_func' Op, not %r." % Tout)
---> 89   Tout = [_execute.make_type(_t, "Tout") for _t in Tout]
     90   _ctx = _context.context()
     91   if _ctx.in_graph_mode():

~/.local/lib/python3.6/site-packages/tensorflow/python/ops/gen_script_ops.py in <listcomp>(.0)
     87         "Expected list for 'Tout' argument to "
     88         "'py_func' Op, not %r." % Tout)
---> 89   Tout = [_execute.make_type(_t, "Tout") for _t in Tout]
     90   _ctx = _context.context()
     91   if _ctx.in_graph_mode():

~/.local/lib/python3.6/site-packages/tensorflow/python/eager/execute.py in make_type(v, arg_name)
    124   except TypeError:
    125     raise TypeError("Expected DataType for argument '%s' not %s." %
--> 126                     (arg_name, repr(v)))
    127   i = v.as_datatype_enum
    128   return i

TypeError: Expected DataType for argument 'Tout' not TensorShape([Dimension(1), Dimension(100), Dimension(100), Dimension(100), Dimension(1), Dimension(1)]).

In [ ]:

    

plt.figure()
to_plot = [2, 4, 8, 16]
means = [np.mean(total_times_dict[num][1:]) for num in to_plot]
ideal = [np.mean(total_times_dict[num][1:]) * 2 / num for num in to_plot]
plt.plot(to_plot, means, label='observed')
plt.plot(to_plot, ideal, label='ideal')
plt.title('Mean time per image as threads increases for 80 thread machine')
plt.xlabel('Threads')
plt.ylabel('mean time')
plt.legend()
plt.grid()
plt.show()


    

In [ ]: