Imports



In [1]:

    
import h5py



In [2]:

    
import numpy as np



In [4]:

    
import random



In [5]:

    
import tensorflow as tf



In [1]:

    
from PIL import Image



In [2]:

    
import matplotlib.pyplot as plt

Visualization



In [3]:

    
def show_image(npimg):
    return Image.fromarray(npimg.astype(np.uint8))



In [4]:

    
def show_normals(npnorms):
    return Image.fromarray(((npnorms+1)/2*255).astype(np.uint8))

Init function



In [8]:

    
file = '../Data/LadickyDataset10.mat'



In [9]:

    
dataset = h5py.File(file,'r')



In [11]:

    
images = np.array(dataset['images']).transpose([0,3,2,1])



In [12]:

    
normals = np.array(dataset['normals']).transpose([0,3,2,1])



In [19]:

    
images[2:].shape









    Out[19]:





(8, 480, 640, 3)



In [49]:

    
show_normals(normals[8])









    Out[49]:



In [389]:

    
show_image(images[8])









    Out[389]:

Resize



In [30]:

    
normImg.resize((320,240),resample=Image.BICUBIC)









    Out[30]:



In [37]:

    
Image.fromarray(scipy.misc.imresize(norm,0.5,interp='bicubic'))









    Out[37]:



In [42]:

    
Image.fromarray((scipy.ndimage.zoom(norm,(0.5,0.5,1),mode='nearest')*255).astype(np.uint8))









    Out[42]:

Next_Index



In [210]:

    
size = images.shape[0]
size = 3
queue = []



In [211]:

    
len(queue)









    Out[211]:





0



In [242]:

    
if (len(queue) == 0):
    queue = list(range(size))
    random.shuffle(queue)



In [245]:

    
queue.pop()









    Out[245]:





0

Get_Batch



In [255]:

    
np.empty([2,3], dtype=np.uint8)









    Out[255]:





array([[0, 0, 0],
       [0, 0, 0]], dtype=uint8)



In [265]:

    
batch_size = 3



In [310]:

    
bimages = np.empty([batch_size, height, width, 3], dtype=np.float32)
bnormals = np.empty([batch_size, height, width, 3], dtype=np.float32)
bmasks = np.empty([batch_size, height, width, 1], dtype=np.uint8)



In [311]:

    
bimages[0,:,:,:] = imgCrop

Image adjustment



In [170]:

    
img = tf.convert_to_tensor(images[6])
out = tf.image.adjust_brightness (img , 0.0)
Image.fromarray(sess.run(out))









    Out[170]:



In [303]:

    
img = tf.convert_to_tensor(images[6])
out = tf.image.adjust_contrast (img , 1.0)
Image.fromarray(sess.run(out))









    Out[303]:



In [189]:

    
img = tf.convert_to_tensor(images[6])
out = tf.image.adjust_saturation (img , 1.5)
Image.fromarray(sess.run(out))









    Out[189]:



In [191]:

    
img = tf.convert_to_tensor(images[6])
out = tf.image.adjust_saturation (img , 0.5)
Image.fromarray(sess.run(out))









    Out[191]:



In [289]:

    
img = tf.convert_to_tensor(images[6])
out = tf.image.adjust_hue (img , -0.1)
Image.fromarray(sess.run(out))









    Out[289]:



In [232]:

    
img = tf.convert_to_tensor(images[6])
flimg = tf.image.convert_image_dtype(img, tf.float32) 
mod = tf.add(flimg, 0.1)
out = tf.image.convert_image_dtype(mod, tf.uint8, saturate=True) 
show_image(sess.run(out))









    Out[232]:



In [363]:

    
change_color = tf.cast(False, tf.bool)
maxHueDelta = 0.1
maxSatDelta = 0.5
hueDelta = tf.random_uniform([], -maxHueDelta, maxHueDelta)
hueDelta = tf.cond(change_color, lambda: hueDelta, lambda: 0.0)
satDelta = tf.random_uniform([], 1.0-maxSatDelta, 1.0+maxSatDelta)
satDelta = tf.cond(change_color, lambda: satDelta, lambda: 0.0)
sess.run((hueDelta,satDelta))









    Out[363]:





(0.0, 0.0)

Build TF Graph



In [11]:

    
sess = tf.Session()



In [43]:

    
tfImgs = tf.placeholder(tf.float32, shape=(None, 480, 640, 3))
tfNorms = tf.placeholder(tf.float32, shape=(None, 480, 640, 3))



In [44]:

    
scale = tf.convert_to_tensor([1.0,1.5])
index = tf.random_uniform([1], minval=0, maxval=2, dtype=tf.int32)
rand_scale = scale[index[0]]



In [45]:

    
size = tf.cast([240*rand_scale,320*rand_scale], tf.int32)



In [46]:

    
reszImgs = tf.image.resize_images(tfImgs, size)
reszNorms = tf.image.resize_images(tfNorms, size)
normNorms = tf.nn.l2_normalize(reszNorms,3)



In [47]:

    
maxHeightIndex = (size[0] - 240)
heightIndex = tf.random_uniform([1], minval=0, maxval=maxHeightIndex+1, dtype=tf.int32)



In [48]:

    
maxWidthIndex = size[1] - 320
widthIndex = tf.random_uniform([1], minval=0, maxval=maxWidthIndex+1, dtype=tf.int32)



In [525]:

    
widthIndex.eval(session=sess)









    Out[525]:





array([0], dtype=int32)



In [50]:

    
cropImgs = tf.image.crop_to_bounding_box(reszImgs, heightIndex[0], widthIndex[0], 240, 320)
cropNorms = tf.image.crop_to_bounding_box(normNorms, heightIndex[0], widthIndex[0], 240, 320)



In [109]:

    
flipImgs = tf.reverse(cropImgs,[2])
revNorms = tf.reverse(cropNorms,[2])
flipNorms = tf.multiply([-1.0,1.0,1.0],revNorms)



In [115]:

    
rand_flip = tf.cast(tf.random_uniform([1], minval=0, maxval=2, dtype=tf.int32), tf.bool)
randFlipImgs = tf.cond(rand_flip[0], lambda: flipImgs, lambda: cropImgs)
randFlipNorms = tf.cond(rand_flip[0], lambda: flipNorms, lambda: cropNorms)



In [135]:

    
(outImgs, outNorms) = sess.run((randFlipImgs,randFlipNorms), feed_dict={tfImgs: images,tfNorms: normals})



In [136]:

    
show_image(outImgs[8])









    Out[136]:



In [137]:

    
show_normals(outNorms[8])









    Out[137]:

Class Definition



In [9]:

    
%%writefile ../Code/LadickyDataset.py
import h5py
import numpy as np
import tensorflow as tf
import random

class LadickyDataset:
    def __init__(self, file, subset=None, seed=1,
                 batch_res=(240,320), scale=(1.0,1.5), flip=True, rotation=5, color=True, brightness=True):
        '''
        subset: A list of the index of images
        '''
        
        self.dataset = h5py.File(file,'r')
        
        # Load images and normals (index, height, width, channels)
        self.images = np.array(self.dataset['images']).transpose([0,3,2,1])
        self.normals = np.array(self.dataset['normals']).transpose([0,3,2,1])

        # Dataset height and width
        self.width = self.images.shape[2]
        self.height = self.images.shape[1]
        # Output height and width
        self.batch_height = batch_res[0]
        self.batch_width = batch_res[1]
        
        #Scaling factors in respect to batch resolution
        if(min(scale) < 1.0):
            print("Error: Scaling factor is in respect to batch_res and must be greater than 1.0 ")
        else:
            self.scale = scale
        
        # Subset 
        if subset == None: # Use all images
            self.validIndices = list(range(self.images.shape[0]))
        else: # Use a subset of dataset 
            self.validIndices = subset
        
        # Number of usable samples in the dataset 
        self.size = len(self.validIndices)
        
        # Random flip
        self.flip = flip
        
        # Random color changes
        self.change_color = color
        self.maxHueDelta = 0.1
        self.maxSatDelta = 0.5
        
        # Queue for choosing the samples
        self.queue = []
        
        # Random seed
        self.seed = seed
        random.seed(seed)
        
        # Building the computional graph
        self._build_tf_graph()
    
    def _build_tf_graph(self):
        
        # Creating the tf session
        tf.set_random_seed(self.seed)
        self.sess = tf.Session()
        
        # Input placeholders
        self.tfImgs = tf.placeholder(tf.float32, shape=(None, self.height, self.width, 3))
        self.tfNorms = tf.placeholder(tf.float32, shape=(None, self.height, self.width, 3))
                
        # Scaling
        # Randomly chooses a scaling factor
        scales = tf.convert_to_tensor(self.scale)
        rand_index = tf.random_uniform([], minval=0, maxval=2, dtype=tf.int32)
        rand_scale = scales[rand_index]
        # Scales
        size = tf.cast([self.batch_height*rand_scale, self.batch_width*rand_scale], tf.int32)
        reszImgs = tf.image.resize_images(self.tfImgs, size)
        reszNorms = tf.image.resize_images(self.tfNorms, size)
        normNorms = tf.nn.l2_normalize(reszNorms,3)
        
        # Random Crop
        # Random height offset 
        maxHeightIndex = size[0] - 240
        heightIndex = tf.random_uniform([], minval=0, maxval=maxHeightIndex+1, dtype=tf.int32)
        # Random width offset
        maxWidthIndex = size[1] - 320
        widthIndex = tf.random_uniform([], minval=0, maxval=maxWidthIndex+1, dtype=tf.int32)
        # Crops
        cropImgs = tf.image.crop_to_bounding_box(reszImgs, heightIndex, widthIndex, 240, 320)
        cropNorms = tf.image.crop_to_bounding_box(normNorms, heightIndex, widthIndex, 240, 320)
        
        # Flip , this is a lazy definition, its excution depends on the rand_flip
        flipImgs = tf.reverse(cropImgs,[2])
        revNorms = tf.reverse(cropNorms,[2])
        flipNorms = tf.multiply([-1.0,1.0,1.0],revNorms)
        # Random flip
        rand_flip = tf.cast(tf.random_uniform([], minval=0, maxval=2, dtype=tf.int32), tf.bool)
        randFlipImgs = tf.cond(rand_flip, lambda: flipImgs, lambda: cropImgs) #Flip or last value
        randFlipNorms = tf.cond(rand_flip, lambda: flipNorms, lambda: cropNorms) # Flip or last value
        
        # Random color changes
        change_color = tf.cast(self.change_color, tf.bool)
        # Delta values
        hueDelta = tf.random_uniform([], -self.maxHueDelta, self.maxHueDelta)
        satFactor = tf.random_uniform([], 1.0-self.maxSatDelta, 1.0+self.maxSatDelta)     
        # Convert image RGB values to [0,1] range
        rngImgs = tf.clip_by_value(tf.divide(randFlipImgs, 255.0), 0.0, 1.0) 
        # Convert RGB images to HSV 
        hsvImgs = tf.image.rgb_to_hsv(rngImgs)
        hue = tf.slice(hsvImgs, [0, 0, 0, 0], [-1, -1, -1, 1])
        saturation = tf.slice(hsvImgs, [0, 0, 0, 1], [-1, -1, -1, 1])
        value = tf.slice(hsvImgs, [0, 0, 0, 2], [-1, -1, -1, 1])
        # Change hue and saturation
        hue = tf.cond(change_color, lambda: tf.mod(hue + (hueDelta + 1.), 1.), lambda: hue)
        saturation = tf.cond(change_color, lambda: tf.clip_by_value(saturation*satFactor, 0.0, 1.0), lambda: saturation)
        # Convert to RGB
        hsv = tf.concat([hue, saturation, value], 3)
        colorImgs = tf.image.hsv_to_rgb(hsv)
        
        # Outputs
        self.tfOutImgs = tf.image.convert_image_dtype(colorImgs, tf.uint8, saturate=True)
        self.tfOutNorms = randFlipNorms
    
    def _next_index(self):
        
        if (len(self.queue) == 0):
            self.queue = self.validIndices[:]
            random.shuffle(self.queue)
        
        return self.queue.pop()
        
    def get_batch(self, batch_size=32):
        ''' A batch with data augmentation '''
        images = np.empty([batch_size, self.height, self.width, 3], dtype=np.float32)
        normals = np.empty([batch_size, self.height, self.width, 3], dtype=np.float32)
        
        for i in range(batch_size):
            ni = self._next_index()
            images[i,:,:,:] = self.images[ni,:,:,:]
            normals[i,:,:,:] = self.normals[ni,:,:,:]
        
        (outImgs, outNorms) = self.sess.run((self.tfOutImgs,self.tfOutNorms),
                                            feed_dict={self.tfImgs: images, self.tfNorms: normals})
        return outImgs, outNorms
    
    def get_data(self, index):
        ''' Resized original data: Image and normal map'''
        
        image = self.images[index,:,:,:]
        normals = self.normals[index,:,:,:]

        tfSize = tf.constant([self.batch_height,self.batch_width], dtype=tf.int32)
        tfImg = tf.constant(image)
        tfNorms = tf.constant(normals)         
        reszImgs = tf.image.resize_images(tfImg, tfSize)
        reszNorms = tf.image.resize_images(tfNorms, tfSize)
        normNorms = tf.nn.l2_normalize(reszNorms, 2)
        (outImg, outNorms) = self.sess.run((reszImgs, normNorms))
                
        return outImg, outNorms









    



Overwriting ../Code/LadickyDataset.py

Testing the class



In [6]:

    
file = '../Data/LadickyDataset10.mat'



In [7]:

    
ds = LadickyDataset(file)



In [13]:

    
imgs, norms = ds.get_batch(4)



In [8]:

    
nrows = 2;
ncols = 2;
fig, axes = plt.subplots(nrows, ncols,figsize=(20,20))
for i in range(4):
    if (i % 2 == 0):
        imgs, norms = ds.get_batch(2)
    plt.subplot(nrows,ncols,i+1)
    plt.imshow(show_image(imgs[i%2]))
plt.show()



In [38]:

    
img, norms = ds.get_data(6)



In [39]:

    
img.shape









    Out[39]:





(240, 320, 3)



In [40]:

    
show_image(img)









    Out[40]:



In [28]:

    
show_normals(norms)









    Out[28]:



In [145]:

    
show_normals(norms[2])









    Out[145]:



In [93]:

    
dataset = LadickyDataset(file, [6,551,0,15])



In [90]:

    
len(dataset.queue) == 0









    Out[90]:





True



In [91]:

    
dataset.validIndices









    Out[91]:





[]



In [42]:

    
img , norm = ds.get_batch(4, 240, 320)



In [46]:

    
i = 0



In [47]:

    
Image.fromarray(img[i,:,:,:].astype(np.uint8))









    Out[47]:



In [48]:

    
Image.fromarray((0.5*(norm[i,:,:,:]+1)*255).astype(np.uint8))









    Out[48]:



In [ ]: