In [1]:
import sys
import os
sys.path.append(os.path.join(os.getcwd(), '../Code/'))
from LadickyDataset import *
In [2]:
import tensorflow as tf
from keras.models import Model, load_model
from keras.applications.vgg16 import VGG16
from keras.layers import Input , Flatten, Dense, Reshape, Lambda
from keras.layers.convolutional import Conv2D
In [3]:
from math import ceil
In [4]:
from PIL import Image
In [5]:
def show_image(npimg):
return Image.fromarray(npimg.astype(np.uint8))
In [6]:
def show_normals(npnorms):
return Image.fromarray(((npnorms+1)/2*255).astype(np.uint8))
In [7]:
file = '../Data/LadickyDataset.mat'
In [8]:
testNdxs = [1,2,9,14,15,16,17,18,21,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,46,47,56,57,59,60,61,62,63,76,77,78,79,84,85,86,87,88,89,90,91,117,118,119,125,126,127,128,129,131,132,133,134,137,153,154,155,167,168,169,171,172,173,174,175,176,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,207,208,209,210,211,212,220,221,222,250,264,271,272,273,279,280,281,282,283,284,285,296,297,298,299,300,301,302,310,311,312,315,316,317,325,326,327,328,329,330,331,332,333,334,335,351,352,355,356,357,358,359,360,361,362,363,364,384,385,386,387,388,389,390,395,396,397,411,412,413,414,430,431,432,433,434,435,441,442,443,444,445,446,447,448,462,463,464,465,466,469,470,471,472,473,474,475,476,477,508,509,510,511,512,513,515,516,517,518,519,520,521,522,523,524,525,526,531,532,533,537,538,539,549,550,551,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,579,580,581,582,583,591,592,593,594,603,604,605,606,607,612,613,617,618,619,620,621,633,634,635,636,637,638,644,645,650,651,656,657,658,663,664,668,669,670,671,672,673,676,677,678,679,680,681,686,687,688,689,690,693,694,697,698,699,706,707,708,709,710,711,712,713,717,718,724,725,726,727,728,731,732,733,734,743,744,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,800,801,802,803,804,810,811,812,813,814,821,822,823,833,834,835,836,837,838,839,840,841,842,843,844,845,846,850,851,852,857,858,859,860,861,862,869,870,871,906,907,908,917,918,919,926,927,928,932,933,934,935,945,946,947,959,960,961,962,965,966,967,970,971,972,973,974,975,976,977,991,992,993,994,995,1001,1002,1003,1004,1010,1011,1012,1021,1022,1023,1032,1033,1034,1038,1039,1048,1049,1052,1053,1057,1058,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1088,1089,1090,1091,1092,1093,1094,1095,1096,1098,1099,1100,1101,1102,1103,1104,1106,1107,1108,1109,1117,1118,1119,1123,1124,1125,1126,1127,1128,1129,1130,1131,1135,1136,1144,1145,1146,1147,1148,1149,1150,1151,1152,1153,1154,1155,1156,1157,1158,1162,1163,1164,1165,1166,1167,1170,1171,1174,1175,1176,1179,1180,1181,1182,1183,1184,1192,1193,1194,1195,1196,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211,1212,1216,1217,1218,1219,1220,1226,1227,1228,1229,1230,1233,1234,1235,1247,1248,1249,1250,1254,1255,1256,1257,1258,1259,1260,1261,1262,1263,1264,1265,1275,1276,1277,1278,1279,1280,1285,1286,1287,1288,1289,1290,1291,1292,1293,1294,1295,1297,1298,1299,1302,1303,1304,1305,1306,1307,1308,1314,1315,1329,1330,1331,1332,1335,1336,1337,1338,1339,1340,1347,1348,1349,1353,1354,1355,1356,1364,1365,1368,1369,1384,1385,1386,1387,1388,1389,1390,1391,1394,1395,1396,1397,1398,1399,1400,1401,1407,1408,1409,1410,1411,1412,1413,1414,1421,1422,1423,1424,1430,1431,1432,1433,1441,1442,1443,1444,1445,1446,1447,1448]
In [9]:
dataset = LadickyDataset(file, testNdxs)
In [10]:
dataset.size
Out[10]:
In [11]:
def mean_dot_product(y_true, y_pred):
dot = tf.einsum('ijkl,ijkl->ijk', y_true, y_pred) # Dot product
n = tf.cast(tf.count_nonzero(dot),tf.float32)
mean = tf.reduce_sum(dot) / n
return -1 * mean
In [12]:
def vgg16_model():
# create model
input_tensor = Input(shape=(240, 320, 3))
base_model = VGG16(input_tensor=input_tensor,weights='imagenet', include_top=False)
x = base_model.output
x = Flatten()(x)
x = Dense(4096, activation='relu', name='fc1')(x)
x = Dense(80*60*3, activation='relu', name='fc2')(x)
x = Reshape((60,80,3))(x)
x = Lambda(lambda x: tf.image.resize_bilinear(x , [240,320]) )(x)
pred = Lambda(lambda x: tf.nn.l2_normalize(x, 3) )(x)
model = Model(inputs=base_model.input, outputs=pred)
# Compile model
model.compile(loss= mean_dot_product, optimizer='sgd')
return model
In [13]:
images = np.empty([len(testNdxs), dataset.batch_height, dataset.batch_width, 3], dtype=np.float32)
normals = np.empty([len(testNdxs), dataset.batch_height, dataset.batch_width, 3], dtype=np.float32)
preds = np.empty([len(testNdxs), dataset.batch_height, dataset.batch_width, 3], dtype=np.float32)
In [14]:
model = load_model('../Data/vgg16-ladicky-model.h5', custom_objects={'mean_dot_product': mean_dot_product, 'tf':tf})
In [15]:
index = 0
for i in testNdxs:
print('Index: '+str(i))
images[index], normals[index] = dataset.get_data(i)
preds[index] = model.predict_on_batch(images[index].reshape((1,dataset.batch_height, dataset.batch_width, 3 )))
index += 1
In [17]:
for i in range(len(testNdxs)):
img = show_image(images[i])
norm = show_normals(normals[i])
pred = show_normals(preds[i])
out = Image.new('RGB', (img.size[0],3*img.size[1]))
out.paste(img.copy())
out.paste(norm.copy(), (0,norm.size[1]))
out.paste(pred.copy(), (0,norm.size[1]+pred.size[1]))
out.save('../Data/Output/'+ str(i)+'.png')
In [18]:
from scipy.io import savemat
In [19]:
savemat('../Data/pred.mat',{'Predictions': preds, 'Normals': normals})
In [1]:
%%writefile ../Code/Experiments/Prediction.py
# Imports
import tensorflow as tf
from keras.models import load_model
import numpy as np
from PIL import Image
from scipy.io import savemat
# Utility functions
def show_image(npimg):
return Image.fromarray(npimg.astype(np.uint8))
def show_normals(npnorms):
return Image.fromarray(((npnorms+1)/2*255).astype(np.uint8))
# Loss function
def mean_dot_product(y_true, y_pred):
dot = tf.einsum('ijkl,ijkl->ijk', y_true, y_pred) # Dot product
n = tf.cast(tf.count_nonzero(dot),tf.float32)
mean = tf.reduce_sum(dot) / n
return -1 * mean
# Prediction
def Predict(ID, Dataset):
# Load data set
print('Loading the data set...')
dataset = Dataset()
# Load model
print('Loading the model...')
model = load_model('Experiments/Outputs/'+ ID + '.h5', custom_objects={'mean_dot_product': mean_dot_product, 'tf':tf})
# Variables
images = np.empty([dataset.size, dataset.batch_height, dataset.batch_width, 3], dtype=np.float32)
normals = np.empty([dataset.size, dataset.batch_height, dataset.batch_width, 3], dtype=np.float32)
preds = np.empty([dataset.size, dataset.batch_height, dataset.batch_width, 3], dtype=np.float32)
# Prediction Loop
print('Normal Estimation...')
index = 0
for i in dataset.validIndices:
print('Index: '+str(i))
images[index], normals[index] = dataset.get_data(i)
preds[index] = model.predict_on_batch(images[index].reshape((1,dataset.batch_height, dataset.batch_width, 3 )))
index += 1
# Saving the result
for i in range(dataset.size):
img = show_image(images[i])
norm = show_normals(normals[i])
pred = show_normals(preds[i])
out = Image.new('RGB', (img.size[0],3*img.size[1]))
out.paste(img.copy())
out.paste(norm.copy(), (0,norm.size[1]))
out.paste(pred.copy(), (0,norm.size[1]+pred.size[1]))
out.save('Experiments/Outputs/'+ID+'/'+str(i)+'.png')
savemat('Experiments/Outputs/'+ ID + '.mat',{'Predictions': preds, 'Normals': normals})
In [ ]: