In [1]:
import os
import cv2
import numpy as np
import math
import sys
import matplotlib.pyplot as plt
%matplotlib inline
In [2]:
def rgb2gray(rgb):
r, g, b = rgb[:,:,0], rgb[:,:,1], rgb[:,:,2]
gray = 0.2989 * r + 0.5870 * g + 0.1140 * b
return gray
In [3]:
img = cv2.imread('figs/front-image.jpg')
img_gray = rgb2gray(img)
plt.figure(figsize=(6,8))
plt.imshow(img_gray, cmap = plt.get_cmap('gray')) #cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
plt.show()
In [4]:
import pandas
# opencv coordinates: origin 0,0 is at top left corner
df = pandas.read_csv('muct76-opencv.csv', header=0, usecols=np.arange(2,154), dtype=float)
X = df.iloc[:, ::2].values
Y = df.iloc[:, 1::2].values
print(X.shape, Y.shape)
df.head()
Out[4]:
In [39]:
img_gray = rgb2gray(img)
for x, y in zip(X[0,:], Y[0,:]):
cv2.circle(img_gray, tuple((int(x),int(y))), 4, color=145, thickness=-1)
plt.figure(figsize=(8,6))
plt.imshow(img_gray, cmap = plt.get_cmap('gray')) #cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
plt.show()
In [27]:
import scipy.spatial
xh = img.shape[1]-1
yh = img.shape[0]-1
xm = np.int(xh/2)
ym = np.int(yh/2)
points = np.array([[0, 0], [xh,0],
[0, yh], [xh, yh],
[xm, 0], [xm, yh],
[0, ym], [xh, ym]
])
points = np.concatenate((points, np.vstack ((X[0,:], Y[0,:])).T))
print(points.shape)
tri = scipy.spatial.Delaunay(points)
plt.figure(figsize=(13,8))
plt.subplot(1, 2, 1)
plt.triplot(points[:,0], points[:,1], tri.simplices.copy())
plt.plot(points[:,0], points[:,1], 'o')
imgcopy = img.copy()
plt.subplot(1, 2, 2)
points = points.astype(int)
for i,j,k in tri.simplices:
cv2.line(imgcopy, tuple(points[i,:]), tuple(points[j,:]), color=(100,255,255), thickness=2)
cv2.line(imgcopy, tuple(points[i,:]), tuple(points[k,:]), color=(100,255,255), thickness=2)
cv2.line(imgcopy, tuple(points[j,:]), tuple(points[k,:]), color=(100,255,255), thickness=2)
cv2.circle(imgcopy, tuple(points[i,:]), 4, color=(0,50,200), thickness=-1)
cv2.circle(imgcopy, tuple(points[j,:]), 4, color=(0,50,200), thickness=-1)
cv2.circle(imgcopy, tuple(points[k,:]), 4, color=(0,50,200), thickness=-1)
plt.imshow(cv2.cvtColor(imgcopy, cv2.COLOR_BGR2RGB))
plt.show()
In [20]:
tri.simplices.shape
Out[20]:
In [8]:
meanface = np.load('models/meanshape.pkl')
Phi_matrix = np.load('models/eigenvectors.pkl')[:,:2]
eig_values = np.load('models/eigenvalues.pkl')
print(meanface.shape)
print(Phi_matrix.shape)
print(eig_values.shape)
In [22]:
def construct_newface(origface, Phi_matrix, b):
print(origface.shape)
face = origface.flatten('F') + np.dot(Phi_matrix, b )
return (face[:76], face[76:])
b = np.zeros(shape=2)
for i,j in enumerate((-0.015, -0.005, 0.005, 0.002)):
b[0] = j*eig_values[0]
newface_x, newface_y = construct_newface(points[8:,:], Phi_matrix, b)
plt.subplot(2, 2, i+1)
plt.plot(newface_x, newface_y)
plt.show()
In [49]:
def draw_shape(img, points, color):
imgout = img.copy()
prev_point = points[0]
for p in points[1:,:]:
cv2.line(imgout, tuple(prev_point), tuple(p),
color=color, thickness=2)
prev_point = p
return imgout
plt.figure(figsize=(6,8))
plt.imshow(cv2.cvtColor(draw_shape(img, points[8:,:], color=(100,255,255)), cv2.COLOR_BGR2RGB))
plt.show()
In [73]:
def transformTriangle(img, tri1, tri2):
"""tri1 and tri2 are the coordinates of
the same triangle in two images"""
r1 = cv2.boundingRect(tri1) # bounding box 1
r2 = cv2.boundingRect(tri2) # bounding box 2
tri1Cropped = np.vstack((tri1[:,0] - r1[0], tri1[:,1]-r1[1])).T
tri2Cropped = np.vstack((tri2[:,0] - r2[0], tri2[:,1]-r2[1])).T
img1Cropped = img[r1[1]:r1[1]+r1[3], r2[0]:r2[0]+r2[2]].copy()
warpMat = cv2.getAffineTransform( np.float32(tri1Cropped), np.float32(tri2Cropped) )
img2Cropped = cv2.warpAffine( img1Cropped, warpMat,
(r2[2], r2[3]), None,
flags=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_REFLECT_101 )
# Get mask by filling triangle
mask = np.zeros((r2[3], r2[2], 3), dtype = np.float32)
cv2.fillConvexPoly(mask, np.int32(tri2Cropped), (1.0, 1.0, 1.0), 16, 0);
# Apply mask to cropped region
img2Cropped = img2Cropped * mask
# Copy triangular region of the rectangular patch to the output image
#img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = \
# img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] \
# * ( (1.0, 1.0, 1.0) - mask )
#img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = \
# * img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] \
# + img2Cropped
#plt.figure(figsize=(6,8))
#plt.imshow(cv2.cvtColor(img2Cropped, cv2.COLOR_BGR2RGB))
#plt.show()
return (img2Cropped, r2, mask, img1Cropped, r1)
img2crop = transformTriangle(img, tcoors1, tcoors2)
In [50]:
plt.figure(figsize=(15,30))
for k,p in enumerate((0.002, 0.004, 0.008, 0.010)):
b = np.zeros(shape=2)
b[1] = p*eig_values[0]
newface_x, newface_y = construct_newface(points[8:,:], Phi_matrix, b)
origface = points[:,:]
newface = np.concatenate((points[:8,:], np.vstack((newface_x, newface_y)).T.astype(int)))
print(origface.shape)
print(newface.shape)
img2 = 255*np.ones(img.shape, img.dtype)
for i,indx in enumerate(tri.simplices[:,:]):
tcoors1 = origface[indx, :]
tcoors2 = newface[indx, :]
img2crop, r2, mask, img1crop, r1 = transformTriangle(img, tcoors1, tcoors2)
img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = \
img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] \
* ( (1.0, 1.0, 1.0) - mask )
img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = \
img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] \
+ img2crop
img1 = img.copy()
plt.subplot(4, 3, 3*k+1)
plt.imshow(cv2.cvtColor(img1, cv2.COLOR_BGR2RGB))
img3 = draw_shape(img1, origface[8:,:], color=(100,255,255))
img3 = draw_shape(img3, newface[8:,:], color=(100,100,255))
plt.subplot(4, 3, 3*k+2)
plt.imshow(cv2.cvtColor(img3, cv2.COLOR_BGR2RGB))
plt.subplot(4, 3, 3*k+3)
plt.imshow(cv2.cvtColor(img2, cv2.COLOR_BGR2RGB))
plt.show()
In [71]:
hullIndex = cv2.convexHull(origface[8:,:], returnPoints = False).flatten()
print(hullIndex)
plt.figure(figsize=(6,8))
plt.imshow(cv2.cvtColor(draw_shape(img, origface[8:,:][hullIndex],
color=(100,255,255)), cv2.COLOR_BGR2RGB))
plt.show()
In [72]:
def triangulateFace(img, points):
tri = scipy.spatial.Delaunay(origface[8:,:])
plt.figure(figsize=(13,8))
plt.subplot(1, 2, 1)
plt.triplot(points[:,0], points[:,1], tri.simplices.copy())
plt.plot(points[:,0], points[:,1], 'o')
imgcopy = img.copy()
plt.subplot(1, 2, 2)
points = points.astype(int)
for i,j,k in tri.simplices:
cv2.line(imgcopy, tuple(points[i,:]), tuple(points[j,:]), color=(100,255,255), thickness=2)
cv2.line(imgcopy, tuple(points[i,:]), tuple(points[k,:]), color=(100,255,255), thickness=2)
cv2.line(imgcopy, tuple(points[j,:]), tuple(points[k,:]), color=(100,255,255), thickness=2)
cv2.circle(imgcopy, tuple(points[i,:]), 4, color=(0,50,200), thickness=-1)
cv2.circle(imgcopy, tuple(points[j,:]), 4, color=(0,50,200), thickness=-1)
cv2.circle(imgcopy, tuple(points[k,:]), 4, color=(0,50,200), thickness=-1)
plt.imshow(cv2.cvtColor(imgcopy, cv2.COLOR_BGR2RGB))
plt.show()
triangulateFace(img, origface[8:,:])
In [ ]: