In [1]:
import os
import cv2
import numpy as np
import math
import sys

import matplotlib.pyplot as plt
%matplotlib inline

In [2]:
def rgb2gray(rgb):

    r, g, b = rgb[:,:,0], rgb[:,:,1], rgb[:,:,2]
    gray = 0.2989 * r + 0.5870 * g + 0.1140 * b

    return gray

In [3]:
img = cv2.imread('figs/front-image.jpg')

img_gray = rgb2gray(img)

plt.figure(figsize=(6,8))
plt.imshow(img_gray, cmap = plt.get_cmap('gray')) #cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
plt.show()



In [4]:
import pandas 

# opencv coordinates: origin 0,0 is at top left corner
df = pandas.read_csv('muct76-opencv.csv', header=0, usecols=np.arange(2,154), dtype=float)

X = df.iloc[:, ::2].values
Y = df.iloc[:, 1::2].values

print(X.shape, Y.shape)

df.head()


(7510, 76) (7510, 76)
Out[4]:
x00 y00 x01 y01 x02 y02 x03 y03 x04 y04 ... x71 y71 x72 y72 x73 y73 x74 y74 x75 y75
0 201.0 348.0 201.0 381.0 202.0 408.0 209.0 435.0 224.0 461.0 ... 235.5 348.5 338.0 333.5 324.0 335.5 326.0 342.5 340.0 340.5
1 162.0 357.0 157.0 387.0 160.0 418.0 167.0 446.0 182.0 477.0 ... 202.5 359.0 305.5 346.0 291.5 348.0 292.0 354.5 306.0 352.5
2 212.0 352.0 203.0 380.0 200.0 407.0 211.0 439.0 224.0 479.0 ... 257.5 355.0 344.5 343.5 334.0 345.0 335.0 351.0 345.5 349.5
3 157.0 316.0 155.0 348.0 154.0 373.0 159.0 407.0 172.0 435.0 ... 192.0 319.0 295.5 311.5 280.0 313.0 282.0 320.0 297.5 318.5
4 201.0 373.0 200.0 408.0 203.0 433.0 213.0 463.0 226.0 481.0 ... 237.0 377.5 335.5 366.5 322.0 369.0 324.5 375.0 338.0 372.5

5 rows × 152 columns


In [39]:
img_gray = rgb2gray(img)

for x, y in zip(X[0,:], Y[0,:]):
    cv2.circle(img_gray, tuple((int(x),int(y))), 4, color=145, thickness=-1)

plt.figure(figsize=(8,6))
plt.imshow(img_gray, cmap = plt.get_cmap('gray')) #cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
plt.show()


Delauny Triangulation


In [27]:
import scipy.spatial

xh = img.shape[1]-1
yh = img.shape[0]-1
xm = np.int(xh/2)
ym = np.int(yh/2)
points = np.array([[0, 0], [xh,0], 
                   [0, yh], [xh, yh],
                   [xm, 0], [xm, yh], 
                   [0, ym], [xh, ym]
                  ])


points = np.concatenate((points, np.vstack ((X[0,:], Y[0,:])).T))
print(points.shape)

tri = scipy.spatial.Delaunay(points)

plt.figure(figsize=(13,8))
plt.subplot(1, 2, 1)
plt.triplot(points[:,0], points[:,1], tri.simplices.copy())
plt.plot(points[:,0], points[:,1], 'o')

imgcopy = img.copy()

plt.subplot(1, 2, 2)
points = points.astype(int)
for i,j,k in tri.simplices:
    cv2.line(imgcopy, tuple(points[i,:]), tuple(points[j,:]), color=(100,255,255), thickness=2)
    cv2.line(imgcopy, tuple(points[i,:]), tuple(points[k,:]), color=(100,255,255), thickness=2)
    cv2.line(imgcopy, tuple(points[j,:]), tuple(points[k,:]), color=(100,255,255), thickness=2)
    cv2.circle(imgcopy, tuple(points[i,:]), 4, color=(0,50,200), thickness=-1)
    cv2.circle(imgcopy, tuple(points[j,:]), 4, color=(0,50,200), thickness=-1)
    cv2.circle(imgcopy, tuple(points[k,:]), 4, color=(0,50,200), thickness=-1)
    
plt.imshow(cv2.cvtColor(imgcopy, cv2.COLOR_BGR2RGB))
plt.show()


(84, 2)

In [20]:
tri.simplices.shape


Out[20]:
(158, 3)

In [8]:
meanface = np.load('models/meanshape.pkl')
Phi_matrix = np.load('models/eigenvectors.pkl')[:,:2]
eig_values = np.load('models/eigenvalues.pkl')

print(meanface.shape)
print(Phi_matrix.shape)
print(eig_values.shape)


(152,)
(152, 2)
(152,)

In [22]:
def construct_newface(origface, Phi_matrix, b):
    print(origface.shape)
    face = origface.flatten('F') + np.dot(Phi_matrix, b )
    
    return (face[:76], face[76:])


b = np.zeros(shape=2)
for i,j in enumerate((-0.015, -0.005, 0.005, 0.002)):
    b[0] = j*eig_values[0]
    newface_x, newface_y = construct_newface(points[8:,:], Phi_matrix, b)
    plt.subplot(2, 2, i+1)
    plt.plot(newface_x, newface_y)
    
plt.show()


(76, 2)
(76, 2)
(76, 2)
(76, 2)

In [49]:
def draw_shape(img, points, color):
    imgout = img.copy()
    prev_point = points[0]
    for p in points[1:,:]:
        cv2.line(imgout, tuple(prev_point), tuple(p), 
                 color=color, thickness=2)
        prev_point = p
    return imgout

plt.figure(figsize=(6,8))
plt.imshow(cv2.cvtColor(draw_shape(img, points[8:,:], color=(100,255,255)), cv2.COLOR_BGR2RGB))
plt.show()



In [73]:
def transformTriangle(img, tri1, tri2):
    """tri1 and tri2 are the coordinates of 
       the same triangle in two images"""
    
    r1 = cv2.boundingRect(tri1) # bounding box 1
    r2 = cv2.boundingRect(tri2) # bounding box 2
    tri1Cropped = np.vstack((tri1[:,0] - r1[0], tri1[:,1]-r1[1])).T
    tri2Cropped = np.vstack((tri2[:,0] - r2[0], tri2[:,1]-r2[1])).T
    img1Cropped = img[r1[1]:r1[1]+r1[3], r2[0]:r2[0]+r2[2]].copy()
    
    warpMat = cv2.getAffineTransform( np.float32(tri1Cropped), np.float32(tri2Cropped) )
    
    img2Cropped = cv2.warpAffine( img1Cropped, warpMat, 
                                 (r2[2], r2[3]), None, 
                                 flags=cv2.INTER_LINEAR, 
                                 borderMode=cv2.BORDER_REFLECT_101 )
    # Get mask by filling triangle
    mask = np.zeros((r2[3], r2[2], 3), dtype = np.float32)
    cv2.fillConvexPoly(mask, np.int32(tri2Cropped), (1.0, 1.0, 1.0), 16, 0);
     
    # Apply mask to cropped region
    img2Cropped = img2Cropped * mask
 
    # Copy triangular region of the rectangular patch to the output image
    #img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = \
    #                    img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] \
    #                    * ( (1.0, 1.0, 1.0) - mask ) 
    #img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = \
    #                    * img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] \
    #                 + img2Cropped
    
    #plt.figure(figsize=(6,8))
    #plt.imshow(cv2.cvtColor(img2Cropped, cv2.COLOR_BGR2RGB))
    #plt.show()
    return (img2Cropped, r2, mask, img1Cropped, r1)
    
img2crop = transformTriangle(img, tcoors1, tcoors2)


---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-73-40f922677260> in <module>()
     35     return (img2Cropped, r2, mask, img1Cropped, r1)
     36 
---> 37 img2crop = transformTriangle(img, tcoors1, tcoors2)

<ipython-input-73-40f922677260> in transformTriangle(img, tri1, tri2)
     14                                  (r2[2], r2[3]), None,
     15                                  flags=cv2.INTER_LINEAR,
---> 16                                  blendMode=cv2.BORDER_REFLECT_101 )
     17     # Get mask by filling triangle
     18     mask = np.zeros((r2[3], r2[2], 3), dtype = np.float32)

TypeError: 'blendMode' is an invalid keyword argument for this function

In [50]:
plt.figure(figsize=(15,30))

for k,p in enumerate((0.002, 0.004, 0.008, 0.010)):
    b = np.zeros(shape=2)
    b[1] = p*eig_values[0]
    newface_x, newface_y = construct_newface(points[8:,:], Phi_matrix, b)

    origface = points[:,:]
    newface = np.concatenate((points[:8,:], np.vstack((newface_x, newface_y)).T.astype(int)))

    print(origface.shape)
    print(newface.shape)

    img2 = 255*np.ones(img.shape, img.dtype)

    for i,indx in enumerate(tri.simplices[:,:]):

        tcoors1 = origface[indx, :]

        tcoors2 = newface[indx, :]
        img2crop, r2, mask, img1crop, r1 = transformTriangle(img, tcoors1, tcoors2)
        img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = \
        img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] \
                        * ( (1.0, 1.0, 1.0) - mask ) 
        img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = \
                        img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] \
                        + img2crop
            
    
    img1 = img.copy()    
    plt.subplot(4, 3, 3*k+1)
    plt.imshow(cv2.cvtColor(img1, cv2.COLOR_BGR2RGB))
    
    img3 = draw_shape(img1, origface[8:,:], color=(100,255,255))
    img3 = draw_shape(img3, newface[8:,:], color=(100,100,255))
    plt.subplot(4, 3, 3*k+2)
    plt.imshow(cv2.cvtColor(img3, cv2.COLOR_BGR2RGB))
    
    plt.subplot(4, 3, 3*k+3)
    plt.imshow(cv2.cvtColor(img2, cv2.COLOR_BGR2RGB))
    
plt.show()


(76, 2)
(84, 2)
(84, 2)
(76, 2)
(84, 2)
(84, 2)
(76, 2)
(84, 2)
(84, 2)
(76, 2)
(84, 2)
(84, 2)

Convex Hull


In [71]:
hullIndex = cv2.convexHull(origface[8:,:], returnPoints = False).flatten()

print(hullIndex)

plt.figure(figsize=(6,8))
plt.imshow(cv2.cvtColor(draw_shape(img, origface[8:,:][hullIndex], 
                                   color=(100,255,255)), cv2.COLOR_BGR2RGB))
plt.show()


[12 11 10  9  8  7  6  5  4  3  2  1  0 21 22 16 15 13]

In [72]:
def triangulateFace(img, points):
    tri = scipy.spatial.Delaunay(origface[8:,:])

    plt.figure(figsize=(13,8))
    plt.subplot(1, 2, 1)
    plt.triplot(points[:,0], points[:,1], tri.simplices.copy())
    plt.plot(points[:,0], points[:,1], 'o')

    imgcopy = img.copy()

    plt.subplot(1, 2, 2)
    points = points.astype(int)
    for i,j,k in tri.simplices:
        cv2.line(imgcopy, tuple(points[i,:]), tuple(points[j,:]), color=(100,255,255), thickness=2)
        cv2.line(imgcopy, tuple(points[i,:]), tuple(points[k,:]), color=(100,255,255), thickness=2)
        cv2.line(imgcopy, tuple(points[j,:]), tuple(points[k,:]), color=(100,255,255), thickness=2)
        cv2.circle(imgcopy, tuple(points[i,:]), 4, color=(0,50,200), thickness=-1)
        cv2.circle(imgcopy, tuple(points[j,:]), 4, color=(0,50,200), thickness=-1)
        cv2.circle(imgcopy, tuple(points[k,:]), 4, color=(0,50,200), thickness=-1)
    
    plt.imshow(cv2.cvtColor(imgcopy, cv2.COLOR_BGR2RGB))
    plt.show()
    
triangulateFace(img, origface[8:,:])



In [ ]: