In [1]:
import numpy as np
import cv2
import matplotlib
from matplotlib import pyplot as plt
%matplotlib inline
import pandas
def simTransform(pref, pcmp, showerror = False):
err_before = np.mean(np.sum(np.square(pref - pcmp), axis=1))
ref_mean = np.mean(pref, axis=0)
prefcentered = np.asmatrix(pref) - np.asmatrix(ref_mean)
cmp_mean = np.mean(pcmp, axis=0)
pcmpcentered = np.asmatrix(pcmp) - np.asmatrix(cmp_mean)
Sxx = np.sum(np.square(pcmpcentered[:,0]))
Syy = np.sum(np.square(pcmpcentered[:,1]))
Sxxr = prefcentered[:,0].T * pcmpcentered[:,0] #(ref_x, x)
Syyr = prefcentered[:,1].T * pcmpcentered[:,1] #(ref_y, y)
Sxyr = prefcentered[:,1].T * pcmpcentered[:,0] #(ref_y, x)
Syxr = prefcentered[:,0].T * pcmpcentered[:,1] #(ref_x, y)
a = (Sxxr + Syyr)/(Sxx + Syy) #(Sxxr + Syyr) / (Sxx + Syy)
b = (Sxyr - Syxr) / (Sxx + Syy)
a = np.asscalar(a)
b = np.asscalar(b)
Rot = np.matrix([[a, -b],[b, a]])
translation = -Rot * np.asmatrix(cmp_mean).T + np.asmatrix(ref_mean).T
outx, outy = [], []
res = Rot * np.asmatrix(pcmp).T + translation
if showerror:
err_after = np.mean(np.sum(np.square(pref - res.T), axis=1))
print("Error before: %.4f after: %.4f\n"%(err_before, err_after))
return np.asarray(res.T)
def draw_shape(img, points, color, closed=False):
#print(points)
imgout = img.copy()
prev_point = points[0,:]
for p in points[1:,:]:
cv2.line(imgout, tuple(prev_point), tuple(p),
color=color, thickness=2)
prev_point = p
if closed:
cv2.line(imgout, tuple(prev_point), tuple(points[0,:]),
color=color, thickness=2)
return imgout
def genMask(src, tri):
mask = np.zeros(src.shape, dtype = src.dtype)
cv2.fillConvexPoly(mask, np.int32(tri), (1.0, 1.0, 1.0), 16, 0);
return mask
################################
# Apply affine transform calculated using srcTri and dstTri to src and
# output an image of size.
def applyAffineTransform(src, srcTri, dstTri, size) :
# Given a pair of triangles, find the affine transform.
warpMat = cv2.getAffineTransform( np.float32(srcTri), np.float32(dstTri) )
#rbound = cv2.boundingRect(np.float32([dstTri]))
#print(rbound)
#size = (rbound[2], rbound[3])
# Apply the Affine Transform just found to the src image
dst = cv2.warpAffine( src, warpMat, (size[0], size[1]), None, flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101 )
return dst
def warpTriangle(img1, img2, t1, t2):
""" Warping triangular patch from image 1 onto image 2
"""
r1 = cv2.boundingRect(np.float32([t1]))
r2 = cv2.boundingRect(np.float32([t2]))
t1_local, t2_local = [], []
for i in range(3):
t1_local.append(((t1[i][0] - r1[0]), t1[i][1] - r1[1]))
t2_local.append(((t2[i][0] - r2[0]), t2[i][1] - r2[1]))
#mask1 = np.zeros((r1[3], r1[2], 3), dtype = np.float32)
mask_t2 = np.zeros((r2[3], r2[2], 3), dtype = np.float32)
#cv2.fillConvexPoly(mask1, np.int32(t1_local), (1.0, 1.0, 1.0), 16, 0)
cv2.fillConvexPoly(mask_t2, np.int32(t2_local), (1.0, 1.0, 1.0), 16, 0)
x1left, x1right, y1left, y1right = r1[1], r1[1]+r1[3], r1[0], r1[0]+r1[2]
x2left, x2right, y2left, y2right = r2[1], r2[1]+r2[3], r2[0], r2[0]+r2[2]
img1_patch = img1[x1left:x1right, y1left:y1right]
img2_patch = img2[x2left:x2right, y2left:y2right]
img1_patch_warped = applyAffineTransform(img1_patch, t1_local, t2_local, (r2[2], r2[3]))
#img2_patch_warped = applyAffineTransform(img2_patch, t2_local, t1_local, (r1[2], r1[3]))
#print(mask2.shape, img1_patch.shape, img1_patch_warped.shape)
img1_patch_warped = img1_patch_warped * mask_t2
#img2_patch_warped = img2_patch_warped * mask1
#out1[x1left:x1right, y1left:y1right] = img1_patch * ( (1.0, 1.0, 1.0) - mask1)
img2[x2left:x2right, y2left:y2right] = img2_patch * ( (1.0, 1.0, 1.0) - mask_t2)
#out1[x1left:x1right, y1left:y1right] = img1_patch + img2_patch_warped
img2[x2left:x2right, y2left:y2right] = img2_patch + img1_patch_warped
################################
# Check if a point is inside a rectangle
def rectContains(rect, point) :
if point[0] < rect[0] :
return False
elif point[1] < rect[1] :
return False
elif point[0] > rect[0] + rect[2] :
return False
elif point[1] > rect[1] + rect[3] :
return False
return True
#calculate delanauy triangle
def calculateDelaunayTriangles(rect, points):
#create subdiv
subdiv = cv2.Subdiv2D(rect);
# Insert points into subdiv
for p in points:
subdiv.insert(p)
triangleList = subdiv.getTriangleList();
delaunayTri = []
pt = []
count= 0
for t in triangleList:
pt.append((t[0], t[1]))
pt.append((t[2], t[3]))
pt.append((t[4], t[5]))
pt1 = (t[0], t[1])
pt2 = (t[2], t[3])
pt3 = (t[4], t[5])
if rectContains(rect, pt1) and rectContains(rect, pt2) and rectContains(rect, pt3):
count = count + 1
ind = []
for j in range(0, 3):
for k in range(0, len(points)):
if(abs(pt[j][0] - points[k][0]) < 1.0 and abs(pt[j][1] - points[k][1]) < 1.0):
ind.append(k)
if len(ind) == 3:
delaunayTri.append((ind[0], ind[1], ind[2]))
pt = []
return delaunayTri
def CloneSeamless(obj, srcimg, hullpoints):
mask = np.zeros(obj.shape, dtype = obj.dtype)
hull = tuple(map(tuple, hullpoints))
cv2.fillConvexPoly(mask, np.int32(hull), (255, 255, 255))
print(hull)
r = cv2.boundingRect(np.float32([hullpoints]))
center = ((r[0]+int(r[2]/2), r[1]+int(r[3]/2)))
output = cv2.seamlessClone(np.uint8(obj), srcimg, mask, center, cv2.NORMAL_CLONE)
return output
In [2]:
df = pandas.read_csv('muct76-opencv.csv', header=0, usecols=np.arange(2,154), dtype=float)
df.head()
Out[2]:
In [3]:
X = df.iloc[:, ::2].values
Y = df.iloc[:, 1::2].values
print(X.shape, Y.shape)
d = np.hstack((X,Y))
print(d.shape)
In [4]:
img1 = cv2.imread('figs/front-image.jpg')
points1 = np.vstack ((X[0,:], Y[0,:])).T # i000qa-fn
points1 = points1.astype(int)
### Plot
plt.figure(figsize=(7,7))
img1shape = draw_shape(img1, points1[:,:], color=(100,255,255))
plt.imshow(cv2.cvtColor(img1shape, cv2.COLOR_BGR2RGB))
plt.show()
In [5]:
x0 = 5
x1 = img1.shape[1]/3-5
x2 = img1.shape[1]*2/3-5
x3 = img1.shape[1]-5
y0 = 5
y1 = img1.shape[0]/4-5
y2 = img1.shape[0]/2-5
y3 = img1.shape[0]*3/4-5
y4 = img1.shape[0]-5
imgborders = np.array([(x0,y0), (x0,y1), (x0,y2), (x0,y3), (x0,y4),
(x3,y0), (x3,y1), (x3,y2), (x3,y3), (x3,y4),
(x1,y0), (x2,y0), (x1,y4), (x2,y4)])
imgborders = imgborders.astype(int)
print(imgborders.shape)
allpoints = np.vstack((points1, imgborders))
print(allpoints.shape)
In [7]:
meanface = np.load('models/meanshape-ocvfmt.pkl')
meanface_transformed = simTransform(points1, meanface).astype(int)
print(meanface_transformed.shape)
allpoints_mean = np.vstack((meanface_transformed, imgborders)).astype(int)
allpoints_mean = np.asarray(allpoints_mean)
print(allpoints_mean.shape)
In [8]:
sizeImg1 = img1.shape
rect = (0, 0, sizeImg1[1], sizeImg1[0])
trilist = calculateDelaunayTriangles(rect, tuple(map(tuple, allpoints)))
tmp1 = img1.copy()
tmp2 = img1.copy()
for tri in trilist:
tp1 = allpoints[tri, :]
tp2 = allpoints_mean[tri, :]
tmp1 = draw_shape(tmp1, tp1, (10,255,255), True)
tmp2 = draw_shape(tmp2, tp2, (10,255,255), True)
plt.figure(figsize=(10,16))
plt.subplot(1,2, 1)
plt.imshow(cv2.cvtColor(tmp1, cv2.COLOR_BGR2RGB))
plt.subplot(1,2, 2)
plt.imshow(cv2.cvtColor(tmp2, cv2.COLOR_BGR2RGB))
plt.show()
In [9]:
res = img1.copy()
for tri in trilist:
tp1 = allpoints[np.array(tri),:]
tp2 = allpoints_mean[np.array(tri),:]
warpTriangle(img1, res, tp1, tp2)
plt.figure(figsize=(10,12))
plt.subplot(1,2, 1)
plt.imshow(cv2.cvtColor(img1 , cv2.COLOR_BGR2RGB))
plt.subplot(1,2, 2)
plt.imshow(cv2.cvtColor(res, cv2.COLOR_BGR2RGB))
plt.show()
In [10]:
hull = []
hullIndex = cv2.convexHull(meanface_transformed, returnPoints = False)
for i in range(0, len(hullIndex)):
hull.append(tuple(points1[hullIndex.flatten()[i]]))
out = CloneSeamless(res, img1, hull)
plt.figure(figsize=(12,10))
plt.subplot(1,3, 1)
plt.imshow(cv2.cvtColor(img1 , cv2.COLOR_BGR2RGB))
plt.subplot(1,3, 2)
plt.imshow(cv2.cvtColor(res , cv2.COLOR_BGR2RGB))
plt.subplot(1,3, 3)
plt.imshow(cv2.cvtColor(out, cv2.COLOR_BGR2RGB))
plt.show()
In [15]:
Phi_matrix = np.matrix(np.load('models/phimatrix.pkl'))
meanvector = np.matrix(np.load('models/meanvector.pkl'))
alignedfaces = np.matrix(np.load('models/alignedfaces.pkl'))
bvector = np.matrix(np.load('models/bvector.pkl'))
print("Phi: ", Phi_matrix.shape)
print("mean: ", meanvector.shape)
print("alignedfaces: ", alignedfaces.shape)
print("b: ", bvector.shape)
In [13]:
bvector[0,:]
Out[13]:
In [20]:
new_shape = meanvector + Phi_matrix * bvector[0,:].T
In [55]:
plt.figure()
plt.plot(np.array(alignedfaces[2,:76])[0], np.array(alignedfaces[2,76:])[0])
plt.show()
In [42]:
np.array(alignedfaces[0,:76])[0]
Out[42]:
In [ ]: