• Facial Feature Detection & Matching
  • Triangulation
  • Warping & Cross-Dissolve
  • Generate Morphing Sequence
In [2]:
cd /Users/james/Desktop/四春/A5图像处理/大作业1/第一次大作业/image_morphing/

/Users/james/Desktop/四春/A5图像处理/大作业1/第一次大作业/image_morphing
In [3]:
import urllib.request
import urllib.error
import time
import json
import numpy as np
import cv2
from os import path
import matplotlib.pyplot as plt
%pylab inline
from tqdm import tqdm_notebook as tqdm
import sys
sys.path.append('bin/')

Populating the interactive namespace from numpy and matplotlib

load template

In [4]:
# setup figure template
figure_template_path = 'bin/'
if figure_template_path not in sys.path:
    sys.path.append(figure_template_path)
from importlib import reload
import utils
# force reload of the module
reload(utils)
from utils import std_plot,display_dataframe, embed_pdf_figure, embed_pdf_pages,rect_contains
fontlegend = {'family':'Arial',
                  'weight' : 'normal', 
              #'linewidth':0.5,
                  'size' : 6.5*1}

key points detection

use Face++ to detect human faces and self annotation of lion face

In [5]:
detected_points_source1 = np.loadtxt('detected_points_source1.txt').astype('int')
detected_points_source2 = np.loadtxt('detected_points_source2.txt').astype('int')
detected_points_target1 = np.loadtxt('detected_points_target1.txt').astype('int')
detected_points_target2 = np.loadtxt('detected_points_target2.txt').astype('int')
detected_points_source2_ = np.loadtxt('detected_points_source2_.txt').astype('int')
detected_points_target2_ = np.loadtxt('detected_points_target2_.txt').astype('int')
source1 = cv2.imread('source1.png')
source2 = cv2.imread('source2.png')
target1 = cv2.imread('target1.png')
target2 = cv2.imread('target2.png')
In [6]:
images = [source1,target1,source2,target2]
detected_points = [detected_points_source1,detected_points_target1,detected_points_source2,detected_points_target2]
detected_points_ = [detected_points_source1,detected_points_target1,detected_points_source2_,detected_points_target2_]
In [7]:
def plot_key_points(image,detected_point):
    fig,ax=plt.subplots(1,3,figsize=(15,6.5))
    ax[0].imshow(image[:,:,::-1],aspect="auto") 
    ax[1].scatter(detected_point[:,1],detected_point[:,0],s=20,edgecolors='black',color='green',alpha=0.6,marker='*')
    ax[1].set_xlim(0,image.shape[1])
    ax[1].set_ylim(0,image.shape[0])
    ax[1].invert_yaxis()
    ax[2].imshow(image[:,:,::-1],aspect="auto")
    ax[2].scatter(detected_point[:,1],detected_point[:,0],s=20,edgecolors='black',color='green',alpha=0.6,marker='o')
    fig.tight_layout()
    embed_pdf_figure()
    fig.savefig('')
In [8]:
for image,detected_point in zip(images,detected_points):
    plot_key_points(image,detected_point)

triangulation

other methods

In [8]:
from scipy.spatial import Delaunay
In [9]:
def plot_triangualation(image,detected_points,points_for_triang,triangularized_points):

    # Triangle Settings
    width = image.shape[1]
    height =image.shape[0]

    center = np.sum(points_for_triang[triangularized_points], axis=1)/3.0
    color = np.array([(x - width/2)**2 + (y - height/2)**2 for x, y in center])

    fig,ax=plt.subplots(1,4,figsize=(20,6.5))
    ax[0].imshow(image[:,:,::-1],aspect="auto") 
    ax[1].scatter(detected_points[:,1],detected_points[:,0],s=20,edgecolors='black',color='green',alpha=0.6,marker='*')
    ax[1].set_xlim(0,image.shape[1])
    ax[1].set_ylim(0,image.shape[0])
    ax[1].invert_yaxis()
    ax[2].imshow(image[:,:,::-1],aspect="auto")
    ax[2].scatter(detected_points[:,1],detected_points[:,0],s=20,edgecolors='black',color='green',alpha=0.6,marker='o')

    ax[3].tripcolor(points_for_triang[:, 0], points_for_triang[:, 1], triangularized_points, facecolors=color,alpha=0.3, edgecolors='k')
    ax[3].set_xlim(0,image.shape[1])
    ax[3].set_ylim(0,image.shape[0])
    ax[3].imshow(image[:,:,::-1],aspect="auto")
    ax[3].invert_yaxis()

    fig.tight_layout()
    embed_pdf_figure()
In [11]:
for image,detected_point in zip(images,detected_points):
    plot_triangualation(image,detected_point,
                        detected_point[:,np.array([1,0])],
                       Delaunay(detected_point[:,np.array([1,0])]).simplices)

self implementation

delaunay triangulation

In [10]:
import delaunay2D
reload(delaunay2D)
from delaunay2D import Delaunay2D
def get_triangularized_points(key_points):
    dt = Delaunay2D()
    for s in key_points:
        dt.addPoint(s)
    return dt.exportTriangles()
In [27]:
for image,detected_point in zip(images,detected_points_):
    plot_triangualation(image,detected_point,
                        detected_point[:,np.array([1,0])],
                       get_triangularized_points(detected_point[:,np.array([1,0])]))
In [ ]:

Morphing

In [12]:
#https://docs.opencv.org/3.4/d4/d61/tutorial_warp_affine.html
#https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_geometric_transformations/py_geometric_transformations.html
def bounding_rec(t):
    y_min,y_max = np.min(np.int16(t)[:,0]),np.max(np.int16(t)[:,0])
    x_min,x_max = np.min(np.int16(t)[:,1]),np.max(np.int16(t)[:,1])
    return y_min,x_min,y_max-y_min+1,x_max-x_min+1

def getAffineTransform(pts1,pts2):
    mx2 = np.concatenate((pts2,np.ones(pts2.shape[0]).reshape(-1,1)),axis=1).T
    mx1_inv = numpy.linalg.inv(np.concatenate((pts1,np.ones(pts1.shape[0]).reshape(-1,1)),axis=1).T)
    return mx2.dot(mx1_inv)[:-1]
In [13]:
def check(p1, p2, base_array):
    """
    Uses the line defined by p1 and p2 to check array of 
    input indices against interpolated value

    Returns boolean array, with True inside and False outside of shape
    """
    idxs = np.indices(base_array.shape) # Create 3D array of indices

    p1 = p1.astype(float)
    p2 = p2.astype(float)

    # Calculate max column idx for each row idx based on interpolated line between two points
    max_col_idx = (idxs[0] - p1[0]) / (p2[0] - p1[0]) * (p2[1] - p1[1]) +  p1[1]    
    sign = np.sign(p2[0] - p1[0])
    return idxs[1] * sign <= max_col_idx * sign

def create_polygon(shape, vertices):
    """
    Creates np.array with dimensions defined by shape
    Fills polygon defined by vertices with ones, all other values zero"""
    base_array = np.zeros(shape, dtype=float)  # Initialize your array of zeros

    fill = np.ones(base_array.shape) * True  # Initialize boolean array defining shape fill

    # Create check array for each edge segment, combine into fill array
    for k in range(vertices.shape[0]):
        fill = np.all([fill, check(vertices[k-1], vertices[k], base_array)], axis=0)
    #print (fill.shape)
    # Set all values inside polygon to one
    base_array[fill] = 1
    base_array_ = np.zeros((base_array.shape), dtype = np.float32)
    base_array_[np.where(base_array!=0)[1],np.where(base_array!=0)[0]] =1
    return base_array_#np.transpose(base_array,(1,0,2))
In [14]:
fig,ax=plt.subplots(figsize=(4,4))
mask = create_polygon((40,30,3), np.array([[0,0],[17,3],[10,18]])[:,np.array([1,0])] )
print (mask.shape)
ax.imshow(mask[:,:,::-1])

(40, 30, 3)
Out[14]:
<matplotlib.image.AxesImage at 0x12c00b1d0>
In [15]:
fig,ax=plt.subplots(figsize=(4,4))
mask = np.zeros((40,30,3), dtype = np.float32)
ax.imshow(cv2.fillConvexPoly(mask, np.array([[0,0],[17,3],[10,18]])[:,np.array([1,0])] ,
                            (1.0, 1.0, 1.0), 16, 0)[:,:,::-1])
print (mask.shape)

(40, 30, 3)
$$ A = \begin{bmatrix} a_{00} & a_{01} \\ a_{10} & a_{11} \end{bmatrix}_{2 \times 2} B = \begin{bmatrix} b_{00} \\ b_{10} \end{bmatrix}_{2 \times 1} \\ M = \begin{bmatrix} A & B \end{bmatrix} = \begin{bmatrix} a_{00} & a_{01} & b_{00} \\ a_{10} & a_{11} & b_{10} \end{bmatrix}_{2 \times 3}\\ \text { Considering that we want to transform a 2D vector } X = \left[ \begin{array} { l } { x } \\ { y } \end{array} \right] \text { by using } A \text { and } B , \text { we can do the same with: }\\ T = A \cdot \left[ \begin{array} { l } { x } \\ { y } \end{array} \right] + B \text { or } T = M \cdot [ x , y , 1 ] ^ { T }\\ T = \left[ \begin{array} { l } { a _ { 00 } x + a _ { 01 } y + b _ { 00 } } \\ { a _ { 10 } x + a _ { 11 } y + b _ { 10 } } \end{array} \right] $$
In [16]:
def morphTriangle(img1, img2, img, t1, t2, t, alpha) :

    # Find bounding rectangle for each triangle
    r1 = bounding_rec(t1)
    r2 = bounding_rec(t2)
    r = bounding_rec(t)


    # Offset points by left top corner of the respective rectangles
    t1Rect = []
    t2Rect = []
    tRect = []


    for i in range(0, 3):
        tRect.append(((t[i][0] - r[0]),(t[i][1] - r[1])))
        t1Rect.append(((t1[i][0] - r1[0]),(t1[i][1] - r1[1])))
        t2Rect.append(((t2[i][0] - r2[0]),(t2[i][1] - r2[1])))


    # Get mask by filling triangle
    mask = np.zeros((r[3], r[2], 3), dtype = np.float32)
    cv2.fillConvexPoly(mask, np.int32(tRect), (1.0, 1.0, 1.0), 16, 0);
    #print (mask.shape)
    #mask = create_polygon((r[3], r[2], 3), np.int32(tRect))
    
    # Apply warpImage to small rectangular patches
    img1Rect = img1[r1[1]:r1[1] + r1[3], r1[0]:r1[0] + r1[2]]
    img2Rect = img2[r2[1]:r2[1] + r2[3], r2[0]:r2[0] + r2[2]]

    size = (r[2], r[3])
    warpImage1 = applyAffineTransform(img1Rect, t1Rect, tRect, size)
    warpImage2 = applyAffineTransform(img2Rect, t2Rect, tRect, size)
    
    # Alpha blend rectangular patches
    imgRect = (1.0 - alpha) * warpImage1 + alpha * warpImage2
    #print (imgRect.shape[0]-img[r[1]:r[1]+r[3], r[0]:r[0]+r[2]].shape[0],
        #  imgRect.shape[1]-img[r[1]:r[1]+r[3], r[0]:r[0]+r[2]].shape[1])
    
    if imgRect.shape[:2]!=img[r[1]:r[1]+r[3], r[0]:r[0]+r[2]].shape[:2]:
        height1,width1 = imgRect.shape[:2]
        height2,width2 = img[r[1]:r[1]+r[3], r[0]:r[0]+r[2]].shape[:2]
        #print (height1,width1,height2,width2)
        if width1 < width2:
            imgRect = np.pad(imgRect, pad_width=((0,0),(0,width2-width1)),mode='reflect')
            mask = np.pad(mask, pad_width=((0,0),(0,width2-width1)),mode='reflect')
        if height1 < height2:
            imgRect = np.pad(imgRect, pad_width=((0,height2-height1),(0,0)),mode='reflect')
            mask = np.pad(mask, pad_width=((0,height2-height1),(0,0)),mode='reflect')
        if width1 > width2:
            imgRect = imgRect[:,:width2]
            mask = mask[:,:width2]
        if height1 > height2:
            imgRect = imgRect[:height2,:]
            mask = mask[:height2,:]
            #print (r,img.shape,img[r[1]:r[1]+r[3], r[0]:r[0]+r[2]].shape,imgRect.shape,mask.shape)
    #print (imgRect)
    # Copy triangular region of the rectangular patch to the output image
    img[r[1]:r[1]+r[3], r[0]:r[0]+r[2]] = img[r[1]:r[1]+r[3], r[0]:r[0]+r[2]] * ( 1 - mask ) + imgRect * mask
    return tRect
from cv2 import *
def applyAffineTransform(src, srcTri, dstTri, size) :
    
    # Given a pair of triangles, find the affine transform.
    #print (np.float32(srcTri),np.float32(dstTri))
    warpMat = getAffineTransform( np.float32(srcTri), np.float32(dstTri) )
    
    # Apply the Affine Transform just found to the src image
    dst = warpAffine( src, warpMat, (size[0], size[1]), None, flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101 )

    return dst

transformed_coord = np.round(warpMat.dot(xy_coordinates.T)).astype('uint8') xy_coordinates = np.concatenate((np.meshgrid(np.arange(0,9),np.arange(0,20))[0].ravel().reshape(-1,1), np.meshgrid(np.arange(0,9),np.arange(0,20))[1].ravel().reshape(-1,1),np.ones([size[0]*size[1]]).reshape(-1,1)),axis=1)

In [17]:
def morph_images(img1,img2,points1,points2,delaunay1,delaunay2,nframes,plot=False):
    count = 0
    imgMorphs = {}

    for a in tqdm(range(nframes)):

        alpha = float(a) / nframes

        points = []         

        # Compute weighted average point coordinates
        for i in range(0, np.min([[len(points1)],[len(points2)]]) ):
            x = ( 1 - alpha ) * points1[i][0] + alpha * points2[i][0]
            y = ( 1 - alpha ) * points1[i][1] + alpha * points2[i][1]
            points.append((x,y))

        # Allocate space for final output
        imgMorph = np.zeros(img1.shape, dtype = img1.dtype)
        for v1, v2, v3 in delaunay1 :
            try:
                t1 = [points1[v1], points1[v2], points1[v3]]
                t2 = [points2[v1], points2[v2], points2[v3]]
                t  = [ points[v1],  points[v2],  points[v3]]
            except:
                pass
            #print (t1)
            # Morph one triangle at a time.
            tRect = morphTriangle(img1, img2, imgMorph, t1, t2, t, alpha)
            #return tRect
        imgMorphs[count] = imgMorph
        count +=1
    if plot:
        fig,ax=plt.subplots(1,nframes,figsize=(5*nframes,6.5))
        for i in range(nframes):
            ax[i].imshow(imgMorphs[i][:,:,::-1],aspect="auto") 
    else:
        return imgMorphs
In [25]:
morph_images(img1=source1,img2=target1 ,points1=detected_points_source1[:,np.array([1,0])],
             points2=detected_points_target1[:,np.array([1,0])],
             delaunay1=get_triangularized_points(detected_points_source1[:,np.array([1,0])]),
             delaunay2=get_triangularized_points(detected_points_target1[:,np.array([1,0])]),
             nframes=5,plot=1)
embed_pdf_figure()










    

























In [ ]:



    


 



    













In [26]:



    


morph_images(img1=source2,img2=target2 ,points1=detected_points_source2_[:,np.array([1,0])],
             points2=detected_points_target2_[:,np.array([1,0])],
             delaunay1=get_triangularized_points(detected_points_source2_[:,np.array([1,0])]),
             delaunay2=get_triangularized_points(detected_points_target2_[:,np.array([1,0])]),
             nframes=5,plot=1)
embed_pdf_figure()



    















    






 
 















    















    

























In [20]:



    


def get_frame_key_points(image,points):
    width=image.shape[0]
    height=image.shape[1]
    add_points = np.array([0,0,width,0,0,height,width/2,0,width/2,height,
          0,height/2,width,height/2,width,height]).astype('int').reshape(-1,2)
    return np.concatenate((points,add_points))



    













In [21]:



    


detected_points_source2_ = np.array([[160,100],[160,220],[250,150],[320,150],
                                    [160,50],[158,129],[158,97],[170,180],[150,250],[165,222], #eye
                                     [160,160],[180,160],[200,160],[220,160],[250,160],[260,160], #nose
                                    [130,40],[140,75],[150,125],[150,180],[140,210],[130,269], #eye brow
                                     [300,50],[340,80],[375,150],[373,229],[299,267],#chin
                                    [310,80],[320,132],[330,160],[320,183],[299,235],[292,160], #mouse
                                    ])
detected_points_target2_ = np.array([[160,100],[160,220],[240,150],[350,160],
                                    [160,50],[170,90],[159,73],[160,190],[145,245],[152,221],
                                     [180,160],[200,160],[220,160],[250,160],[275,160],[295,160],
                                     [130,40],[140,75],[150,125],[150,180],[140,210],[130,269], 
                                     [300,65],[342,76],[420,168],[365,233],[299,260],
                                      [350,100],[375,132],[360,162],[373,200],[376,237],[330,163], 
                                    ])
detected_points_target2_ = get_frame_key_points(target2,detected_points_target2_)
detected_points_source2_ = get_frame_key_points(source2,detected_points_source2_)
detected_points_ = [detected_points_source1,detected_points_target1,detected_points_source2_,detected_points_target2_]



    















source3 = cv2.imread('test1.png')
target3 = cv2.imread('test2.png')
detected_points_source3 = np.loadtxt('detected_points_source3.txt').astype('int')
detected_points_target3 = np.loadtxt('detected_points_target3.txt').astype('int')
morph_images(img1=source3,img2=target3 ,points1=detected_points_source3[:,np.array([1,0])],
             points2=detected_points_target3[:,np.array([1,0])],
             delaunay1=get_triangularized_points(detected_points_source3[:,np.array([1,0])]),
             delaunay2=get_triangularized_points(detected_points_target3[:,np.array([1,0])]),
             nframes=10,plot=1)
padding = True
pad_value = 0
if padding:
    source2 = np.pad(source2,((pad_value,pad_value),(pad_value,pad_value),(0,0)),'reflect')
    target2 = np.pad(target2,((pad_value,pad_value),(pad_value,pad_value),(0,0)),'reflect')













In [ ]:



    


 



    













In [ ]: