NOTE: This Use Case is not purposed for resource constrained devices.
KEY FACIAL POINTS DETECTION AND FACE RECOGNITION USING DLIB¶
Credit: AITS Cainvas Community
Photo by Gleb Kuznetsov on Dribbble
IMPORT LIBRARIES/DATASETS AND PERFORM PRELIMINARY DATA PROCESSING¶
In [1]:
# Import the necessary packages
import pandas as pd
import numpy as np
import tensorflow as tf
from tensorflow.python.keras import Sequential
from tensorflow.keras import layers, optimizers
from tensorflow.keras.applications import DenseNet121
from tensorflow.keras.applications.resnet50 import ResNet50
from tensorflow.keras.layers import *
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.initializers import glorot_uniform
from tensorflow.keras.utils import plot_model
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint, LearningRateScheduler
from IPython.display import display
from tensorflow.keras import backend as K
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from keras import optimizers
Loading the Dataset¶
DATASET: Source of Data¶
The Dataset contains a csv file which includes co-ordinates of Key Facial Points and the last column contains pixel values of the image.The architecture of the model used in this Notebook is inspired by ResNet architectures.¶
In [2]:
# load the data
facialpoints_df = pd.read_csv('https://cainvas-static.s3.amazonaws.com/media/user_data/cainvas-admin/KeyFacialPoints.csv')
In [3]:
facialpoints_df.head()
Out[3]:
In [4]:
#Extracting Insights from the data
facialpoints_df.info()
In [5]:
# Since values for the image is given as space separated string, we will need to separate the values using ' ' as separator.
# Then convert this into numpy array using np.fromstring and convert the obtained 1D array into 2D array of shape (96,96)
facialpoints_df['Image'] = facialpoints_df['Image'].apply(lambda x: np.fromstring(x, dtype= int, sep = ' ').reshape(96,96))
In [6]:
# Let's obtain the shape of the resized image
facialpoints_df['Image'][1].shape
Out[6]:
In [7]:
#Checking for NULL values
facialpoints_df.isnull().sum()
Out[7]:
VISUALIZING IMAGE¶
In [8]:
# Plot a random image from the dataset along with facial keypoints.
i = np.random.randint(1, len(facialpoints_df))
plt.imshow(facialpoints_df['Image'][i],cmap='gray')
Out[8]:
In [9]:
# Plotting the co-ordinates of key facial points available in the data-frame upon the image
plt.figure()
plt.imshow(facialpoints_df['Image'][i],cmap='gray')
for j in range(1,31,2):
plt.plot(facialpoints_df.loc[i][j-1], facialpoints_df.loc[i][j], 'r.')
In [10]:
# Let's view more images in a grid format
fig = plt.figure(figsize=(20, 20))
for i in range(16):
ax = fig.add_subplot(4, 4, i + 1)
image = plt.imshow(facialpoints_df['Image'][i], cmap = 'gray')
for j in range(1,31,2):
plt.plot(facialpoints_df.loc[i][j-1], facialpoints_df.loc[i][j], 'r.')
PERFORMING IMAGE AUGMENTATION TO INCREASE THE AMOUNT OF DATA¶
In [11]:
# Create a new copy of the dataframe
import copy
facialpoints_df_copy = copy.copy(facialpoints_df)
In [12]:
# obtain the header of the DataFrame (names of columns)
columns = facialpoints_df_copy.columns[:-1]
columns
Out[12]:
In [13]:
# Take a look at the pixel values of a sample image and see if it makes sense!
facialpoints_df['Image'][0]
Out[13]:
In [14]:
# plot the sample image
plt.imshow(facialpoints_df['Image'][0], cmap = 'gray')
Out[14]:
In [15]:
# Now Let's flip the image column horizontally
facialpoints_df_copy['Image'] = facialpoints_df_copy['Image'].apply(lambda x: np.flip(x, axis = 1))
In [16]:
# Now take a look at the flipped image and do a sanity check!
# Notice that the values of pixels are now flipped
facialpoints_df_copy['Image'][0]
Out[16]:
In [17]:
# Notice that the image is flipped now
plt.imshow(facialpoints_df_copy['Image'][0], cmap = 'gray')
Out[17]:
In [18]:
# Since we are flipping the images horizontally, y coordinate values would be the same
# X coordinate values only would need to change, all we have to do is to subtract our initial x-coordinate values from width of the image(96)
for i in range(len(columns)):
if i%2 == 0:
facialpoints_df_copy[columns[i]] = facialpoints_df_copy[columns[i]].apply(lambda x: 96. - float(x) )
In [19]:
# View the Original image
plt.imshow(facialpoints_df['Image'][0],cmap='gray')
for j in range(1, 31, 2):
plt.plot(facialpoints_df.loc[0][j-1], facialpoints_df.loc[0][j], 'r.')
In [20]:
# View the Horizontally flipped image
plt.imshow(facialpoints_df_copy['Image'][0], cmap='gray')
for j in range(1, 31, 2):
plt.plot(facialpoints_df_copy.loc[0][j-1], facialpoints_df_copy.loc[0][j], 'r.')
In [21]:
# Concatenate the original dataframe with the augmented dataframe
facialpoints_df_augmented = np.concatenate((facialpoints_df,facialpoints_df_copy))
In [22]:
facialpoints_df_augmented.shape
Out[22]:
In [23]:
# Let's try to perform another image augmentation by randomly increasing images brightness
import random
facialpoints_df_copy = copy.copy(facialpoints_df)
facialpoints_df_copy['Image'] = facialpoints_df['Image'].apply(lambda x:np.clip(random.uniform(1, 2) * x, 0.0, 255.0))
facialpoints_df_augmented = np.concatenate((facialpoints_df_augmented, facialpoints_df_copy))
facialpoints_df_augmented.shape
Out[23]:
In [24]:
# Let's view image with increased brightness
plt.imshow(facialpoints_df_copy['Image'][0], cmap = 'gray')
for j in range(1, 31, 2):
plt.plot(facialpoints_df_copy.loc[0][j-1], facialpoints_df_copy.loc[0][j], 'r.')
PERFORMING NORMALIZATION AND TRAINING DATA PREPARATION¶
In [25]:
# Obtain the value of 'Images' which is in 31st column of DataFrame and normalize it
img = facialpoints_df_augmented[:, 30]
img = img/255.
# Create an empty array of shape (6420, 96, 96, 1) to train the model
X = np.empty((len(img), 96, 96, 1))
# Iterate through the normalized images list and add image values to the empty array
# Note that we need to expand it's dimension from (96,96) to (96,96,1)
for i in range(len(img)):
X[i,] = np.expand_dims(img[i], axis = 2)
# Convert the array type to float32
X = np.asarray(X).astype(np.float32)
X.shape
Out[25]:
In [26]:
# Obtain the values of key face points coordinates, which are to used as target to train the model
y = facialpoints_df_augmented[:,:30]
y = np.asarray(y).astype(np.float32)
y.shape
Out[26]:
In [27]:
# Split the data into training and testing data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1)
In [28]:
X_train.shape
Out[28]:
In [29]:
# Let's view more images in a grid format
fig = plt.figure(figsize=(20, 20))
for i in range(64):
ax = fig.add_subplot(8, 8, i + 1)
image = plt.imshow(X_train[i].reshape(96,96), cmap = 'gray')
for j in range(1,31,2):
plt.plot(y_train[i][j-1], y_train[i][j], 'r.')
BUILDING A CUSTOM RESIDUAL NEURAL NETWORK ARCHITECTURE¶
In [30]:
def res_block(X, filter, stage):
# CONVOLUTIONAL BLOCK
X_copy = X
f1 , f2, f3 = filter
# Main Path
X = Conv2D(f1, (1,1), strides = (1,1), name ='res_'+str(stage)+'_conv_a', kernel_initializer= glorot_uniform(seed = 0))(X)
X = MaxPool2D((2,2))(X)
X = BatchNormalization(axis =3, name = 'bn_'+str(stage)+'_conv_a')(X)
X = Activation('relu')(X)
X = Conv2D(f2, kernel_size = (3,3), strides =(1,1), padding = 'same', name ='res_'+str(stage)+'_conv_b', kernel_initializer= glorot_uniform(seed = 0))(X)
X = BatchNormalization(axis =3, name = 'bn_'+str(stage)+'_conv_b')(X)
X = Activation('relu')(X)
X = Conv2D(f3, kernel_size = (1,1), strides =(1,1),name ='res_'+str(stage)+'_conv_c', kernel_initializer= glorot_uniform(seed = 0))(X)
X = BatchNormalization(axis =3, name = 'bn_'+str(stage)+'_conv_c')(X)
# Short path
X_copy = Conv2D(f3, kernel_size = (1,1), strides =(1,1),name ='res_'+str(stage)+'_conv_copy', kernel_initializer= glorot_uniform(seed = 0))(X_copy)
X_copy = MaxPool2D((2,2))(X_copy)
X_copy = BatchNormalization(axis =3, name = 'bn_'+str(stage)+'_conv_copy')(X_copy)
# Add data from main and short paths
X = Add()([X,X_copy])
X = Activation('relu')(X)
# IDENTITY BLOCK 1
X_copy = X
# Main Path
X = Conv2D(f1, (1,1),strides = (1,1), name ='res_'+str(stage)+'_identity_1_a', kernel_initializer= glorot_uniform(seed = 0))(X)
X = BatchNormalization(axis =3, name = 'bn_'+str(stage)+'_identity_1_a')(X)
X = Activation('relu')(X)
X = Conv2D(f2, kernel_size = (3,3), strides =(1,1), padding = 'same', name ='res_'+str(stage)+'_identity_1_b', kernel_initializer= glorot_uniform(seed = 0))(X)
X = BatchNormalization(axis =3, name = 'bn_'+str(stage)+'_identity_1_b')(X)
X = Activation('relu')(X)
X = Conv2D(f3, kernel_size = (1,1), strides =(1,1),name ='res_'+str(stage)+'_identity_1_c', kernel_initializer= glorot_uniform(seed = 0))(X)
X = BatchNormalization(axis =3, name = 'bn_'+str(stage)+'_identity_1_c')(X)
# Add both paths together (Note that we feed the original input as is hence the name "identity")
X = Add()([X,X_copy])
X = Activation('relu')(X)
# IDENTITY BLOCK 2
X_copy = X
# Main Path
X = Conv2D(f1, (1,1),strides = (1,1), name ='res_'+str(stage)+'_identity_2_a', kernel_initializer= glorot_uniform(seed = 0))(X)
X = BatchNormalization(axis =3, name = 'bn_'+str(stage)+'_identity_2_a')(X)
X = Activation('relu')(X)
X = Conv2D(f2, kernel_size = (3,3), strides =(1,1), padding = 'same', name ='res_'+str(stage)+'_identity_2_b', kernel_initializer= glorot_uniform(seed = 0))(X)
X = BatchNormalization(axis =3, name = 'bn_'+str(stage)+'_identity_2_b')(X)
X = Activation('relu')(X)
X = Conv2D(f3, kernel_size = (1,1), strides =(1,1),name ='res_'+str(stage)+'_identity_2_c', kernel_initializer= glorot_uniform(seed = 0))(X)
X = BatchNormalization(axis =3, name = 'bn_'+str(stage)+'_identity_2_c')(X)
# Add both paths together (Note that we feed the original input as is hence the name "identity")
X = Add()([X,X_copy])
X = Activation('relu')(X)
return X
In [31]:
input_shape = (96,96,1)
# Input tensor shape
X_input = Input(input_shape)
# Zero-padding
X = ZeroPadding2D((3,3))(X_input)
# Stage #1
X = Conv2D(64, (7,7), strides= (2,2), name = 'conv1', kernel_initializer= glorot_uniform(seed = 0))(X)
X = BatchNormalization(axis =3, name = 'bn_conv1')(X)
X = Activation('relu')(X)
X = MaxPooling2D((3,3), strides= (2,2))(X)
# Stage #2
X = res_block(X, filter= [64,64,256], stage= 2)
# Stage #3
X = res_block(X, filter= [128,128,512], stage= 3)
# Average Pooling
X = AveragePooling2D((2,2), name = 'Averagea_Pooling')(X)
# Final layer
X = Flatten()(X)
X = Dense(4096, activation = 'relu')(X)
X = Dropout(0.2)(X)
X = Dense(2048, activation = 'relu')(X)
X = Dropout(0.1)(X)
X = Dense(30, activation = 'relu')(X)
model = Model( inputs= X_input, outputs = X)
model.summary()
COMPILING AND TRAINING DEEP LEARNING MODEL¶
In [32]:
adam = tf.keras.optimizers.Adam(lr = 0.001, beta_1=0.9, beta_2=0.999, amsgrad=False)
model.compile(loss="mean_squared_error", optimizer = adam, metrics = ['accuracy'])
In [33]:
# save the best model with least validation loss
checkpointer = ModelCheckpoint(filepath = "model.h5", verbose = 2, save_best_only = True)
In [34]:
history = model.fit(X_train, y_train, batch_size = 256, epochs= 150, validation_split = 0.05, callbacks=[checkpointer])
ASSESSING TRAINED MODEL PERFORMANCE¶
In [35]:
# Loading trained model
new_model = tf.keras.models.load_model('model.h5')
new_model.summary()
In [36]:
# Getting the model history keys
history.history.keys()
Out[36]:
In [37]:
# plot the training artifacts
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train_loss','val_loss'], loc = 'upper right')
plt.show()
In [38]:
# Make prediction using the testing dataset
df_predict = model.predict(X_test)
In [39]:
# Print the rmse loss values
from sklearn.metrics import mean_squared_error
from math import sqrt
rms = sqrt(mean_squared_error(y_test, df_predict))
print("RMSE value : {}".format(rms))
In [40]:
# Convert the predicted values into a dataframe
df_predict= pd.DataFrame(df_predict, columns = columns)
df_predict.head()
Out[40]:
VISUALIZING THE RESULTS¶
In [41]:
# Plot the test images and their predicted keypoints
fig = plt.figure(figsize=(20, 20))
for i in range(8):
ax = fig.add_subplot(4, 2, i + 1)
# Using squeeze to convert the image shape from (96,96,1) to (96,96)
plt.imshow(X_test[i].squeeze(),cmap='gray')
for j in range(1,31,2):
plt.plot(df_predict.loc[i][j-1], df_predict.loc[i][j], 'r.')
COMPILING THE MODEL USING DEEPC¶
In [42]:
!deepCC model.h5
FACE RECOGNITION MODEL USING DLIB LIBRARY¶
DLIB is a library which conatains models which are trained to find KEY FACIAL POINTS using a very large dataset and with a more complex Neural Network Architecture.It has been trained for a sufficiently long time and it can be used for various Face Applications¶
In [43]:
#Import dlib library
import dlib
dlib.__version__
Out[43]:
In [44]:
!wget -N "https://cainvas-static.s3.amazonaws.com/media/user_data/cainvas-admin/dlib_face_recognition.zip"
!unzip -o dlib_face_recognition.zip
!rm dlib_face_recognition.zip
In [45]:
#Initializing various modules which we will be using for face recognition
detector = dlib.get_frontal_face_detector()
sp = dlib.shape_predictor("dlib_face_recognition/shape_predictor_5_face_landmarks.dat")
detec_model = dlib.face_recognition_model_v1("dlib_face_recognition/dlib_face_recognition_resnet_model_v1.dat")
In [46]:
#Loading images of Celebs
downey1_img = dlib.load_rgb_image("dlib_face_recognition/downey1.jpg")
downey2_img = dlib.load_rgb_image("dlib_face_recognition/downey2.jpg")
chris_img = dlib.load_rgb_image("dlib_face_recognition/chris.jpg")
In [47]:
#Detecting face using Dlib
img1_detected = detector(downey1_img)
img2_detected = detector(downey2_img)
img3_detected = detector(chris_img)
In [48]:
#Determinig the key facial landmarks
img1_shape = sp(downey1_img, img1_detected[0])
img2_shape = sp(downey2_img, img2_detected[0])
img3_shape = sp(chris_img, img3_detected[0])
In [49]:
#Aligning the face for better results using Dlib
img1_aligned = dlib.get_face_chip(downey1_img, img1_shape)
img2_aligned = dlib.get_face_chip(downey2_img, img2_shape)
img3_aligned = dlib.get_face_chip(chris_img, img3_shape)
In [50]:
#Converting the aligned image in the form of vectors
img1_rep = detec_model.compute_face_descriptor(img1_aligned)
img2_rep = detec_model.compute_face_descriptor(img2_aligned)
img3_rep = detec_model.compute_face_descriptor(img3_aligned)
In [51]:
#Function to find euclidean distance between source points and target poinst which will be later used for comaprisons between different faces
def findeuclidean(source_rep, test_rep):
dist = source_rep - test_rep
dist = np.sum(np.multiply(dist, dist))
dist = np.sqrt(dist)
return dist
In [52]:
#Function to check whether the target image matches with the source image
#We will be comparing the key facial points co-ordinates by calculating their euclidean distance
#If the distance is less than a threshold value then the images are of same person otherwise different persons
def check_result(img1_rep, img2_rep):
threshold = 0.6
distance = findeuclidean(img1_rep, img2_rep)
if(distance < threshold):
print("Both presonalities are same!")
else:
print("Both personalities are different!")
ACCESSING THE PERFORMANCE OF THE MODEL¶
In [53]:
plt.imshow(img1_aligned)
plt.show()
plt.imshow(img2_aligned)
plt.show()
img1_rep = np.array(img1_rep)
img2_rep = np.array(img2_rep)
In [54]:
check_result(img1_rep, img2_rep)
In [55]:
plt.imshow(img1_aligned)
plt.show()
plt.imshow(img3_aligned)
plt.show()
img1_rep = np.array(img1_rep)
img3_rep = np.array(img3_rep)
In [56]:
check_result(img1_rep, img3_rep)