Driver Drowsiness Detection using CNN¶
According to a report, around 40% of road accidents that happen on highways are caused by Drowsy Driving. This project aims at detecting whether a driver is feeling drowsy or is active while driving based on whether both the eyes of the driver are closed representing drowsiness or both the eyes are open using a CNN architecture.
Importing necessary libraries..¶
In [1]:
import numpy as np
import cv2
from tensorflow.keras.layers import Input, Conv2D, BatchNormalization, Dropout, Flatten, Dense, MaxPool2D
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.initializers import glorot_uniform
from tensorflow.keras.optimizers import Adam, SGD
import matplotlib.pyplot as plt
from sklearn.metrics import classification_report
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
from tensorflow.keras.utils import to_categorical
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import classification_report, confusion_matrix
import seaborn as sns
from tensorflow.keras.regularizers import l2
import tensorflow as tf
import pandas as pd
from imutils.face_utils import FaceAligner
from imutils.face_utils import rect_to_bb
import imutils
import dlib
Loading the dataset and the test images¶
You can upload your own test images also.
In [6]:
!wget https://cainvas-static.s3.amazonaws.com/media/user_data/Yuvnish17/driver_drowsiness_detection_modified.zip
!unzip -qo driver_drowsiness_detection_modified.zip
Loading the images and labels and Preprocessing the Dataset¶
Input Shape of Images for the CNN model - (32, 32, 3)
Dataset os split into Train Set and Test Set with test set containing 20% of the total number of images
Labels -> 0 - Closed Eye
1 - Open Eye
In [7]:
data = np.load('driver_drowsiness_detection/dataset_compressed.npz', allow_pickle=True)
X = data['arr_0']
Y = data['arr_1']
X = list(X)
Y = list(Y)
print(len(X))
print(len(Y))
In [8]:
for i in range(len(X)):
img = X[i]
img = cv2.resize(img, (32, 32))
X[i] = img
print(len(X))
print(X[0].shape)
In [9]:
label_encoder = LabelEncoder()
Y = label_encoder.fit_transform(Y)
print(Y.shape)
print(Y[0])
print(set(Y))
In [10]:
X = np.array(X)
Y = np.array(Y)
print(X.shape)
print(Y.shape)
In [11]:
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)
In [12]:
print(X_train.shape)
print(Y_train.shape)
print(X_test.shape)
print(Y_test.shape)
Y_train = to_categorical(Y_train)
Y_test = to_categorical(Y_test)
print(X_train.shape)
print(Y_train.shape)
print(X_test.shape)
print(Y_test.shape)
Visualizing images of Closed Eye and Open Eye from the Dataset¶
In [13]:
figure1 = plt.figure(figsize=(5, 5))
idx_closed = np.where(Y==0)
img_closed = X[idx_closed[0][0]]
plt.imshow(img_closed)
plt.title('Image of Closed Eye representing Driver is sleeping')
plt.axis('off')
plt.show()
figure2 = plt.figure(figsize=(5, 5))
idx_open = np.where(Y==1)
img_open = X[idx_open[0][0]]
plt.imshow(img_open)
plt.title('Image of Open Eye representing Driver is not sleeping')
plt.axis('off')
plt.show()
Visualizing the Data Distribution in Train and Test Set¶
In [11]:
unique_train, count = np.unique(Y_train, return_counts=True)
plt.figure(figsize=(20, 10))
sns.barplot(unique_train, count).set_title("Number of training images per category:")
plt.show()
In [12]:
unique_test, count_test = np.unique(Y_test, return_counts=True)
plt.figure(figsize=(20, 10))
sns.barplot(unique_test, count_test).set_title("Number of training images per category:")
plt.show()
As can be seen from the bar plots that the dataset is already balanced so no need to perform dataset balancing
Defining the Model Architecture¶
In [13]:
def driver_drowsiness_detection_model(input_shape=(32, 32, 3)):
model = Sequential()
model.add(Input(shape=input_shape))
model.add(Conv2D(32, (3, 3), padding='same', strides=(1, 1), name='conv1', activation='relu',
kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization())
model.add(Conv2D(32, (3, 3), padding='same', strides=(1, 1), name='conv2', activation='relu',
kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization())
model.add(Dropout(0.2))
model.add(MaxPool2D((2, 2), strides=(2, 2)))
model.add(Conv2D(64, (3, 3), padding='same', strides=(1, 1), name='conv3', activation='relu',
kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization())
model.add(MaxPool2D((2, 2), strides=(2, 2)))
model.add(Conv2D(64, (3, 3), padding='same', strides=(1, 1), name='conv4', activation='relu',
kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization())
model.add(Dropout(0.3))
model.add(MaxPool2D((2, 2), strides=(2, 2)))
model.add(Conv2D(64, (3, 3), padding='same', strides=(1, 1), name='conv5', activation='relu',
kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization())
model.add(Conv2D(64, (3, 3), padding='same', strides=(1, 1), name='conv6', activation='relu',
kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization())
model.add(Conv2D(64, (3, 3), padding='same', strides=(1, 1), name='conv7', activation='relu',
kernel_initializer=glorot_uniform(seed=0)))
model.add(BatchNormalization())
model.add(Dropout(0.4))
model.add(MaxPool2D((2, 2), strides=(2, 2)))
model.add(Flatten())
model.add(Dense(128, activation='relu', kernel_initializer=glorot_uniform(seed=0), name='fc1'))
model.add(Dropout(0.5))
model.add(Dense(128, activation='relu', kernel_initializer=glorot_uniform(seed=0), name='fc2'))
model.add(Dropout(0.5))
model.add(Dense(2, activation='softmax', kernel_initializer=glorot_uniform(seed=0), name='fc3'))
optimizer = Adam(0.0001)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
return model
In [14]:
model= driver_drowsiness_detection_model(input_shape=(32, 32, 3))
model.summary()
Training the Model¶
Learning Rate - 0.0001
Optimizer - Adam
Number of Epochs - 200
Batch Size = 128
In [15]:
aug = ImageDataGenerator(rotation_range=20, zoom_range=0.2, horizontal_flip=True)
hist = model.fit(aug.flow(X_train, Y_train, batch_size=128), batch_size=128, epochs=200, validation_data=(X_test, Y_test))
Plotting Loss Values and Accuracy against Number of Epochs for Train Set and Test Set¶
In [16]:
figure = plt.figure(figsize=(10, 10))
plt.plot(hist.history['accuracy'], label='Train_accuracy')
plt.plot(hist.history['val_accuracy'], label='Test_accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend(loc="upper left")
plt.show()
figure2 = plt.figure(figsize=(10, 10))
plt.plot(hist.history['loss'], label='Train_loss')
plt.plot(hist.history['val_loss'], label='Test_loss')
plt.title('Model Loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend(loc="upper left")
plt.show()
Evaluating Model on Test Set¶
In [17]:
pred = model.evaluate(X_test, Y_test)
print(f'Test Set Accuracy: {pred[1]}')
print(f'Test Set Loss: {pred[0]}')
Classification Report¶
In [18]:
ypred = model.predict(X_test)
ypred = np.argmax(ypred, axis=1)
Y_test_pred = np.argmax(Y_test, axis=1)
print(classification_report(Y_test_pred, ypred))
Confusion Matrix¶
In [19]:
matrix = confusion_matrix(Y_test_pred, ypred)
df_cm = pd.DataFrame(matrix, index=[0, 1], columns=[0, 1])
figure = plt.figure(figsize=(5, 5))
sns.heatmap(df_cm, annot=True, fmt='d')
Out[19]:
Saving the Model¶
In [20]:
model.save('Driver_Drowsiness_Detection.h5')
In [21]:
labels = ['Closed', 'Open']
img_closed1 = cv2.imread('driver_drowsiness_detection/closed_eye.jpg')
img_closed2 = cv2.imread('driver_drowsiness_detection/closed_eye2.jpg')
img_open1 = cv2.imread('driver_drowsiness_detection/open_eye.jpg')
img_open2 = cv2.imread('driver_drowsiness_detection/open_eye2.jpg')
img_closed1 = cv2.resize(img_closed1, (32, 32))
img_closed2 = cv2.resize(img_closed2, (32, 32))
img_open1 = cv2.resize(img_open1, (32, 32))
img_open2 = cv2.resize(img_open2, (32, 32))
img_closed1 = np.array(img_closed1)
img_closed2 = np.array(img_closed2)
img_open1 = np.array(img_open1)
img_open2 = np.array(img_open2)
img_closed1 = np.expand_dims(img_closed1, axis=0)
img_closed2 = np.expand_dims(img_closed2, axis=0)
img_open1 = np.expand_dims(img_open1, axis=0)
img_open2 = np.expand_dims(img_open2, axis=0)
In [22]:
ypred_closed1 = model.predict(img_closed1)
ypred_closed2 = model.predict(img_closed2)
ypred_open1 = model.predict(img_open1)
ypred_open2 = model.predict(img_open2)
In [23]:
figure = plt.figure(figsize=(2, 2))
img_closed1 = np.squeeze(img_closed1, axis=0)
plt.imshow(img_closed1)
plt.axis('off')
plt.title(f'Prediction by the model: {labels[np.argmax(ypred_closed1[0], axis=0)]}')
plt.show()
In [24]:
figure = plt.figure(figsize=(2, 2))
img_closed2 = np.squeeze(img_closed2, axis=0)
plt.imshow(img_closed2)
plt.axis('off')
plt.title(f'Prediction by the model: {labels[np.argmax(ypred_closed2[0], axis=0)]}')
plt.show()
In [25]:
figure = plt.figure(figsize=(2, 2))
img_open1 = np.squeeze(img_open1, axis=0)
plt.imshow(img_open1)
plt.axis('off')
plt.title(f'Prediction by the model: {labels[np.argmax(ypred_open1[0], axis=0)]}')
plt.show()
In [26]:
figure = plt.figure(figsize=(2, 2))
img_open2 = np.squeeze(img_open2, axis=0)
plt.imshow(img_open2)
plt.axis('off')
plt.title(f'Prediction by the model: {labels[np.argmax(ypred_open2[0], axis=0)]}')
plt.show()
Creating Pipeline for making predictions on full face images¶
In [27]:
def full_face_detection_pipeline(input_image_path):
face_cascade = cv2.CascadeClassifier('driver_drowsiness_detection/haarcascade_frontalface_default.xml')
eye_cascade = cv2.CascadeClassifier('driver_drowsiness_detection/haarcascade_eye.xml')
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('driver_drowsiness_detection/shape_predictor_68_face_landmarks.dat')
fa = FaceAligner(predictor, desiredFaceWidth=256)
test_image = cv2.imread(input_image_path)
test_image = imutils.resize(test_image, width=800)
test_image_gray = cv2.cvtColor(test_image, cv2.COLOR_BGR2GRAY)
rects = detector(test_image_gray, 2)
for rect in rects:
(x, y, w, h) = rect_to_bb(rect)
faceOrig = imutils.resize(test_image[y:y+h, x:x+w], width=256)
faceAligned = fa.align(test_image, test_image_gray, rect)
faceAligned_gray = cv2.cvtColor(faceAligned, cv2.COLOR_BGR2GRAY)
plt.imshow(faceAligned_gray)
plt.axis('off')
plt.title('Aligned Face')
plt.show()
eyes = eye_cascade.detectMultiScale(faceAligned_gray, 1.1, 4)
predictions = []
for (ex, ey, ew, eh) in eyes:
eye = faceAligned[ey:ey+eh, ex:ex+ew]
# cv2.rectangle(test_image, (x+ex, y+ey), (x+ex+ew, y+ey+eh), (0, 0, 255), 8)
eye = cv2.resize(eye, (32, 32))
eye = np.array(eye)
eye = np.expand_dims(eye, axis=0)
ypred = model.predict(eye)
ypred = np.argmax(ypred[0], axis=0)
predictions.append(ypred)
if all(i==0 for i in predictions):
cv2.rectangle(test_image, (x, y), (x+w, y+h), (0, 0, 255), 8)
cv2.putText(test_image, 'Sleeping', (x, y-10), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (0, 0, 255), 3)
else:
cv2.rectangle(test_image, (x, y), (x+w, y+h), (0, 255, 0), 8)
cv2.putText(test_image, 'Not Sleeping', (x, y-10), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (0, 255, 0), 3)
output_path = 'driver_drowsiness_detection/test_image_prediction.jpg'
cv2.imwrite(output_path, test_image)
return output_path
In [28]:
figure = plt.figure(figsize=(5, 5))
predicted_image = cv2.imread(full_face_detection_pipeline('driver_drowsiness_detection/active_person.jpg'))
predicted_image = cv2.cvtColor(predicted_image, cv2.COLOR_BGR2RGB)
plt.imshow(predicted_image)
plt.axis('off')
plt.show()
In [29]:
figure = plt.figure(figsize=(5, 5))
predicted_image = cv2.imread(full_face_detection_pipeline('driver_drowsiness_detection/drowsy_person.jpg'))
predicted_image = cv2.cvtColor(predicted_image, cv2.COLOR_BGR2RGB)
plt.imshow(predicted_image)
plt.axis('off')
plt.show()
In [30]:
figure = plt.figure(figsize=(5, 5))
predicted_image = cv2.imread(full_face_detection_pipeline('driver_drowsiness_detection/sleepy-driver.jpg'))
predicted_image = cv2.cvtColor(predicted_image, cv2.COLOR_BGR2RGB)
plt.imshow(predicted_image)
plt.axis('off')
plt.show()
In [ ]: