tensorflow入门教程(四十五)通过MTCNN动态获取左右眼图像数据

最新推荐文章于 2021-01-12 01:50:35 发布

__Fang Wei__

最新推荐文章于 2021-01-12 01:50:35 发布

阅读量1.5k

点赞数

分类专栏： tensorflow 文章标签： tensorflow mtcnn 眼睛检测

本文链接：https://blog.youkuaiyun.com/rookie_wei/article/details/90744341

版权

tensorflow 专栏收录该内容

70 篇文章

订阅专栏

#
#作者：韦访
#博客：https://blog.youkuaiyun.com/rookie_wei
#微信：1007895847
#添加微信的备注一下是优快云的
#欢迎大家一起学习
#

1、概述

之前有蛮多网友问我怎么通过MTCNN获取左右眼的图像数据，因为通过MTCNN只获取了左眼和右眼中心点的坐标。虽然在回答留言时我说了一下思路，但是我并没有真的去做过，今天有空我就写写吧。

https://blog.youkuaiyun.com/rookie_wei/article/details/86651369

2、思路

做什么事都得有思路才行，

如上图所示，假设我两只眼睛之间的距离为180像素点，我要画的眼眶的长的一半和宽的一半分别为60和30像素点。现在假设两眼之间的距离为d，则眼眶的宽度的一半为d*60/180=d/3，同理，眼眶的高度的一半为d*30/180=d/6。我们这里不考虑斜角，如果脸比较斜的话，MTCNN检测不出你脸的。

3、通过opencv实时检测人脸关键点

思路有了，现在就开始干了。我们以前的博客分析MTCNN时，它是用来对数据集做人脸检测和对其等操作，并没有使用摄像头的数据，现在我们先来结合opencv将通过摄像头检测到的人脸和人脸的五个关键点实时显示出来。

3.1、首先将FaceNet下的src/align所有文件拷贝到我们工程目录下

如果你连FaceNet的源码都不知道去哪里下载的话，就下载这个吧，

https://download.youkuaiyun.com/download/rookie_wei/10938739

3.2、在新的工程下新建demo1.py文件

这个demo1就是我们要自己写的，文件名你随便取，我写博客的教程习惯用这个比较土的命名而已。在demo1.py里，先导入各种我们需要用到的库，

import tensorflow as tf
import os
import detect_face
import cv2
import numpy as np

然后就是入口函数main函数，

if __name__ == '__main__':
    main()

来看main函数，

def main():

    # 读摄像头数据还是本地图片数据
    isFromCamera = True

    with tf.Session() as sess:
        #获取P-Net，R-Net，O-Net网络
        pnet, rnet, onet = detect_face.create_mtcnn(sess, None)

        minsize = 20  # minimum size of face
        threshold = [0.6, 0.7, 0.7]  # three steps's threshold
        factor = 0.709  # scale factor
        margin = 0

        # 看是读取摄像头数据呢还是图片呢
        if isFromCamera:
            cam = cv2.VideoCapture(0)
        else:
            imagefile = 'images/01.jpg'
            if os.path.exists(imagefile):
                image = cv2.imread(imagefile, cv2.IMREAD_COLOR)
            else:
                print('图片文件不存在！')
                return False

因为MTCNN可以同时检测多个人脸的，而用摄像头的话，就我一个人，不会做多人脸的检测，所以，我就下载了一个有多个人脸的图片用来检测。

while True:

    if isFromCamera:
        ret, image = cam.read()
        if not ret:
            continue

    # 检测人脸，bounding_boxes可能包含多张人脸框数据，
    # 一张人脸框有5个数据，第一和第二个数据表示框左上角坐标，第三个第四个数据表示框右下角坐标，
    # 最后一个数据应该是可信度
    bounding_boxes, points = detect_face.detect_face(image, minsize, pnet, rnet, onet, threshold, factor)
    # print('bounding_boxes:', bounding_boxes)
    # print('points:', points)

    # 获得的人脸数量
    nrof_faces = bounding_boxes.shape[0]
    if nrof_faces > 0:
        det = bounding_boxes[:, 0:4]
        det_arr = []
        
        # 原图片大小
        img_size = np.asarray(image.shape)[0:2]
        if nrof_faces > 1:
            # 如果要检测多张人脸的话
            for i in range(nrof_faces):
                det_arr.append(np.squeeze(det[i]))
        else:
            # 只有一个人脸框的话，那就没得选了
            det_arr.append(np.squeeze(det))

        for i, det in enumerate(det_arr):
            det = np.squeeze(det)
            bb = np.zeros(4, dtype=np.int32)
            # 边界框周围的裁剪边缘，就是我们这里要裁剪的人脸框要比MTCNN获取的人脸框大一点，
            # 至于大多少，就由margin参数决定了
            bb[0] = np.maximum(det[0] - margin / 2, 0)
            bb[1] = np.maximum(det[1] - margin / 2, 0)
            bb[2] = np.minimum(det[2] + margin / 2, img_size[1])
            bb[3] = np.minimum(det[3] + margin / 2, img_size[0])

            # 人脸框的上横
            cv2.line(image, (bb[0], bb[1]), (bb[2], bb[1]), (0, 0, 255), 3)
            # 人脸框的左竖
            cv2.line(image, (bb[0], bb[1]), (bb[0], bb[3]), (0, 0, 255), 3)
            # 人脸框的右竖
            cv2.line(image, (bb[2], bb[1]), (bb[2], bb[3]), (0, 0, 255), 3)
            # 人脸框的下横
            cv2.line(image, (bb[0], bb[3]), (bb[2], bb[3]), (0, 0, 255), 3)

            # 画出5个关键点
            cv2.circle(image, (points[0][i], points[5][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)
            cv2.circle(image, (points[1][i], points[6][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)
            cv2.circle(image, (points[2][i], points[7][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)
            cv2.circle(image, (points[3][i], points[8][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)
            cv2.circle(image, (points[4][i], points[9][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)

    cv2.imshow('mtcnn', image)

    if cv2.waitKey(1) == 27:
        break

cv2.destroyAllWindows()

上面while循环里就通过opencv一帧一帧地读取摄像头的数据，然后对每一帧数据进行人脸检测和人脸关键点检测，通过detect_face函数，这个以前的博客分析过这个函数了，这里就不赘述。

得到这些数据以后，我们再通过opencv将它们画出来，这里我们先只画出人脸框和5个关键点，一步一步来，不要猴急。完整代码如下，

import tensorflow as tf
import os
import detect_face
import cv2
import numpy as np


def main():

    # 读摄像头数据还是本地图片数据
    isFromCamera = True

    with tf.Session() as sess:
        #获取P-Net，R-Net，O-Net网络
        pnet, rnet, onet = detect_face.create_mtcnn(sess, None)

        minsize = 20  # minimum size of face
        threshold = [0.6, 0.7, 0.7]  # three steps's threshold
        factor = 0.709  # scale factor
        margin = 0

        # 看是读取摄像头数据呢还是图片呢
        if isFromCamera:
            cam = cv2.VideoCapture(0)
        else:
            imagefile = 'images/01.jpg'
            if os.path.exists(imagefile):
                image = cv2.imread(imagefile, cv2.IMREAD_COLOR)
            else:
                print('图片文件不存在！')
                return False

        while True:

            if isFromCamera:
                ret, image = cam.read()
                if not ret:
                    continue

            # 检测人脸，bounding_boxes可能包含多张人脸框数据，
            # 一张人脸框有5个数据，第一和第二个数据表示框左上角坐标，第三个第四个数据表示框右下角坐标，
            # 最后一个数据应该是可信度
            bounding_boxes, points = detect_face.detect_face(image, minsize, pnet, rnet, onet, threshold, factor)
            # print('bounding_boxes:', bounding_boxes)
            # print('points:', points)

            # 获得的人脸数量
            nrof_faces = bounding_boxes.shape[0]
            if nrof_faces > 0:
                det = bounding_boxes[:, 0:4]
                det_arr = []

                # 原图片大小
                img_size = np.asarray(image.shape)[0:2]
                if nrof_faces > 1:
                    # 如果要检测多张人脸的话
                    for i in range(nrof_faces):
                        det_arr.append(np.squeeze(det[i]))
                else:
                    # 只有一个人脸框的话，那就没得选了
                    det_arr.append(np.squeeze(det))

                for i, det in enumerate(det_arr):
                    det = np.squeeze(det)
                    bb = np.zeros(4, dtype=np.int32)
                    # 边界框周围的裁剪边缘，就是我们这里要裁剪的人脸框要比MTCNN获取的人脸框大一点，
                    # 至于大多少，就由margin参数决定了
                    bb[0] = np.maximum(det[0] - margin / 2, 0)
                    bb[1] = np.maximum(det[1] - margin / 2, 0)
                    bb[2] = np.minimum(det[2] + margin / 2, img_size[1])
                    bb[3] = np.minimum(det[3] + margin / 2, img_size[0])

                    # 人脸框的上横
                    cv2.line(image, (bb[0], bb[1]), (bb[2], bb[1]), (0, 0, 255), 3)
                    # 人脸框的左竖
                    cv2.line(image, (bb[0], bb[1]), (bb[0], bb[3]), (0, 0, 255), 3)
                    # 人脸框的右竖
                    cv2.line(image, (bb[2], bb[1]), (bb[2], bb[3]), (0, 0, 255), 3)
                    # 人脸框的下横
                    cv2.line(image, (bb[0], bb[3]), (bb[2], bb[3]), (0, 0, 255), 3)

                    # 画出5个关键点
                    cv2.circle(image, (points[0][i], points[5][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)
                    cv2.circle(image, (points[1][i], points[6][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)
                    cv2.circle(image, (points[2][i], points[7][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)
                    cv2.circle(image, (points[3][i], points[8][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)
                    cv2.circle(image, (points[4][i], points[9][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)

            cv2.imshow('mtcnn', image)

            if cv2.waitKey(1) == 27:
                break

        cv2.destroyAllWindows()

if __name__ == '__main__':
    main()

运行结果，

上面的结果是实时的啊，我只是截了个图。

4、框出眼睛

在上面的基础上，我们就可以做框出左右眼的操作了。新建文件demo2.py，将demo1.py的代码复制过来，在demo1的基础上修改。

导库什么的代码跟demo1是一样的，我们来看不一样的，

# 新建保存左右眼数据的文件夹
index = 0
leftEyePath = 'leftEye'
rightEyePath = 'rightEye'

if not os.path.exists(leftEyePath):
    os.mkdir(leftEyePath)

if not os.path.exists(rightEyePath):
    os.mkdir(rightEyePath)

上面的代码是创建两个文件夹，用来保存我们抠出来的左右眼眶的图像的。

while True:

    if isFromCamera:
        ret, image = cam.read()
        if not ret:
            continue

    # 检测人脸，bounding_boxes可能包含多张人脸框数据，
    # 一张人脸框有5个数据，第一和第二个数据表示框左上角坐标，第三个第四个数据表示框右下角坐标，
    # 最后一个数据应该是可信度
    bounding_boxes, points = detect_face.detect_face(image, minsize, pnet, rnet, onet, threshold, factor)
    # print('bounding_boxes:', bounding_boxes)
    # print('points:', points)

    # 获得的人脸数量
    nrof_faces = bounding_boxes.shape[0]
    if nrof_faces > 0:
        det = bounding_boxes[:, 0:4]
        det_arr = []
        # 原图片大小
        img_size = np.asarray(image.shape)[0:2]
        if nrof_faces > 1:
            # 如果要检测多张人脸的话
            for i in range(nrof_faces):
                det_arr.append(np.squeeze(det[i]))
        else:
            # 只有一个人脸框的话，那就没得选了
            det_arr.append(np.squeeze(det))

        for i, det in enumerate(det_arr):
            det = np.squeeze(det)
            bb = np.zeros(4, dtype=np.int32)
            # 边界框周围的裁剪边缘，就是我们这里要裁剪的人脸框要比MTCNN获取的人脸框大一点，
            # 至于大多少，就由margin参数决定了
            bb[0] = np.maximum(det[0] - margin / 2, 0)
            bb[1] = np.maximum(det[1] - margin / 2, 0)
            bb[2] = np.minimum(det[2] + margin / 2, img_size[1])
            bb[3] = np.minimum(det[3] + margin / 2, img_size[0])

这部分跟demo1还是一样的。

# 左右眼的距离
leftToRightDist = int(np.sqrt(np.square(points[1][i] - points[0][i]) + np.square(points[6][i] - points[5][i])))

# 图片的宽高
imageW = img_size[1]
imageH = img_size[0]

# 左眼框的左上角坐标
leftEyeLX = max(0, int(points[0][i] - leftToRightDist / 3))
leftEyeLY = max(0, int(points[5][i] - leftToRightDist / 6))
# 左眼框的右下角坐标
leftEyeRX = min(imageW, int(points[0][i] + leftToRightDist / 3))
leftEyeRY = min(imageH, int(points[5][i] + leftToRightDist / 6))

# 右眼框的左上角坐标
rightEyeLX = max(0, int(points[1][i] - leftToRightDist / 3))
rightEyeLY = max(0, int(points[6][i] - leftToRightDist / 6))
# 右眼框的右下角坐标
rightEyeRX = min(imageW, int(points[1][i] + leftToRightDist / 3))
rightEyeRY = min(imageH, int(points[6][i] + leftToRightDist / 6))

# 抠出左眼框图像数据
leftEyeImage = image[leftEyeLY:leftEyeRY, leftEyeLX:leftEyeRX, :]
# 抠出右眼框图像数据
lrightEyeImage = image[rightEyeLY:rightEyeRY, rightEyeLX:rightEyeRX, :]

# 将抠出来的左右眼眶数据保存到本地
cv2.imwrite(os.path.join(leftEyePath, leftEyePath + str(index) + '.jpg'), leftEyeImage)
cv2.imwrite(os.path.join(rightEyePath, rightEyePath + str(index) + '.jpg'), lrightEyeImage)
index += 1

# 画出左眼框上横
cv2.line(image, (leftEyeLX, leftEyeLY), (leftEyeRX, leftEyeLY), (0, 255, 0), 3)
# 画出左眼框左竖
cv2.line(image, (leftEyeLX, leftEyeLY), (leftEyeLX, leftEyeRY), (0, 255, 0), 3)
# 画出左眼框右竖
cv2.line(image, (leftEyeRX, leftEyeLY), (leftEyeRX, leftEyeRY), (0, 255, 0), 3)
# 画出左眼框下横
cv2.line(image, (leftEyeLX, leftEyeRY), (leftEyeRX, leftEyeRY), (0, 255, 0), 3)

# 画出右眼框上横
cv2.line(image, (rightEyeLX, rightEyeLY), (rightEyeRX, rightEyeLY), (0, 255, 0), 3)
# 画出右眼框左竖
cv2.line(image, (rightEyeLX, rightEyeLY), (rightEyeLX, rightEyeRY), (0, 255, 0), 3)
# 画出右眼框右竖
cv2.line(image, (rightEyeRX, rightEyeLY), (rightEyeRX, rightEyeRY), (0, 255, 0), 3)
# 画出右眼框下横
cv2.line(image, (rightEyeLX, rightEyeRY), (rightEyeRX, rightEyeRY), (0, 255, 0), 3)

上面就是我们抠出左右眼的算法了。先计算出左右眼之间的距离，然后，分别算出每个眼眶的左上角坐标和右上角坐标。然后就可以将眼睛抠出来了，为了比较直观的看，我们还将这个框也实时显示出来了，并将抠出来的左右眼睛保存到本地。完整代码如下，

import tensorflow as tf
import os
import detect_face
import cv2
import numpy as np


def main():

    # 读摄像头数据还是本地图片数据
    isFromCamera = True

    with tf.Session() as sess:
        #获取P-Net，R-Net，O-Net网络
        pnet, rnet, onet = detect_face.create_mtcnn(sess, None)

        minsize = 20  # minimum size of face
        threshold = [0.6, 0.7, 0.7]  # three steps's threshold
        factor = 0.709  # scale factor
        margin = 0

        # 看是读取摄像头数据呢还是图片呢
        if isFromCamera:
            cam = cv2.VideoCapture(0)
        else:
            imagefile = 'images/01.jpg'
            if os.path.exists(imagefile):
                image = cv2.imread(imagefile, cv2.IMREAD_COLOR)
            else:
                print('图片文件不存在！')
                return False


        # 新建保存左右眼数据的文件夹
        index = 0
        leftEyePath = 'leftEye'
        rightEyePath = 'rightEye'

        if not os.path.exists(leftEyePath):
            os.mkdir(leftEyePath)

        if not os.path.exists(rightEyePath):
            os.mkdir(rightEyePath)

        while True:

            if isFromCamera:
                ret, image = cam.read()
                if not ret:
                    continue

            # 检测人脸，bounding_boxes可能包含多张人脸框数据，
            # 一张人脸框有5个数据，第一和第二个数据表示框左上角坐标，第三个第四个数据表示框右下角坐标，
            # 最后一个数据应该是可信度
            bounding_boxes, points = detect_face.detect_face(image, minsize, pnet, rnet, onet, threshold, factor)
            # print('bounding_boxes:', bounding_boxes)
            # print('points:', points)

            # 获得的人脸数量
            nrof_faces = bounding_boxes.shape[0]
            if nrof_faces > 0:
                det = bounding_boxes[:, 0:4]
                det_arr = []
                # 原图片大小
                img_size = np.asarray(image.shape)[0:2]
                if nrof_faces > 1:
                    # 如果要检测多张人脸的话
                    for i in range(nrof_faces):
                        det_arr.append(np.squeeze(det[i]))
                else:
                    # 只有一个人脸框的话，那就没得选了
                    det_arr.append(np.squeeze(det))

                for i, det in enumerate(det_arr):
                    det = np.squeeze(det)
                    bb = np.zeros(4, dtype=np.int32)
                    # 边界框周围的裁剪边缘，就是我们这里要裁剪的人脸框要比MTCNN获取的人脸框大一点，
                    # 至于大多少，就由margin参数决定了
                    bb[0] = np.maximum(det[0] - margin / 2, 0)
                    bb[1] = np.maximum(det[1] - margin / 2, 0)
                    bb[2] = np.minimum(det[2] + margin / 2, img_size[1])
                    bb[3] = np.minimum(det[3] + margin / 2, img_size[0])

                    # 左右眼的距离
                    leftToRightDist = int(np.sqrt(np.square(points[1][i] - points[0][i]) + np.square(points[6][i] - points[5][i])))

                    # 图片的宽高
                    imageW = img_size[1]
                    imageH = img_size[0]

                    # 左眼框的左上角坐标
                    leftEyeLX = max(0, int(points[0][i] - leftToRightDist / 3))
                    leftEyeLY = max(0, int(points[5][i] - leftToRightDist / 6))
                    # 左眼框的右下角坐标
                    leftEyeRX = min(imageW, int(points[0][i] + leftToRightDist / 3))
                    leftEyeRY = min(imageH, int(points[5][i] + leftToRightDist / 6))

                    # 右眼框的左上角坐标
                    rightEyeLX = max(0, int(points[1][i] - leftToRightDist / 3))
                    rightEyeLY = max(0, int(points[6][i] - leftToRightDist / 6))
                    # 右眼框的右下角坐标
                    rightEyeRX = min(imageW, int(points[1][i] + leftToRightDist / 3))
                    rightEyeRY = min(imageH, int(points[6][i] + leftToRightDist / 6))

                    # 抠出左眼框图像数据
                    leftEyeImage = image[leftEyeLY:leftEyeRY, leftEyeLX:leftEyeRX, :]
                    # 抠出右眼框图像数据
                    lrightEyeImage = image[rightEyeLY:rightEyeRY, rightEyeLX:rightEyeRX, :]

                    # 将抠出来的左右眼眶数据保存到本地
                    cv2.imwrite(os.path.join(leftEyePath, leftEyePath + str(index) + '.jpg'), leftEyeImage)
                    cv2.imwrite(os.path.join(rightEyePath, rightEyePath + str(index) + '.jpg'), lrightEyeImage)
                    index += 1

                    # 画出左眼框上横
                    cv2.line(image, (leftEyeLX, leftEyeLY), (leftEyeRX, leftEyeLY), (0, 255, 0), 3)
                    # 画出左眼框左竖
                    cv2.line(image, (leftEyeLX, leftEyeLY), (leftEyeLX, leftEyeRY), (0, 255, 0), 3)
                    # 画出左眼框右竖
                    cv2.line(image, (leftEyeRX, leftEyeLY), (leftEyeRX, leftEyeRY), (0, 255, 0), 3)
                    # 画出左眼框下横
                    cv2.line(image, (leftEyeLX, leftEyeRY), (leftEyeRX, leftEyeRY), (0, 255, 0), 3)

                    # 画出右眼框上横
                    cv2.line(image, (rightEyeLX, rightEyeLY), (rightEyeRX, rightEyeLY), (0, 255, 0), 3)
                    # 画出右眼框左竖
                    cv2.line(image, (rightEyeLX, rightEyeLY), (rightEyeLX, rightEyeRY), (0, 255, 0), 3)
                    # 画出右眼框右竖
                    cv2.line(image, (rightEyeRX, rightEyeLY), (rightEyeRX, rightEyeRY), (0, 255, 0), 3)
                    # 画出右眼框下横
                    cv2.line(image, (rightEyeLX, rightEyeRY), (rightEyeRX, rightEyeRY), (0, 255, 0), 3)

                    # 人脸框的上横
                    cv2.line(image, (bb[0], bb[1]), (bb[2], bb[1]), (0, 0, 255), 3)
                    # 人脸框的左竖
                    cv2.line(image, (bb[0], bb[1]), (bb[0], bb[3]), (0, 0, 255), 3)
                    # 人脸框的右竖
                    cv2.line(image, (bb[2], bb[1]), (bb[2], bb[3]), (0, 0, 255), 3)
                    # 人脸框的下横
                    cv2.line(image, (bb[0], bb[3]), (bb[2], bb[3]), (0, 0, 255), 3)

                    # 画出5个关键点
                    cv2.circle(image, (points[0][i], points[5][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)
                    cv2.circle(image, (points[1][i], points[6][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)
                    cv2.circle(image, (points[2][i], points[7][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)
                    cv2.circle(image, (points[3][i], points[8][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)
                    cv2.circle(image, (points[4][i], points[9][i]), 1, (255, 0, 0), thickness=3, lineType=8, shift=0)

            cv2.imshow('mtcnn', image)

            if cv2.waitKey(1) == 27:
                break

        cv2.destroyAllWindows()

if __name__ == '__main__':
    main()

运行结果，