Skip to content
The Chat Archive
o1-preview-2024-09-12· OpenAI WildChat-4.8M (AI2)

import cv2 import dlib import numpy as np from imutils import faceutils from…

  1. User

    import cv2 import dlib import numpy as np from imutils import face_utils from scipy.spatial import distance

    detector = dlib.get_frontal_face_detector() landmark_predictor = dlib.shape_predictor('./models/shape_predictor_68_face_landmarks.dat')

    def get_center(shape): center_x = int(round(np.sum(shape[:, 0]) / 68)) center_y = int(round(np.sum(shape[:, 1]) / 68)) - 25 return center_x, center_y

    def crop_face(vc, norm_x, norm_y, vid_writer, crop_size=224): print('Crop face..') idx = 0 scale_factor = None prev_center_x = None prev_center_y = None prev_roiX1 = None prev_roiY1 = None prev_roiX2 = None prev_roiY2 = None

    while True:
        bgr_img = vc.read()[1]
        if bgr_img is None:
            break
        rgb_img = cv2.cvtColor(bgr_img, cv2.COLOR_BGR2RGB)
        # dlib face detection
        dlib_rects = detector(rgb_img, 1)  # dlib bbox detection
        list_dlib_rect = list(dlib_rects)
    
        # Exception handling 1: Detected more than one, find closest to given coordinate.
        if len(list_dlib_rect) > 1:
            frame_h, frame_w, _ = rgb_img.shape
            abs_x = round(norm_x * frame_w)
            abs_y = round(norm_y * frame_h)
            closest_rect = None
            closest_distance = 9999
            for k, rect in enumerate(list_dlib_rect):
                (x, y, w, h) = face_utils.rect_to_bb(rect)
                center_x = (x + (x + w)) / 2
                center_y = (y + (y + h)) / 2
                cur_distance = distance.euclidean((abs_x, abs_y), (center_x, center_y))
                if cur_distance < closest_distance:
                    closest_rect = rect
                    closest_distance = cur_distance
            list_dlib_rect = [closest_rect]
    
        # Predict landmark
        list_landmarks = []
        for rect in list_dlib_rect:
            shape = landmark_predictor(rgb_img, rect)
            list_points = list(map(lambda p: (p.x, p.y), shape.parts()))
            list_landmarks.append(list_points)
            shape = face_utils.shape_to_np(shape)
            center_x, center_y = get_center(shape)
            # Calculate inter-ocular distance at first detected frame and resize image.
            if scale_factor is None:
                left_eye_x = (shape[36][0] + shape[39][0]) / 2
                left_eye_y = (shape[38][1] + shape[40][1]) / 2
                right_eye_x = (shape[42][0] + shape[45][0]) / 2
                right_eye_y = (shape[43][1] + shape[46][1]) / 2
                interocular_distance = round(
                    distance.euclidean((left_eye_x, left_eye_y), (right_eye_x, right_eye_y)))
                scale_factor = 55 / interocular_distance
            rgb_img = cv2.resize(rgb_img, None, fx=scale_factor, fy=scale_factor)
            frame_h, frame_w, _ = rgb_img.shape
            abs_x = round(norm_x * frame_w)
            abs_y = round(norm_y * frame_h)
    
            # At first detected frame, set previous center coordinate as given coordinate
            if prev_center_x is None and prev_center_y is None:
                prev_center_x, prev_center_y = abs_x, abs_y
    
            center_x = round(center_x * scale_factor)
            center_y = round(center_y * scale_factor)
    
            # Exception handling 2: Detected face center is too far from previous frame's center.
            # There's a possibiliy that detected face is not target's.
            # 1. At first frame, skip this video
            # 2. or just use previous center as current frame's center.
            if abs(center_x - prev_center_x) > 62 or abs(center_y - prev_center_y) > 42:
                if idx == 0:
                    return False
                center_x = prev_center_x
                center_y = prev_center_y
            else:
                prev_center_x = center_x
                prev_center_y = center_y
    
        # Crop face and save landamrk.
        margin = int(crop_size / 2)
        if len(list_landmarks) != 0:
            roiX1 = center_x - margin if center_x - margin > 0 else 0
            roiY1 = center_y - margin if center_y - margin > 0 else 0
            roiX2 = center_x + margin if center_x + margin < rgb_img.shape[1] else rgb_img.shape[1]
            roiY2 = center_y + margin if center_y + margin < rgb_img.shape[0] else rgb_img.shape[0]
        else:
            # Exception handling 3: Detected nothing in,
            # 1. at (second to last) frame, use same ROI of previous frame.
            # 2. or, at frist frame, skip this video.
            if prev_roiX1 is not None:
                if scale_factor is not None:
                    rgb_img = cv2.resize(rgb_img, None, fx=scale_factor, fy=scale_factor)
                roiX1 = prev_roiX1
                roiY1 = prev_roiY1
                roiX2 = prev_roiX2
                roiY2 = prev_roiY2
            else:
                return False
        prev_roiX1 = roiX1
        prev_roiY1 = roiY1
        prev_roiX2 = roiX2
        prev_roiY2 = roiY2
        img_face = rgb_img[roiY1:roiY2, roiX1:roiX2].copy()
        img_face = cv2.cvtColor(img_face, cv2.COLOR_RGB2BGR)
        img_face = cv2.resize(img_face, (224, 224))
        vid_writer.write(img_face)
        idx += 1
    print('Cropping is done.')
    return True
    

    cropped_path = r'D:\2023project\project\0win_os\05speech_Pro\diff2lip\thirdparty\wav2lip384_my2\data_test\crop01.mp4' fourcc = cv2.VideoWriter_fourcc(*'avc1') vid_writer = cv2.VideoWriter(cropped_path, fourcc, 25, (224, 224)) cut_video_path_ext = r'D:\2023project\project\0win_os\05speech_Pro\diff2lip\thirdparty\wav2lip384_my2\data_test__BsfDOm0zU_139.575767-142.575767.mp4' vc= cv2.VideoCapture(cut_video_path_ext) norm_x, norm_y= 0.271875,0.345833 flag = crop_face(vc, norm_x, norm_y, vid_writer, crop_size=224)

    将人脸检测detector = dlib.get_frontal_face_detector() landmark_predictor = dlib.shape_predictor('./models/shape_predictor_68_face_landmarks.dat') 修改为 https://github.com/deepinsight/insightface onnx 实现方式,中文回复

  2. Assistant

    以下是使用 InsightFace 替换原始代码中 dlib 人脸检测和关键点识别的实现方式。

    首先,需要安装 InsightFace 库:

    pip install insightface
    

    然后,修改代码如下:

    import cv2
    import numpy as np
    import insightface
    from insightface.app import FaceAnalysis
    from scipy.spatial import distance
    
    # 初始化 InsightFace 的人脸分析模型
    app = FaceAnalysis(name='buffalo_l')  # 使用包含106个关键点的模型
    app.prepare(ctx_id=0, nms=0.4)
    
    def get_center(landmarks):
        # 根据106个关键点计算人脸中心
        center_x = int(round(np.sum(landmarks[:, 0]) / 106))
        center_y = int(round(np.sum(landmarks[:, 1]) / 106)) - 25
        return center_x, center_y
    
    def crop_face(vc, norm_x, norm_y, vid_writer, crop_size=224):
        print('正在裁剪人脸...')
        idx = 0
        scale_factor = None
        prev_center_x = None
        prev_center_y = None
        prev_roiX1 = None
        prev_roiY1 = None
        prev_roiX2 = None
        prev_roiY2 = None
    
        while True:
            ret, bgr_img = vc.read()
            if not ret or bgr_img is None:
                break
    
            # 使用 InsightFace 进行人脸检测和关键点提取
            faces = app.get(bgr_img)
    
            # 异常处理1:检测到多个人脸,选择最接近给定坐标的那个人脸
            if len(faces) > 1:
                frame_h, frame_w, _ = bgr_img.shape
                abs_x = round(norm_x * frame_w)
                abs_y = round(norm_y * frame_h)
                closest_face = None
                closest_distance = 9999
                for face in faces:
                    bbox = face.bbox.astype(np.int)
                    center_x = (bbox[0] + bbox[2]) / 2
                    center_y = (bbox[1] + bbox[3]) / 2
                    cur_distance = distance.euclidean((abs_x, abs_y), (center_x, center_y))
                    if cur_distance < closest_distance:
                        closest_face = face
                        closest_distance = cur_distance
                faces = [closest_face]
    
            # 提取关键点
            list_landmarks = []
            for face in faces:
                landmarks = face.landmark_2d_106  # 获取106个关键点
                list_landmarks.append(landmarks)
                center_x, center_y = get_center(landmarks)
                # 计算两眼之间的距离,用于确定缩放比例
                if scale_factor is None:
                    # 根据关键点索引获取左眼和右眼的关键点
                    left_eye = landmarks[60:68, :]  # 左眼区域(索引可能需要根据模型调整)
                    right_eye = landmarks[68:76, :]  # 右眼区域(索引可能需要根据模型调整)
                    left_eye_center = np.mean(left_eye, axis=0)
                    right_eye_center = np.mean(right_eye, axis=0)
                    interocular_distance = round(distance.euclidean(left_eye_center, right_eye_center))
                    scale_factor = 55 / interocular_distance
                bgr_img = cv2.resize(bgr_img, None, fx=scale_factor, fy=scale_factor)
                frame_h, frame_w, _ = bgr_img.shape
                abs_x = round(norm_x * frame_w)
                abs_y = round(norm_y * frame_h)
    
                # 在初始帧,将前一帧的中心坐标设置为给定坐标
                if prev_center_x is None and prev_center_y is None:
                    prev_center_x, prev_center_y = abs_x, abs_y
    
                center_x = round(center_x * scale_factor)
                center_y = round(center_y * scale_factor)
    
                # 异常处理2:检测到的人脸中心与前一帧的中心距离过大,可能不是目标人脸
                if abs(center_x - prev_center_x) > 62 or abs(center_y - prev_center_y) > 42:
                    if idx == 0:
                        return False
                    center_x = prev_center_x
                    center_y = prev_center_y
                else:
                    prev_center_x = center_x
                    prev_center_y = center_y
    
            # 裁剪人脸并保存
            margin = int(crop_size / 2)
            if len(list_landmarks) != 0:
                roiX1 = center_x - margin if center_x - margin > 0 else 0
                roiY1 = center_y - margin if center_y - margin > 0 else 0
                roiX2 = center_x + margin if center_x + margin < bgr_img.shape[1] else bgr_img.shape[1]
                roiY2 = center_y + margin if center_y + margin < bgr_img.shape[0] else bgr_img.shape[0]
            else:
                # 异常处理3:未检测到人脸
                if prev_roiX1 is not None:
                    if scale_factor is not None:
                        bgr_img = cv2.resize(bgr_img, None, fx=scale_factor, fy=scale_factor)
                    roiX1 = prev_roiX1
                    roiY1 = prev_roiY1
                    roiX2 = prev_roiX2
                    roiY2 = prev_roiY2
                else:
                    return False
            prev_roiX1 = roiX1
            prev_roiY1 = roiY1
            prev_roiX2 = roiX2
            prev_roiY2 = roiY2
            img_face = bgr_img[roiY1:roiY2, roiX1:roiX2].copy()
            img_face = cv2.resize(img_face, (224, 224))
            vid_writer.write(img_face)
            idx += 1
        print('裁剪完成。')
        return True
    
    # 示例调用
    cropped_path = r'D:\2023project\project\0win_os\05speech_Pro\diff2lip\thirdparty\wav2lip384_my2\data_test\crop01.mp4'
    fourcc = cv2.VideoWriter_fourcc(*'avc1')
    vid_writer = cv2.VideoWriter(cropped_path,
                                 fourcc,
                                 25,
                                 (224, 224))
    video_path = r'D:\2023project\project\0win_os\05speech_Pro\diff2lip\thirdparty\wav2lip384_my2\data_test\__BsfDOm0zU_139.575767-142.575767.mp4'
    vc = cv2.VideoCapture(video_path)
    norm_x, norm_y = 0.271875, 0.345833
    flag = crop_face(vc, norm_x, norm_y, vid_writer, crop_size=224)
    

    说明:

    1. 引入 InsightFace 库并初始化模型:使用 app = FaceAnalysis(name='buffalo_l') 初始化人脸分析模型,该模型包含人脸检测和 106 个关键点检测功能。app.prepare(ctx_id=0, nms=0.4) 用于加载模型,需要注意 ctx_id=-1 表示使用 CPU,ctx_id=0 表示使用 GPU。

    2. 替换人脸检测和关键点提取:在循环中,使用 faces = app.get(bgr_img) 进行人脸检测和关键点提取。faces 是检测到的人脸列表,每个人脸都有 bbox(边界框)和 landmark_2d_106(106 个关键点)。

    3. 处理多个人脸的情况:如果检测到多个人脸,计算每个人脸中心与给定坐标的距离,选择距离最近的人脸进行后续处理。

    4. 计算人脸中心和缩放比例:使用 get_center 函数计算人脸中心位置。为了计算缩放比例,我们使用两眼之间的距离。需要根据实际模型的关键点索引获取左眼和右眼的关键点集合。

    5. 异常处理:与原始代码类似,如果检测到的人脸中心位置与前一帧偏差过大,可能不是目标人脸,此时使用前一帧的人脸中心位置。

    6. 裁剪人脸并保存:按照计算的 ROI(Region of Interest) 裁剪人脸区域,将其调整为指定大小后写入视频文件。

    注意事项:

    • 关键点索引:由于使用的是 106 个关键点的模型,左眼和右眼的关键点索引用于计算两眼之间的距离。上述代码中的索引(左眼 60-67,右眼 68-75)可能需要根据 InsightFace 的关键点定义进行调整。可以通过打印 landmarks 数组来确认关键点的位置。

    • 性能:InsightFace 的模型在 GPU 上运行速度较快,如果使用 CPU 可能会较慢。

    • 模型文件:第一次运行时,InsightFace 会自动下载所需的模型文件。如果下载速度较慢,可以手动从官方仓库下载模型文件并放置在指定目录。

    示例调用:

    在示例中,我们指定了输入视频路径和输出视频路径,调用 crop_face 函数进行人脸裁剪。


    希望以上修改和说明能够满足您的需求,实现将人脸检测和关键点识别从 dlib 替换为 InsightFace 的实现方式。如有疑问,欢迎继续提问。