import cv2 import dlib import numpy as np from imutils import faceutils from…
- User
import cv2 import dlib import numpy as np from imutils import face_utils from scipy.spatial import distance
detector = dlib.get_frontal_face_detector() landmark_predictor = dlib.shape_predictor('./models/shape_predictor_68_face_landmarks.dat')
def get_center(shape): center_x = int(round(np.sum(shape[:, 0]) / 68)) center_y = int(round(np.sum(shape[:, 1]) / 68)) - 25 return center_x, center_y
def crop_face(vc, norm_x, norm_y, vid_writer, crop_size=224): print('Crop face..') idx = 0 scale_factor = None prev_center_x = None prev_center_y = None prev_roiX1 = None prev_roiY1 = None prev_roiX2 = None prev_roiY2 = None
while True: bgr_img = vc.read()[1] if bgr_img is None: break rgb_img = cv2.cvtColor(bgr_img, cv2.COLOR_BGR2RGB) # dlib face detection dlib_rects = detector(rgb_img, 1) # dlib bbox detection list_dlib_rect = list(dlib_rects) # Exception handling 1: Detected more than one, find closest to given coordinate. if len(list_dlib_rect) > 1: frame_h, frame_w, _ = rgb_img.shape abs_x = round(norm_x * frame_w) abs_y = round(norm_y * frame_h) closest_rect = None closest_distance = 9999 for k, rect in enumerate(list_dlib_rect): (x, y, w, h) = face_utils.rect_to_bb(rect) center_x = (x + (x + w)) / 2 center_y = (y + (y + h)) / 2 cur_distance = distance.euclidean((abs_x, abs_y), (center_x, center_y)) if cur_distance < closest_distance: closest_rect = rect closest_distance = cur_distance list_dlib_rect = [closest_rect] # Predict landmark list_landmarks = [] for rect in list_dlib_rect: shape = landmark_predictor(rgb_img, rect) list_points = list(map(lambda p: (p.x, p.y), shape.parts())) list_landmarks.append(list_points) shape = face_utils.shape_to_np(shape) center_x, center_y = get_center(shape) # Calculate inter-ocular distance at first detected frame and resize image. if scale_factor is None: left_eye_x = (shape[36][0] + shape[39][0]) / 2 left_eye_y = (shape[38][1] + shape[40][1]) / 2 right_eye_x = (shape[42][0] + shape[45][0]) / 2 right_eye_y = (shape[43][1] + shape[46][1]) / 2 interocular_distance = round( distance.euclidean((left_eye_x, left_eye_y), (right_eye_x, right_eye_y))) scale_factor = 55 / interocular_distance rgb_img = cv2.resize(rgb_img, None, fx=scale_factor, fy=scale_factor) frame_h, frame_w, _ = rgb_img.shape abs_x = round(norm_x * frame_w) abs_y = round(norm_y * frame_h) # At first detected frame, set previous center coordinate as given coordinate if prev_center_x is None and prev_center_y is None: prev_center_x, prev_center_y = abs_x, abs_y center_x = round(center_x * scale_factor) center_y = round(center_y * scale_factor) # Exception handling 2: Detected face center is too far from previous frame's center. # There's a possibiliy that detected face is not target's. # 1. At first frame, skip this video # 2. or just use previous center as current frame's center. if abs(center_x - prev_center_x) > 62 or abs(center_y - prev_center_y) > 42: if idx == 0: return False center_x = prev_center_x center_y = prev_center_y else: prev_center_x = center_x prev_center_y = center_y # Crop face and save landamrk. margin = int(crop_size / 2) if len(list_landmarks) != 0: roiX1 = center_x - margin if center_x - margin > 0 else 0 roiY1 = center_y - margin if center_y - margin > 0 else 0 roiX2 = center_x + margin if center_x + margin < rgb_img.shape[1] else rgb_img.shape[1] roiY2 = center_y + margin if center_y + margin < rgb_img.shape[0] else rgb_img.shape[0] else: # Exception handling 3: Detected nothing in, # 1. at (second to last) frame, use same ROI of previous frame. # 2. or, at frist frame, skip this video. if prev_roiX1 is not None: if scale_factor is not None: rgb_img = cv2.resize(rgb_img, None, fx=scale_factor, fy=scale_factor) roiX1 = prev_roiX1 roiY1 = prev_roiY1 roiX2 = prev_roiX2 roiY2 = prev_roiY2 else: return False prev_roiX1 = roiX1 prev_roiY1 = roiY1 prev_roiX2 = roiX2 prev_roiY2 = roiY2 img_face = rgb_img[roiY1:roiY2, roiX1:roiX2].copy() img_face = cv2.cvtColor(img_face, cv2.COLOR_RGB2BGR) img_face = cv2.resize(img_face, (224, 224)) vid_writer.write(img_face) idx += 1 print('Cropping is done.') return Truecropped_path = r'D:\2023project\project\0win_os\05speech_Pro\diff2lip\thirdparty\wav2lip384_my2\data_test\crop01.mp4' fourcc = cv2.VideoWriter_fourcc(*'avc1') vid_writer = cv2.VideoWriter(cropped_path, fourcc, 25, (224, 224)) cut_video_path_ext = r'D:\2023project\project\0win_os\05speech_Pro\diff2lip\thirdparty\wav2lip384_my2\data_test__BsfDOm0zU_139.575767-142.575767.mp4' vc= cv2.VideoCapture(cut_video_path_ext) norm_x, norm_y= 0.271875,0.345833 flag = crop_face(vc, norm_x, norm_y, vid_writer, crop_size=224)
将人脸检测detector = dlib.get_frontal_face_detector() landmark_predictor = dlib.shape_predictor('./models/shape_predictor_68_face_landmarks.dat') 修改为 https://github.com/deepinsight/insightface onnx 实现方式,中文回复
- Assistant
以下是使用 InsightFace 替换原始代码中 dlib 人脸检测和关键点识别的实现方式。
首先,需要安装 InsightFace 库:
pip install insightface然后,修改代码如下:
import cv2 import numpy as np import insightface from insightface.app import FaceAnalysis from scipy.spatial import distance # 初始化 InsightFace 的人脸分析模型 app = FaceAnalysis(name='buffalo_l') # 使用包含106个关键点的模型 app.prepare(ctx_id=0, nms=0.4) def get_center(landmarks): # 根据106个关键点计算人脸中心 center_x = int(round(np.sum(landmarks[:, 0]) / 106)) center_y = int(round(np.sum(landmarks[:, 1]) / 106)) - 25 return center_x, center_y def crop_face(vc, norm_x, norm_y, vid_writer, crop_size=224): print('正在裁剪人脸...') idx = 0 scale_factor = None prev_center_x = None prev_center_y = None prev_roiX1 = None prev_roiY1 = None prev_roiX2 = None prev_roiY2 = None while True: ret, bgr_img = vc.read() if not ret or bgr_img is None: break # 使用 InsightFace 进行人脸检测和关键点提取 faces = app.get(bgr_img) # 异常处理1:检测到多个人脸,选择最接近给定坐标的那个人脸 if len(faces) > 1: frame_h, frame_w, _ = bgr_img.shape abs_x = round(norm_x * frame_w) abs_y = round(norm_y * frame_h) closest_face = None closest_distance = 9999 for face in faces: bbox = face.bbox.astype(np.int) center_x = (bbox[0] + bbox[2]) / 2 center_y = (bbox[1] + bbox[3]) / 2 cur_distance = distance.euclidean((abs_x, abs_y), (center_x, center_y)) if cur_distance < closest_distance: closest_face = face closest_distance = cur_distance faces = [closest_face] # 提取关键点 list_landmarks = [] for face in faces: landmarks = face.landmark_2d_106 # 获取106个关键点 list_landmarks.append(landmarks) center_x, center_y = get_center(landmarks) # 计算两眼之间的距离,用于确定缩放比例 if scale_factor is None: # 根据关键点索引获取左眼和右眼的关键点 left_eye = landmarks[60:68, :] # 左眼区域(索引可能需要根据模型调整) right_eye = landmarks[68:76, :] # 右眼区域(索引可能需要根据模型调整) left_eye_center = np.mean(left_eye, axis=0) right_eye_center = np.mean(right_eye, axis=0) interocular_distance = round(distance.euclidean(left_eye_center, right_eye_center)) scale_factor = 55 / interocular_distance bgr_img = cv2.resize(bgr_img, None, fx=scale_factor, fy=scale_factor) frame_h, frame_w, _ = bgr_img.shape abs_x = round(norm_x * frame_w) abs_y = round(norm_y * frame_h) # 在初始帧,将前一帧的中心坐标设置为给定坐标 if prev_center_x is None and prev_center_y is None: prev_center_x, prev_center_y = abs_x, abs_y center_x = round(center_x * scale_factor) center_y = round(center_y * scale_factor) # 异常处理2:检测到的人脸中心与前一帧的中心距离过大,可能不是目标人脸 if abs(center_x - prev_center_x) > 62 or abs(center_y - prev_center_y) > 42: if idx == 0: return False center_x = prev_center_x center_y = prev_center_y else: prev_center_x = center_x prev_center_y = center_y # 裁剪人脸并保存 margin = int(crop_size / 2) if len(list_landmarks) != 0: roiX1 = center_x - margin if center_x - margin > 0 else 0 roiY1 = center_y - margin if center_y - margin > 0 else 0 roiX2 = center_x + margin if center_x + margin < bgr_img.shape[1] else bgr_img.shape[1] roiY2 = center_y + margin if center_y + margin < bgr_img.shape[0] else bgr_img.shape[0] else: # 异常处理3:未检测到人脸 if prev_roiX1 is not None: if scale_factor is not None: bgr_img = cv2.resize(bgr_img, None, fx=scale_factor, fy=scale_factor) roiX1 = prev_roiX1 roiY1 = prev_roiY1 roiX2 = prev_roiX2 roiY2 = prev_roiY2 else: return False prev_roiX1 = roiX1 prev_roiY1 = roiY1 prev_roiX2 = roiX2 prev_roiY2 = roiY2 img_face = bgr_img[roiY1:roiY2, roiX1:roiX2].copy() img_face = cv2.resize(img_face, (224, 224)) vid_writer.write(img_face) idx += 1 print('裁剪完成。') return True # 示例调用 cropped_path = r'D:\2023project\project\0win_os\05speech_Pro\diff2lip\thirdparty\wav2lip384_my2\data_test\crop01.mp4' fourcc = cv2.VideoWriter_fourcc(*'avc1') vid_writer = cv2.VideoWriter(cropped_path, fourcc, 25, (224, 224)) video_path = r'D:\2023project\project\0win_os\05speech_Pro\diff2lip\thirdparty\wav2lip384_my2\data_test\__BsfDOm0zU_139.575767-142.575767.mp4' vc = cv2.VideoCapture(video_path) norm_x, norm_y = 0.271875, 0.345833 flag = crop_face(vc, norm_x, norm_y, vid_writer, crop_size=224)说明:
-
引入 InsightFace 库并初始化模型:使用
app = FaceAnalysis(name='buffalo_l')初始化人脸分析模型,该模型包含人脸检测和 106 个关键点检测功能。app.prepare(ctx_id=0, nms=0.4)用于加载模型,需要注意ctx_id=-1表示使用 CPU,ctx_id=0表示使用 GPU。 -
替换人脸检测和关键点提取:在循环中,使用
faces = app.get(bgr_img)进行人脸检测和关键点提取。faces是检测到的人脸列表,每个人脸都有bbox(边界框)和landmark_2d_106(106 个关键点)。 -
处理多个人脸的情况:如果检测到多个人脸,计算每个人脸中心与给定坐标的距离,选择距离最近的人脸进行后续处理。
-
计算人脸中心和缩放比例:使用
get_center函数计算人脸中心位置。为了计算缩放比例,我们使用两眼之间的距离。需要根据实际模型的关键点索引获取左眼和右眼的关键点集合。 -
异常处理:与原始代码类似,如果检测到的人脸中心位置与前一帧偏差过大,可能不是目标人脸,此时使用前一帧的人脸中心位置。
-
裁剪人脸并保存:按照计算的 ROI(Region of Interest) 裁剪人脸区域,将其调整为指定大小后写入视频文件。
注意事项:
-
关键点索引:由于使用的是 106 个关键点的模型,左眼和右眼的关键点索引用于计算两眼之间的距离。上述代码中的索引(左眼 60-67,右眼 68-75)可能需要根据 InsightFace 的关键点定义进行调整。可以通过打印
landmarks数组来确认关键点的位置。 -
性能:InsightFace 的模型在 GPU 上运行速度较快,如果使用 CPU 可能会较慢。
-
模型文件:第一次运行时,InsightFace 会自动下载所需的模型文件。如果下载速度较慢,可以手动从官方仓库下载模型文件并放置在指定目录。
示例调用:
在示例中,我们指定了输入视频路径和输出视频路径,调用
crop_face函数进行人脸裁剪。
希望以上修改和说明能够满足您的需求,实现将人脸检测和关键点识别从 dlib 替换为 InsightFace 的实现方式。如有疑问,欢迎继续提问。
-