Skip to content
TheChatArchiveThe Chat Archive
o1-preview-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

import os import numpy as np import cv2 import subprocess import time import threading…

  1. User

    import os import numpy as np import cv2 import subprocess import time import threading import queue from scipy.io import wavfile from talkingface.audio_model import AudioModel from talkingface.render_model import RenderModel

    初始化模型

    audioModel = AudioModel() audioModel.loadModel("checkpoint/audio.pkl")

    renderModel = RenderModel() renderModel.loadModel("checkpoint/render.pth")

    定义参数

    sample_rate = 16000 frame_rate = 25 frame_duration = 1 / frame_rate running = True

    创建命名管道

    video_pipe_path = 'video_pipe' audio_pipe_path = 'audio_pipe'

    if not os.path.exists(video_pipe_path): os.mkfifo(video_pipe_path) if not os.path.exists(audio_pipe_path): os.mkfifo(audio_pipe_path)

    FFmpeg 命令

    ffmpeg_command = [ 'ffmpeg', '-y', '-probesize', '32', '-analyzeduration', '0', '-f', 'rawvideo', '-pix_fmt', 'bgr24', '-s', '640x480', '-r', str(frame_rate), '-i', video_pipe_path, '-f', 's16le', '-ar', str(sample_rate), '-ac', '1', '-i', audio_pipe_path, '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-b:a', '128k', '-f', 'flv', 'rtmp://localhost/live/livestream' ]

    启动 FFmpeg

    ffmpeg_process = subprocess.Popen(ffmpeg_command)

    数据队列

    data_queue = queue.Queue(maxsize=50) # 限制队列大小,防止内存占用过多

    def audio_thread(): audio_chunk_size = int(sample_rate * frame_duration) silence_chunk = np.zeros(audio_chunk_size, dtype=np.int16) audio_data_buffer = np.array([], dtype=np.int16)

    with open(audio_pipe_path, 'wb') as audio_pipe:
        while running:
            # 读取音频文件或使用静音数据
            # ...(省略音频文件读取代码)
    
            # 音频数据处理
            while len(audio_data_buffer) >= audio_chunk_size:
                current_chunk = audio_data_buffer[:audio_chunk_size]
                audio_data_buffer = audio_data_buffer[audio_chunk_size:]
    
                # 将音频数据写入音频管道
                audio_pipe.write(current_chunk.tobytes())
                audio_pipe.flush()
    
                # 生成 mouth_frame
                pcm_data = current_chunk.astype(np.float32) / 32768.0
                mouth_frame = audioModel.interface_frame(pcm_data)
    
                # 将数据放入队列
                data_queue.put(mouth_frame)
    
                time.sleep(frame_duration)
    

    def video_thread(): with open(video_pipe_path, 'wb') as video_pipe: while running: try: # 从队列中获取 mouth_frame mouth_frame = data_queue.get(timeout=1) except queue.Empty: continue

            # 生成视频帧
            frame = renderModel.interface(mouth_frame)
    
            # 确保帧大小匹配
            frame = cv2.resize(frame, (640, 480))
    
            # 将视频帧写入视频管道
            video_pipe.write(frame.tobytes())
            video_pipe.flush()
    
            time.sleep(frame_duration)
    

    启动线程

    audio_thread = threading.Thread(target=audio_thread) video_thread = threading.Thread(target=video_thread)

    audio_thread.start() video_thread.start()

    主线程等待

    try: while True: time.sleep(1) except KeyboardInterrupt: running = False audio_thread.join() video_thread.join() ffmpeg_process.terminate() if os.path.exists(video_pipe_path): os.remove(video_pipe_path) if os.path.exists(audio_pipe_path): os.remove(audio_pipe_path) pls modify it

  2. Assistant

    Certainly! It looks like your code is intended to create a live talking face generator that captures audio, processes it, and streams synchronized audio and video using FFmpeg to an RTMP server.

    However, the code you provided has a placeholder where the audio reading logic should be. I'll modify your code to include real-time audio capture from the microphone using the sounddevice library, handle threading properly, and make sure everything synchronizes correctly.

    Here's the modified code with explanations:

    import os
    import numpy as np
    import cv2
    import subprocess
    import time
    import threading
    import queue
    import sounddevice as sd
    # from scipy.io import wavfile  # Not needed since we're capturing real-time audio
    from talkingface.audio_model import AudioModel
    from talkingface.render_model import RenderModel
    
    # Initialize models
    audioModel = AudioModel()
    audioModel.loadModel("checkpoint/audio.pkl")
    
    renderModel = RenderModel()
    renderModel.loadModel("checkpoint/render.pth")
    
    # Parameters
    sample_rate = 16000       # Audio sample rate
    frame_rate = 25           # Video frame rate
    frame_duration = 1 / frame_rate
    running = True
    
    # Create named pipes for FFmpeg
    video_pipe_path = 'video_pipe'
    audio_pipe_path = 'audio_pipe'
    
    if not os.path.exists(video_pipe_path):
        os.mkfifo(video_pipe_path)
    if not os.path.exists(audio_pipe_path):
        os.mkfifo(audio_pipe_path)
    
    # FFmpeg command to combine audio and video and stream via RTMP
    ffmpeg_command = [
        'ffmpeg',
        '-y',
        '-f', 'rawvideo',
        '-pix_fmt', 'bgr24',
        '-s', '640x480',
        '-r', str(frame_rate),
        '-i', video_pipe_path,
        '-f', 's16le',
        '-ar', str(sample_rate),
        '-ac', '1',
        '-i', audio_pipe_path,
        '-c:v', 'libx264',
        '-pix_fmt', 'yuv420p',
        '-c:a', 'aac',
        '-b:a', '128k',
        '-f', 'flv',
        'rtmp://localhost/live/livestream'
    ]
    
    # Start FFmpeg subprocess
    ffmpeg_process = subprocess.Popen(ffmpeg_command)
    
    # Queue for communication between audio and video threads
    data_queue = queue.Queue(maxsize=50)  # Prevent excessive memory usage
    
    def audio_thread():
        audio_chunk_size = int(sample_rate * frame_duration)  # Number of samples per frame
        # Buffer to store audio data if needed
        # audio_data_buffer = np.array([], dtype=np.int16)  # Not needed since we're processing in real-time
    
        with open(audio_pipe_path, 'wb') as audio_pipe:
            # Define an audio stream callback function
            def callback(indata, frames, time_info, status):
                if status:
                    print(f"Audio Status: {status}", flush=True)
                current_chunk = indata.copy().flatten()
                # Convert float32 [-1.0, 1.0] to int16
                current_chunk_int16 = (current_chunk * 32768).astype(np.int16)
    
                # Write audio data to audio pipe
                audio_pipe.write(current_chunk_int16.tobytes())
                audio_pipe.flush()
    
                # Normalize audio data for the model
                pcm_data = current_chunk_int16.astype(np.float32) / 32768.0
    
                # Generate mouth_frame using the audio model
                mouth_frame = audioModel.interface_frame(pcm_data)
    
                # Put the mouth_frame into the queue
                try:
                    data_queue.put_nowait(mouth_frame)
                except queue.Full:
                    print("Data queue is full. Dropping frame.", flush=True)
    
            # Start the input stream
            with sd.InputStream(samplerate=sample_rate, channels=1, dtype='float32',
                                callback=callback, blocksize=audio_chunk_size):
                while running:
                    time.sleep(0.1)  # Keep the thread alive
    
    def video_thread():
        with open(video_pipe_path, 'wb') as video_pipe:
            while running:
                try:
                    # Get the mouth_frame from the queue
                    mouth_frame = data_queue.get(timeout=1)
                except queue.Empty:
                    print("No mouth_frame available. Skipping frame.", flush=True)
                    continue
    
                # Generate video frame using the render model
                frame = renderModel.interface(mouth_frame)
    
                # Resize frame to match specified output size
                frame_resized = cv2.resize(frame, (640, 480))
    
                # Write the video frame to the video pipe
                video_pipe.write(frame_resized.tobytes())
                video_pipe.flush()
    
                # Sleep to maintain video frame rate
                time.sleep(frame_duration)
    
    # Create and start the threads
    audio_thread_obj = threading.Thread(target=audio_thread)
    video_thread_obj = threading.Thread(target=video_thread)
    
    audio_thread_obj.start()
    video_thread_obj.start()
    
    # Main thread waits until interrupted
    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        print("Interrupted by user. Shutting down.", flush=True)
        running = False
        audio_thread_obj.join()
        video_thread_obj.join()
        ffmpeg_process.terminate()
        # Remove named pipes
        if os.path.exists(video_pipe_path):
            os.remove(video_pipe_path)
        if os.path.exists(audio_pipe_path):
            os.remove(audio_pipe_path)
        print("Cleanup complete.")
    

    Explanation of Modifications:

    1. Audio Capture with sounddevice:

      • Imported sounddevice to capture audio from the microphone in real-time.
      • Removed any unnecessary imports (wavfile) since we're capturing live audio.
      • In the audio_thread, we use sd.InputStream to capture audio. The callback function processes each audio block.
    2. Threading and Synchronization:

      • The callback function now processes audio data and immediately generates the mouth_frame, placing it into the data_queue.
      • Used data_queue.put_nowait() to prevent the callback from blocking if the queue is full, ensuring real-time performance.
    3. Data Types and Conversion:

      • Converted audio from float32 to int16 for compatibility with FFmpeg and the rest of the pipeline.
      • Normalized audio data before feeding it to the model.
    4. Video Thread Adjustments:

      • Added error handling for queue.Empty to ensure the video thread continues running smoothly even if no data is available.
      • Resized the output frame to match the expected dimensions for FFmpeg.
    5. Improved Logging and Error Handling:

      • Added print statements with flush=True to ensure messages are printed in real-time.
      • Provided feedback if the data queue is full or if frames are being dropped.
    6. Graceful Shutdown:

      • Ensured that all threads are joined and resources are cleaned up when the program is interrupted.
      • Removed named pipes after termination.

    Additional Notes:

    • Real-Time Performance: By processing audio data within the callback function, we ensure minimal latency and real-time performance.
    • Synchronization: The time.sleep() calls are adjusted to maintain synchronization without causing delays or race conditions.
    • Error Handling: The code now handles potential overflows, full queues, and other exceptions that may occur during real-time processing.

    Dependencies:

    Make sure you have the following libraries installed:

    pip install numpy opencv-python sounddevice
    

    Also, ensure that your AudioModel and RenderModel are correctly implemented and thread-safe.

    Testing:

    • Microphone Access: Ensure that your system allows microphone access to the Python script.
    • RTMP Server: Replace 'rtmp://localhost/live/livestream' with the actual RTMP server URL you're using.
    • Model Verification: Verify that audioModel.interface_frame() and renderModel.interface() functions are working as expected with the given input data.

    Troubleshooting:

    • Audio Issues: If you encounter issues with audio capture (e.g., permissions, device not found), check your microphone settings and ensure the correct default input device is selected.
    • Performance: If the system can't keep up in real-time, consider optimizing the model processing functions or running them on a GPU if possible.
    • FFmpeg Errors: Ensure FFmpeg is installed and accessible from your environment.

    Let me know if you need further assistance or additional modifications!