Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-05-13· OpenAIAi2 WildChat-4.8M (AI2)

在# --coding:utf-8-- import torch from utils.loadingutils import loadmodel, getdevice…

  1. User

    在# --coding:utf-8-- import torch from utils.loading_utils import load_model, get_device import numpy as np import argparse import pandas as pd import os import cv2 import pdb

    the following functions are borrowed from: https://github.com/uzh-rpg/rpg_e2vid

    from utils.event_readers import FixedSizeEventReader, FixedDurationEventReader from utils.inference_utils import events_to_voxel_grid, events_to_voxel_grid_pytorch from utils.timers import Timer

    import time from image_reconstructor import ImageReconstructor from options.inference_options import set_inference_options from PIL import Image from dv import AedatFile

    def extract_davis(aedat_file_path, dvs_img_interval): frame_no = 0 frame_all = [] frame_exposure_time = [] frame_interval_time = [] # use_mode = 'frame_exposure_time' #### default mode use_mode = 'frame_interval_time' with AedatFile(aedat_file_path) as f: # list all the names of streams in the file print(f.names) # extract timestamps of each frame for frame in f['frames']: frame_all.append(frame.image) frame_exposure_time.append([frame.timestamp_start_of_exposure, frame.timestamp_end_of_exposure]) ## [1607928583397102, 1607928583401102] frame_interval_time.append( [frame.timestamp_start_of_frame, frame.timestamp_end_of_frame]) ## [1607928583387944, 1607928583410285]

            # pdb.set_trace()
    
        if use_mode == 'frame_exposure_time':
            frame_timestamp = frame_exposure_time
        elif use_mode == 'frame_interval_time':
            frame_timestamp = frame_interval_time
    
        frame_num = len(frame_timestamp)
        # Access dimensions of the event stream
        height, width = f['events'].size  # 260  346
        event_frame = 255 * np.ones((height, width, 3), dtype=np.uint8)
        # loop through the "events" stream
        # events = np.hstack([packet for packet in f['events'].numpy()])
    
        # # Access information of all events by type
        # timestamps, x, y, polarities = events['timestamp'], events['x'], events['y'], events['polarity']
        # pdb.set_trace()
        # save event_img
        idx = np.round(np.linspace(0, len(frame_timestamp) - 1, int(frame_num / dvs_img_interval))).astype(
            int)  ## frame index [0, 1, 2, ... , 3847]
        frame_timestamp = np.array(frame_timestamp)[idx]  ## (3848, 2)
    
        # another reader # speed up
        # events will be a named numpy array
        events = np.hstack([packet for packet in f['events'].numpy()])
        timestamps, x, y, polarities = events['timestamp'], events['x'], events['y'], events['polarity']
    
        return timestamps, x, y, polarities, frame_num, frame_timestamp
    
    #######################################################################################################
    

    Main Function

    ####################################################################################################### if name == "main": device = torch.device("cuda:0") # data_path = r"/home/ioe/xxxxx/EventTracking/COESOT/train" data_path = r"/data2/local_userdata/heyuting/datasets/COESOT/test" save_path = r"/data2/local_userdata/heyuting/datasets/COESOT/test" video_files = os.listdir(data_path) # 由于数据集中有txt文件,因此需要排除 video_files = [file for file in video_files if file not in ('list.txt', 'val.txt')] for videoID in range(len(video_files)): foldName = video_files[videoID] print("==>> foldName: ", foldName) if foldName in ['dvSave-2022_01_26_20_01_46', 'dvSave-2022_03_21_11_09_50', 'dvSave-2022_03_21_11_12_27', 'dvSave-2022_03_21_11_19_37', 'dvSave-2022_02_25_13_21_41', 'dvSave-2022_02_25_13_31_18', 'dvSave-2022_03_21_09_05_49']: print("==>> pass it: ", foldName) continue # if foldName not in ['dvSave-2021_12_21_16_26_09']: # print("==>> pass it: ", foldName) # continue

        # if foldName not in ['dvSave-2022_08_21_13_21_14']:
        #     continue
    
        aedat_file_path = os.path.join(data_path, foldName, foldName+'.aedat4')
        save_img_path = os.path.join(save_path, foldName, foldName+'_recon')
        if not os.path.exists(save_img_path):
            os.makedirs(save_img_path)
    # dvs_img_interval = 1
    # for root, dirs, files in os.walk(aedat_path):
    #     for file in files:
    #         (filename, extension) = os.path.splitext(file)
            # print("==>> filename: ", filename)
            # if (extension == '.aedat4'):
            #     if filename + '_aps' in os.listdir(root) or filename + '_dvs' in os.listdir(root):
            #         print("==>> Skip this video ... ")
            #         continue
    
                    # save_path_dvs = os.path.join(root, filename, filename+'_dvs')
                # save_path_aps = os.path.join(root, filename, filename+'_aps')
                # aedat_file_path = os.path.join(root, filename + '.aedat4')
                # filename_txt_path = os.path.join(root, filename+'_timestamp.txt')
                # croped_events_path = os.path.join(root, filename+'_events.txt')
    
                # if not os.path.exists(save_path_dvs):
                #     os.makedirs(save_path_dvs)
                # if not os.path.exists(save_path_aps):
                #     os.makedirs(save_path_aps)
    
        timestamps, x, y, polarities, frame_num, frame_timestamp = extract_davis(aedat_file_path, 1)  ## (4391964,)
    
        # all_events = [timestamps, x, y, polarities]
    
        all_events = np.zeros((timestamps.shape[0], 4))
    
        all_events[:, 0] = timestamps
        all_events[:, 1] = x
        all_events[:, 2] = y
        all_events[:, 3] = polarities
        # pdb.set_trace()
    
        parser = argparse.ArgumentParser(description='Evaluating a trained network')
        parser.add_argument('-c', '--path_to_model', required=True, type=str, help='path to model weights')
        parser.add_argument('-i', '--input_file', required=False, type=str)
        parser.add_argument('--fixed_duration', dest='fixed_duration', action='store_true')
        parser.set_defaults(fixed_duration=False)
        parser.add_argument('-N', '--window_size', default=500000, type=int,
                            help="Size of each event window, in number of events. Ignored if --fixed_duration=True")
        parser.add_argument('-T', '--window_duration', default=33.33, type=float,
                            help="Duration of each event window, in milliseconds. Ignored if --fixed_duration=False")
        parser.add_argument('--num_events_per_pixel', default=0.35, type=float,
                            help='in case N (window size) is not specified, it will be automatically computed as N = width * height * num_events_per_pixel')
        parser.add_argument('--skipevents', default=0, type=int)
        parser.add_argument('--suboffset', default=0, type=int)
        parser.add_argument('--compute_voxel_grid_on_cpu', dest='compute_voxel_grid_on_cpu',
                            action='store_true')
        parser.set_defaults(compute_voxel_grid_on_cpu=False)
    
        set_inference_options(parser)
    
        args = parser.parse_args()
    
        # Load model
        device = get_device(args.use_gpu)
        model = load_model(args.path_to_model)
    
        model = model.to(device)
        model.eval()
    
        # pdb.set_trace()
        height, width = 260, 346
        reconstructor = ImageReconstructor(model, height, width, model.num_bins, args)
        start_s = 0
        """ Read chunks of events using Pandas """
        for frame_no in range(0, frame_num-1):
            # Loop through the events and reconstruct images
            N = frame_timestamp[frame_no][1] - frame_timestamp[frame_no][0]
    
            # if not args.fixed_duration:
            #     if N is None:
            #         N = int(width * height * args.num_events_per_pixel)
            #         print(
            #             'Will use {} events per tensor (automatically estimated with num_events_per_pixel={:0.2f}).'.format(
            #                 N, args.num_events_per_pixel))
            #     else:
            #         print('Will use {} events per tensor (user-specified)'.format(N))
            #         mean_num_events_per_pixel = float(N) / float(width * height)
            #         if mean_num_events_per_pixel < 0.1:
            #             print('!!Warning!! the number of events used ({}) seems to be low compared to the sensor size. \
            #                 The reconstruction results might be suboptimal.'.format(N))
            #         elif mean_num_events_per_pixel > 1.5:
            #             print('!!Warning!! the number of events used ({}) seems to be high compared to the sensor size. \
            #                 The reconstruction results might be suboptimal.'.format(N))
    
            initial_offset = args.skipevents
            sub_offset = args.suboffset
            start_index = initial_offset + sub_offset
    
            if args.compute_voxel_grid_on_cpu:
                print('Will compute voxel grid on CPU.')
    
            with Timer('Processing entire dataset'):
                # event_window = all_events[:N, :]
                # event_window = all_events[start_s: start_s+N, :]
                # last_timestamp = event_window[-1, 0]
                # start_s += N
    
                # start_idx = np.where(all_events[:, 0] >= frame_timestamp[frame_no][0])[0][0]
                # end_idx = np.where(all_events[:, 0] >= frame_timestamp[frame_no][1])[0][0]
                # event_window = all_events[start_idx:end_idx]
                # last_timestamp = event_window[-1, 0]
                #################################
                start_idx_array = np.where(all_events[:, 0] >= frame_timestamp[frame_no][0])[0]
                end_idx_array = np.where(all_events[:, 0] < frame_timestamp[frame_no][1])[0]  # 注意对终止条件的修改
    
                if start_idx_array.size > 0 and end_idx_array.size > 0:
                    start_idx = start_idx_array[0]
                    end_idx = end_idx_array[-1]  # 使用 [-1] 是为了获取这个条件下最后一个满足条件的事件的索引
    
                    # 确保 start_idx 小于 end_idx,这样提取出的窗口才有意义
                    if start_idx < end_idx:
                        event_window = all_events[start_idx:end_idx]
                        if event_window.size > 0:
                            last_timestamp = event_window[-1, 0]
                            # 进行后续处理…
                        else:
                            print(f'No events found between indices {start_idx} and {end_idx}')
                            continue
                    else:
                        print(f'Start index {start_idx} is not less than end index {end_idx}')
                        continue
                else:
                    print(f'No valid start or end indices for frame {frame_no}, skipping…')
                    continue
    
                ##################################
    
                with Timer('Building event tensor'):
                    if args.compute_voxel_grid_on_cpu:
                        # pdb.set_trace()
                        event_tensor = events_to_voxel_grid(event_window, num_bins=model.num_bins, width=width,
                                                            height=height)
                        event_tensor = torch.from_numpy(event_tensor)
                    else:
                        # pdb.set_trace()
                        event_tensor = events_to_voxel_grid_pytorch(event_window, num_bins=model.num_bins,
                                                                    width=width, height=height, device=device)
    
                num_events_in_window = event_window.shape[0]
                reconstructor.update_reconstruction(event_tensor, start_index + num_events_in_window,
                                                    last_timestamp, frame_no, save_img_path)
                # reconstructor.update_reconstruction(event_tensor, start_index + num_events_in_window, last_timestamp)
                start_index += num_events_in_window运行中,生成的recon目录下,很多空文件。
    
  2. Assistant

    在处理生成的重建目录时,发现有很多空文件,这可能是由于以下几个原因导致的:

    1. 事件窗口为空:在提取事件窗口时,如果没有符合条件的事件,或者提取出的事件窗口为空,则不会有数据传给 reconstructor.update_reconstruction 函数,从而可能导致空文件生成。
    2. 重建失败:在调用 update_reconstruction 函数时,如果重建的过程失败,或者函数内部未能正确处理输入的事件数据,则同样可能导致空文件生成。
    3. 文件写入问题:在保存重建图像时,若过程中出现文件写入错误,也可能导致空文件生成。

    为了解决这些问题,可以从以下几个方面入手:

    1. 确认事件窗口的提取过程

    确保提取出的事件窗口不是空的。如果为空,可以直接跳过该帧的处理。

    event_window_size = event_window.size
    if event_window_size > 0:
        reconstructor.update_reconstruction(event_tensor, start_index + event_window.shape[0], last_timestamp, frame_no, save_img_path)
    else:
        print(f'Event window for frame {frame_no} is empty, skipping...')
        continue
    

    2. 检查和调试 update_reconstruction 函数

    确保在 update_reconstruction 函数中,输入的数据被正确处理。如果其中出现错误,则需要在函数内部添加更多的检测和日志输出,以便识别并解决问题。

    3. 验证文件写入过程

    确保在保存文件时没有出错,可以通过在写入文件前检查内容的有效性:

    # Example inside update_reconstruction function
    output_image = self.reconstruct_image()  # replace with actual reconstruction function
    if output_image is not None and output_image.size > 0:
        output_path = os.path.join(save_img_path, f'{frame_no}.png')
        Image.fromarray(output_image).save(output_path)
    else:
        print(f'No valid output for frame {frame_no}, skipping file save...')
    

    修改后的代码段

    在主循环中,添加对 event_window 的大小检查,并确保在 update_reconstruction 调用前确认输入有效:

    if __name__ == "__main__":
        # ...[省略部分代码]...
    
        for frame_no in range(0, frame_num-1):
            # ...[省略部分代码]...
    
            start_idx_array = np.where(all_events[:, 0] >= frame_timestamp[frame_no][0])[0]
            end_idx_array = np.where(all_events[:, 0] < frame_timestamp[frame_no][1])[0]
    
            if start_idx_array.size > 0 and end_idx_array.size > 0:
                start_idx = start_idx_array[0]
                end_idx = end_idx_array[-1]
    
                if start_idx < end_idx:
                    event_window = all_events[start_idx:end_idx]
                    if event_window.size > 0:
                        last_timestamp = event_window[-1, 0]
                    else:
                        print(f'No events found between indices {start_idx} and {end_idx}')
                        continue
                else:
                    print(f'Start index {start_idx} is not less than end index {end_idx}')
                    continue
            else:
                print(f'No valid start or end indices for frame {frame_no}, skipping...')
                continue
    
            with Timer('Building event tensor'):
                if args.compute_voxel_grid_on_cpu:
                    event_tensor = events_to_voxel_grid(event_window, num_bins=model.num_bins, width=width, height=height)
                    event_tensor = torch.from_numpy(event_tensor)
                else:
                    event_tensor = events_to_voxel_grid_pytorch(event_window, num_bins=model.num_bins, width=width, height=height, device=device)
    
            if event_window.shape[0] > 0:
                num_events_in_window = event_window.shape[0]
                reconstructor.update_reconstruction(event_tensor, start_index + num_events_in_window, last_timestamp, frame_no, save_img_path)
                start_index += num_events_in_window
            else:
                print(f'Event window is empty for frame {frame_no}, skipping update_reconstruction...')
                continue
    

    以上改动确保只有在事件窗口有实际内容时,才会调用 update_reconstruction 函数。另外需要在 update_reconstruction 函数中仔细检查输入数据,并确保数据处理和文件保存过程的正常进行。

  3. User

    但是缺少文件,那么在测评的时候就会缺少结果

  4. Assistant

    为了防止生成空文件却导致测评时数据缺失的问题,我们需要确保每个帧都能生成一个有效的重建输出。为此,我们可以在主处理循环中减少添加更严格的检查,并在缺少事件数据或生成失败时生成一个占位图像(如全黑或全白图像)以确保文件存在。

    修改后在没有有效数据时生成占位图的代码段

    在主循环中添加对于空事件窗口的处理,并生成占位图以确保每个帧都有输出文件:

    import numpy as np
    
    def create_placeholder_image(height, width):
        # 创建一个占位图像,此处为全黑图像
        return np.zeros((height, width), dtype=np.uint8)
    
    if __name__ == "__main__":
        # ...[省略部分代码]...
    
        for frame_no in range(0, frame_num-1):
            # ...[省略部分代码]...
    
            start_idx_array = np.where(all_events[:, 0] >= frame_timestamp[frame_no][0])[0]
            end_idx_array = np.where(all_events[:, 0] < frame_timestamp[frame_no][1])[0]
    
            if start_idx_array.size > 0 and end_idx_array.size > 0:
                start_idx = start_idx_array[0]
                end_idx = end_idx_array[-1]
    
                if start_idx < end_idx:
                    event_window = all_events[start_idx:end_idx]
                    if event_window.size > 0:
                        last_timestamp = event_window[-1, 0]
                    else:
                        print(f'No events found between indices {start_idx} and {end_idx}')
                        event_window = None  # 标记事件窗口为空
                else:
                    print(f'Start index {start_idx} is not less than end index {end_idx}')
                    event_window = None  # 标记事件窗口为空
            else:
                print(f'No valid start or end indices for frame {frame_no}, skipping...')
                event_window = None  # 标记事件窗口为空
    
            if event_window is not None and event_window.size > 0:
                with Timer('Building event tensor'):
                    if args.compute_voxel_grid_on_cpu:
                        event_tensor = events_to_voxel_grid(event_window, num_bins=model.num_bins, width=width, height=height)
                        event_tensor = torch.from_numpy(event_tensor)
                    else:
                        event_tensor = events_to_voxel_grid_pytorch(event_window, num_bins=model.num_bins, width=width, height=height, device=device)
    
                num_events_in_window = event_window.shape[0]
                reconstructor.update_reconstruction(event_tensor, start_index + num_events_in_window, last_timestamp, frame_no, save_img_path)
                start_index += num_events_in_window
            else:
                # 生成占位图像并保存
                placeholder_image = create_placeholder_image(height, width)
                placeholder_image_path = os.path.join(save_img_path, f'{frame_no}.png')
                Image.fromarray(placeholder_image).save(placeholder_image_path)
                print(f'Generated placeholder image for frame {frame_no}')
    

    update_reconstruction 函数内部调整

    如果 update_reconstruction 函数内部有额外的步骤也可能失败或生成空文件,可以添加更多的错误处理逻辑,保证即使在失败的情况下,也能生成占位图像。

    完整示例

    这是针对整个主处理循环添加占位图逻辑,以及在 update_reconstruction 中进行应对的完整示例代码:

    主循环代码:

    if __name__ == "__main__":
        # ...[省略部分代码]...
    
        for frame_no in range(0, frame_num-1):
            try:
                # ...[事件窗口提取省略部分代码]...
    
                if event_window is not None and event_window.size > 0:
                    with Timer('Building event tensor'):
                        if args.compute_voxel_grid_on_cpu:
                            event_tensor = events_to_voxel_grid(event_window, num_bins=model.num_bins, width=width, height=height)
                            event_tensor = torch.from_numpy(event_tensor)
                        else:
                            event_tensor = events_to_voxel_grid_pytorch(event_window, num_bins=model.num_bins, width=width, height=height, device=device)
    
                    num_events_in_window = event_window.shape[0]
                    reconstructor.update_reconstruction(event_tensor, start_index + num_events_in_window, last_timestamp, frame_no, save_img_path)
                    start_index += num_events_in_window
                else:
                    raise ValueError(f'No events found for frame {frame_no}')
            except Exception as e:
                print(f"Exception occurred while processing frame {frame_no}: {e}")
                placeholder_image = create_placeholder_image(height, width)
                placeholder_image_path = os.path.join(save_img_path, f'{frame_no}.png')
                Image.fromarray(placeholder_image).save(placeholder_image_path)
                print(f'Generated placeholder image for frame {frame_no} due to error.')
    

    update_reconstruction 函数内部的调整(如果需要处理可能导致失败的步骤):

    class ImageReconstructor:
        # ... [原有代码保留]
        
        def update_reconstruction(self, event_tensor, start_index, last_timestamp, frame_no, save_img_path):
            try:
                # ... [图像重建和写入代码]
                output_image = self.reconstruct_image()  # replace with actual reconstruction function
                output_path = os.path.join(save_img_path, f'{frame_no}.png')
                if output_image is not None and output_image.size > 0:
                    Image.fromarray(output_image).save(output_path)
                else:
                    raise ValueError(f'Reconstructed image is empty for frame {frame_no}')
            except Exception as e:
                print(f"Failed to reconstruct or save image for frame {frame_no}: {e}")
                placeholder_image = create_placeholder_image(self.height, self.width)
                Image.fromarray(placeholder_image).save(output_path)
    

    这样,通过以上调整,即便在提取事件或图像重建过程中发生问题也不会生成空文件,而是确保所有帧都有一个有效(即使是占位)的输出文件,从而在后续测评过程中不会缺失数据。

  5. User

    在import torch from torch.utils.data.distributed import DistributedSampler

    datasets related

    from lib.train.dataset import Coesot, Fe108, VisEvent from lib.train.data import sampler, opencv_loader, processing, LTRLoader import lib.train.data.transforms as tfm from lib.utils.misc import is_main_process

    def update_settings(settings, cfg): settings.print_interval = cfg.TRAIN.PRINT_INTERVAL settings.search_area_factor = {'template': cfg.DATA.TEMPLATE.FACTOR, 'search': cfg.DATA.SEARCH.FACTOR} settings.output_sz = {'template': cfg.DATA.TEMPLATE.SIZE, 'search': cfg.DATA.SEARCH.SIZE} settings.center_jitter_factor = {'template': cfg.DATA.TEMPLATE.CENTER_JITTER, 'search': cfg.DATA.SEARCH.CENTER_JITTER} settings.scale_jitter_factor = {'template': cfg.DATA.TEMPLATE.SCALE_JITTER, 'search': cfg.DATA.SEARCH.SCALE_JITTER} settings.grad_clip_norm = cfg.TRAIN.GRAD_CLIP_NORM settings.print_stats = None settings.batchsize = cfg.TRAIN.BATCH_SIZE settings.scheduler_type = cfg.TRAIN.SCHEDULER.TYPE

    def names2datasets(name_list: list, settings, image_loader): assert isinstance(name_list, list) datasets = [] for name in name_list: assert name in ["LASOT", "GOT10K_vottrain", "GOT10K_votval", "GOT10K_train_full", "GOT10K_official_val", "COCO17", "VID", "TRACKINGNET", "COESOT", "COESOT_VAL", "FE108", "FE108_VAL", "VisEvent", "VisEvent_VAL"] if name == "LASOT": if settings.use_lmdb: print("Building lasot dataset from lmdb") datasets.append(Lasot_lmdb(settings.env.lasot_lmdb_dir, split='train', image_loader=image_loader)) else: datasets.append(Lasot(settings.env.lasot_dir, split='train', image_loader=image_loader)) if name == "GOT10K_vottrain": if settings.use_lmdb: print("Building got10k from lmdb") datasets.append(Got10k_lmdb(settings.env.got10k_lmdb_dir, split='vottrain', image_loader=image_loader)) else: datasets.append(Got10k(settings.env.got10k_dir, split='vottrain', image_loader=image_loader)) if name == "GOT10K_train_full": if settings.use_lmdb: print("Building got10k_train_full from lmdb") datasets.append(Got10k_lmdb(settings.env.got10k_lmdb_dir, split='train_full', image_loader=image_loader)) else: datasets.append(Got10k(settings.env.got10k_dir, split='train_full', image_loader=image_loader)) if name == "GOT10K_votval": if settings.use_lmdb: print("Building got10k from lmdb") datasets.append(Got10k_lmdb(settings.env.got10k_lmdb_dir, split='votval', image_loader=image_loader)) else: datasets.append(Got10k(settings.env.got10k_dir, split='votval', image_loader=image_loader)) if name == "GOT10K_official_val": if settings.use_lmdb: raise ValueError("Not implement") else: datasets.append(Got10k(settings.env.got10k_val_dir, split=None, image_loader=image_loader)) if name == "COCO17": if settings.use_lmdb: print("Building COCO2017 from lmdb") datasets.append(MSCOCOSeq_lmdb(settings.env.coco_lmdb_dir, version="2017", image_loader=image_loader)) else: datasets.append(MSCOCOSeq(settings.env.coco_dir, version="2017", image_loader=image_loader)) if name == "VID": if settings.use_lmdb: print("Building VID from lmdb") datasets.append(ImagenetVID_lmdb(settings.env.imagenet_lmdb_dir, image_loader=image_loader)) else: datasets.append(ImagenetVID(settings.env.imagenet_dir, image_loader=image_loader)) if name == "TRACKINGNET": if settings.use_lmdb: print("Building TrackingNet from lmdb") datasets.append(TrackingNet_lmdb(settings.env.trackingnet_lmdb_dir, image_loader=image_loader)) else: # raise ValueError("NOW WE CAN ONLY USE TRACKINGNET FROM LMDB") datasets.append(TrackingNet(settings.env.trackingnet_dir, image_loader=image_loader)) if name == "COESOT": datasets.append(Coesot(settings.env.coesot_dir, split='train', image_loader=image_loader)) if name == "COESOT_VAL": datasets.append(Coesot(settings.env.coesot_val_dir, split='val', image_loader=image_loader)) if name == "FE108": datasets.append(Fe108(settings.env.fe108_dir, split='train', image_loader=image_loader)) if name == "FE108_VAL": datasets.append(Fe108(settings.env.fe108_val_dir, split='val', image_loader=image_loader)) if name == "VisEvent": datasets.append(VisEvent(settings.env.visevent_dir, split='train', image_loader=image_loader)) if name == "VisEvent_VAL": datasets.append(VisEvent(settings.env.visevent_val_dir, split='val', image_loader=image_loader))

    return datasets
    

    def build_dataloaders(cfg, settings): # Data transform # transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05), tfm.RandomHorizontalFlip(0.5)) transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05)) # for FE108 p=0.5 else 0.05

    transform_train = tfm.Transform(tfm.ToTensor(),     # tfm.RandomHorizontalFlip_Norm(0.5),  # FE108, not normalize.
                                   tfm.Normalize(mean=cfg.DATA.MEAN, std=cfg.DATA.STD)
                                    )
    transform_val = tfm.Transform(tfm.ToTensor(),
                                  tfm.Normalize(mean=cfg.DATA.MEAN, std=cfg.DATA.STD)
                                  )
    
    # The tracking pairs processing module
    output_sz = settings.output_sz
    search_area_factor = settings.search_area_factor
    
    data_processing_train = processing.STARKProcessing(search_area_factor=search_area_factor,
                                                       output_sz=output_sz,
                                                       center_jitter_factor=settings.center_jitter_factor,
                                                       scale_jitter_factor=settings.scale_jitter_factor,
                                                       mode='sequence',
                                                       transform=transform_train,
                                                       joint_transform=transform_joint,
                                                       settings=settings)
    
    data_processing_val = processing.STARKProcessing(search_area_factor=search_area_factor,
                                                     output_sz=output_sz,
                                                     center_jitter_factor=settings.center_jitter_factor,
                                                     scale_jitter_factor=settings.scale_jitter_factor,
                                                     mode='sequence',
                                                     transform=transform_val,
                                                     joint_transform=transform_joint,
                                                     settings=settings)
    
    # Train sampler and loader
    settings.num_template = getattr(cfg.DATA.TEMPLATE, "NUMBER", 1)
    settings.num_search = getattr(cfg.DATA.SEARCH, "NUMBER", 1)
    sampler_mode = getattr(cfg.DATA, "SAMPLER_MODE", "causal")
    train_cls = getattr(cfg.TRAIN, "TRAIN_CLS", False)
    print("sampler_mode", sampler_mode)
    dataset_train = sampler.TrackingSampler(datasets=names2datasets(cfg.DATA.TRAIN.DATASETS_NAME, settings, opencv_loader),
                                            p_datasets=cfg.DATA.TRAIN.DATASETS_RATIO,
                                            samples_per_epoch=cfg.DATA.TRAIN.SAMPLE_PER_EPOCH,
                                            max_gap=cfg.DATA.MAX_SAMPLE_INTERVAL, num_search_frames=settings.num_search,
                                            num_template_frames=settings.num_template, processing=data_processing_train,
                                            frame_sample_mode=sampler_mode, train_cls=train_cls)
    
    train_sampler = DistributedSampler(dataset_train) if settings.local_rank != -1 else None
    shuffle = False if settings.local_rank != -1 else True
    
    loader_train = LTRLoader('train', dataset_train, training=True, batch_size=cfg.TRAIN.BATCH_SIZE, shuffle=shuffle,
                             num_workers=cfg.TRAIN.NUM_WORKER, drop_last=True, stack_dim=1, sampler=train_sampler)
    
    # Validation samplers and loaders
    dataset_val = sampler.TrackingSampler(datasets=names2datasets(cfg.DATA.VAL.DATASETS_NAME, settings, opencv_loader),
                                          p_datasets=cfg.DATA.VAL.DATASETS_RATIO,
                                          samples_per_epoch=cfg.DATA.VAL.SAMPLE_PER_EPOCH,
                                          max_gap=cfg.DATA.MAX_SAMPLE_INTERVAL, num_search_frames=settings.num_search,
                                          num_template_frames=settings.num_template, processing=data_processing_val,
                                          frame_sample_mode=sampler_mode, train_cls=train_cls)
    val_sampler = DistributedSampler(dataset_val) if settings.local_rank != -1 else None
    loader_val = LTRLoader('val', dataset_val, training=False, batch_size=cfg.TRAIN.BATCH_SIZE,
                           num_workers=cfg.TRAIN.NUM_WORKER, drop_last=True, stack_dim=1, sampler=val_sampler,
                           epoch_interval=cfg.TRAIN.VAL_EPOCH_INTERVAL)
    
    return loader_train, loader_val
    

    def get_optimizer_scheduler(net, cfg): train_cls = getattr(cfg.TRAIN, "TRAIN_CLS", False) if train_cls: print("Only training classification head. Learnable parameters are shown below.") param_dicts = [ {"params": [p for n, p in net.named_parameters() if "cls" in n and p.requires_grad]} ]

        for n, p in net.named_parameters():
            if "cls" not in n:
                p.requires_grad = False
            else:
                print(n)
    else:
        param_dicts = [
            {"params": [p for n, p in net.named_parameters() if "backbone" not in n and p.requires_grad]},
            {
                "params": [p for n, p in net.named_parameters() if "backbone" in n and p.requires_grad],
                "lr": cfg.TRAIN.LR * cfg.TRAIN.BACKBONE_MULTIPLIER,
            },
        ]
        if is_main_process():
            print("Learnable parameters are shown below.")
            for n, p in net.named_parameters():
                if p.requires_grad:
                    print(n)
    
    if cfg.TRAIN.OPTIMIZER == "ADAMW":
        optimizer = torch.optim.AdamW(param_dicts, lr=cfg.TRAIN.LR,
                                      weight_decay=cfg.TRAIN.WEIGHT_DECAY)
    else:
        raise ValueError("Unsupported Optimizer")
    if cfg.TRAIN.SCHEDULER.TYPE == 'step':
        lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, cfg.TRAIN.LR_DROP_EPOCH)
    elif cfg.TRAIN.SCHEDULER.TYPE == "Mstep":
        lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
                                                            milestones=cfg.TRAIN.SCHEDULER.MILESTONES,
                                                            gamma=cfg.TRAIN.SCHEDULER.GAMMA)
    else:
        raise ValueError("Unsupported scheduler")
    return optimizer, lr_scheduler中    # transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05), tfm.RandomHorizontalFlip(0.5))
    transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05))           # for FE108 p=0.5 else 0.05
    
    transform_train = tfm.Transform(tfm.ToTensor(),     # tfm.RandomHorizontalFlip_Norm(0.5),  # FE108, not normalize.的注释中,对FE108的备注是什么意思?
    
  6. Assistant

    在注释中提到“for FE108 p=0.5 else 0.05”和“FE108, not normalize.”,这是指对数据集 FE108 的特定数据预处理和增强操作的设置。这些注释的目的是提醒开发者,在处理 FE108 数据集时需要特别注意的数据变换设置。

    注释解释:

    1. transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05))

    注释:# for FE108 p=0.5 else 0.05

    这条注释的意思是,对于 FE108 数据集,ToGrayscale 转换的概率应设置为 0.5,而对于其他数据集,概率应设置为 0.05。当前的代码设置将 ToGrayscale 转换的概率设置为 0.05。

    要符合注释所提到的不同数据集的概率要求,可以根据数据集的名称进行条件分支调整,例如:

    if 'FE108' in cfg.DATA.TRAIN.DATASETS_NAME or 'FE108_VAL' in cfg.DATA.VAL.DATASETS_NAME:
        transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.5))
    else:
        transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05))
    

    2. transform_train = tfm.Transform(tfm.ToTensor())

    注释:# FE108, not normalize.

    这里的注释表明,对于 FE108 数据集,在 train 模式下不进行归一化(Normalizing)操作。然而,为了满足其他数据集的需求,通常还是需要进行归一化操作。

    要根据数据集的需求进行条件处理,可以像这样编码:

    if 'FE108' in cfg.DATA.TRAIN.DATASETS_NAME or 'FE108_VAL' in cfg.DATA.VAL.DATASETS_NAME:
        transform_train = tfm.Transform(tfm.ToTensor())
    else:
        transform_train = tfm.Transform(
            tfm.ToTensor(),     
            tfm.Normalize(mean=cfg.DATA.MEAN, std=cfg.DATA.STD)
        )
    

    完整调整代码示例:

    下面是根据上述注释提供的建议,对代码进行调整后的示例:

    import torch
    from torch.utils.data.distributed import DistributedSampler
    # datasets related
    from lib.train.dataset import Coesot, Fe108, VisEvent
    from lib.train.data import sampler, opencv_loader, processing, LTRLoader
    import lib.train.data.transforms as tfm
    from lib.utils.misc import is_main_process
    
    
    def update_settings(settings, cfg):
        settings.print_interval = cfg.TRAIN.PRINT_INTERVAL
        settings.search_area_factor = {'template': cfg.DATA.TEMPLATE.FACTOR,
                                       'search': cfg.DATA.SEARCH.FACTOR}
        settings.output_sz = {'template': cfg.DATA.TEMPLATE.SIZE,
                              'search': cfg.DATA.SEARCH.SIZE}
        settings.center_jitter_factor = {'template': cfg.DATA.TEMPLATE.CENTER_JITTER,
                                         'search': cfg.DATA.SEARCH.CENTER_JITTER}
        settings.scale_jitter_factor = {'template': cfg.DATA.TEMPLATE.SCALE_JITTER,
                                        'search': cfg.DATA.SEARCH.SCALE_JITTER}
        settings.grad_clip_norm = cfg.TRAIN.GRAD_CLIP_NORM
        settings.print_stats = None
        settings.batchsize = cfg.TRAIN.BATCH_SIZE
        settings.scheduler_type = cfg.TRAIN.SCHEDULER.TYPE
    
    
    def names2datasets(name_list: list, settings, image_loader):
        assert isinstance(name_list, list)
        datasets = []
        for name in name_list:
            assert name in ["LASOT", "GOT10K_vottrain", "GOT10K_votval", "GOT10K_train_full", "GOT10K_official_val",
                            "COCO17", "VID", "TRACKINGNET", "COESOT", "COESOT_VAL", "FE108", "FE108_VAL", "VisEvent", "VisEvent_VAL"]
            if name == "LASOT":
                if settings.use_lmdb:
                    print("Building lasot dataset from lmdb")
                    datasets.append(Lasot_lmdb(settings.env.lasot_lmdb_dir, split='train', image_loader=image_loader))
                else:
                    datasets.append(Lasot(settings.env.lasot_dir, split='train', image_loader=image_loader))
            if name == "GOT10K_vottrain":
                if settings.use_lmdb:
                    print("Building got10k from lmdb")
                    datasets.append(Got10k_lmdb(settings.env.got10k_lmdb_dir, split='vottrain', image_loader=image_loader))
                else:
                    datasets.append(Got10k(settings.env.got10k_dir, split='vottrain', image_loader=image_loader))
            if name == "GOT10K_train_full":
                if settings.use_lmdb:
                    print("Building got10k_train_full from lmdb")
                    datasets.append(Got10k_lmdb(settings.env.got10k_lmdb_dir, split='train_full', image_loader=image_loader))
                else:
                    datasets.append(Got10k(settings.env.got10k_dir, split='train_full', image_loader=image_loader))
            if name == "GOT10K_votval":
                if settings.use_lmdb:
                    print("Building got10k from lmdb")
                    datasets.append(Got10k_lmdb(settings.env.got10k_lmdb_dir, split='votval', image_loader=image_loader))
                else:
                    datasets.append(Got10k(settings.env.got10k_dir, split='votval', image_loader=image_loader))
            if name == "GOT10K_official_val":
                if settings.use_lmdb:
                    raise ValueError("Not implement")
                else:
                    datasets.append(Got10k(settings.env.got10k_val_dir, split=None, image_loader=image_loader))
            if name == "COCO17":
                if settings.use_lmdb:
                    print("Building COCO2017 from lmdb")
                    datasets.append(MSCOCOSeq_lmdb(settings.env.coco_lmdb_dir, version="2017", image_loader=image_loader))
                else:
                    datasets.append(MSCOCOSeq(settings.env.coco_dir, version="2017", image_loader=image_loader))
            if name == "VID":
                if settings.use_lmdb:
                    print("Building VID from lmdb")
                    datasets.append(ImagenetVID_lmdb(settings.env.imagenet_lmdb_dir, image_loader=image_loader))
                else:
                    datasets.append(ImagenetVID(settings.env.imagenet_dir, image_loader=image_loader))
            if name == "TRACKINGNET":
                if settings.use_lmdb:
                    print("Building TrackingNet from lmdb")
                    datasets.append(TrackingNet_lmdb(settings.env.trackingnet_lmdb_dir, image_loader=image_loader))
                else:
                    # raise ValueError("NOW WE CAN ONLY USE TRACKINGNET FROM LMDB")
                    datasets.append(TrackingNet(settings.env.trackingnet_dir, image_loader=image_loader))
            if name == "COESOT":
                datasets.append(Coesot(settings.env.coesot_dir, split='train', image_loader=image_loader))
            if name == "COESOT_VAL":
                datasets.append(Coesot(settings.env.coesot_val_dir, split='val', image_loader=image_loader))
            if name == "FE108":
                datasets.append(Fe108(settings.env.fe108_dir, split='train', image_loader=image_loader))
            if name == "FE108_VAL":
                datasets.append(Fe108(settings.env.fe108_val_dir, split='val', image_loader=image_loader))
            if name == "VisEvent":
                datasets.append(VisEvent(settings.env.visevent_dir, split='train', image_loader=image_loader))
            if name == "VisEvent_VAL":
                datasets.append(VisEvent(settings.env.visevent_val_dir, split='val', image_loader=image_loader))
    
        return datasets
    
    
    def build_dataloaders(cfg, settings):
        # Data transform
        if 'FE108' in cfg.DATA.TRAIN.DATASETS_NAME or 'FE108_VAL' in cfg.DATA.VAL.DATASETS_NAME:
            transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.5))
            transform_train = tfm.Transform(tfm.ToTensor())
        else:
            transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05))
            transform_train = tfm.Transform(
                tfm.ToTensor(),
                tfm.Normalize(mean=cfg.DATA.MEAN, std=cfg.DATA.STD)
            )
        
        transform_val = tfm.Transform(
            tfm.ToTensor(),
            tfm.Normalize(mean=cfg.DATA.MEAN, std=cfg.DATA.STD)
        )
    
        # The tracking pairs processing module
        output_sz = settings.output_sz
        search_area_factor = settings.search_area_factor
    
        data_processing_train = processing.STARKProcessing(search_area_factor=search_area_factor,
                                                           output_sz=output_sz,
                                                           center_jitter_factor=settings.center_jitter_factor,
                                                           scale_jitter_factor=settings.scale_jitter_factor,
                                                           mode='sequence',
                                                           transform=transform_train,
                                                           joint_transform=transform_joint,
                                                           settings=settings)
    
        data_processing_val = processing.STARKProcessing(search_area_factor=search_area_factor,
                                                         output_sz=output_sz,
                                                         center_jitter_factor=settings.center_jitter_factor,
                                                         scale_jitter_factor=settings.scale_jitter_factor,
                                                         mode='sequence',
                                                         transform=transform_val,
                                                         joint_transform=transform_joint,
                                                         settings=settings)
    
        # Train sampler and loader
        settings.num_template = getattr(cfg.DATA.TEMPLATE, "NUMBER", 1)
        settings.num_search = getattr(cfg.DATA.SEARCH, "NUMBER", 1)
        sampler_mode = getattr(cfg.DATA, "SAMPLER_MODE", "causal")
        train_cls = getattr(cfg.TRAIN, "TRAIN_CLS", False)
        print("sampler_mode", sampler_mode)
        dataset_train = sampler.TrackingSampler(datasets=names2datasets(cfg.DATA.TRAIN.DATASETS_NAME, settings, opencv_loader),
                                                p_datasets=cfg.DATA.TRAIN.DATASETS_RATIO,
                                                samples_per_epoch=cfg.DATA.TRAIN.SAMPLE_PER_EPOCH,
                                                max_gap=cfg.DATA.MAX_SAMPLE_INTERVAL, num_search_frames=settings.num_search,
                                                num_template_frames=settings.num_template, processing=data_processing_train,
                                                frame_sample_mode=sampler_mode, train_cls=train_cls)
    
        train_sampler = DistributedSampler(dataset_train) if settings.local_rank != -1 else None
        shuffle = False if settings.local_rank != -1 else True
    
        loader_train = LTRLoader('train', dataset_train, training=True, batch_size=cfg.TRAIN.BATCH_SIZE, shuffle=shuffle,
                                 num_workers=cfg.TRAIN.NUM_WORKER, drop_last=True, stack_dim=1, sampler=train_sampler)
    
        # Validation samplers and loaders
        dataset_val = sampler.TrackingSampler(datasets=names2datasets(cfg.DATA.VAL.DATASETS_NAME, settings, opencv_loader),
                                              p_datasets=cfg.DATA.VAL.DATASETS_RATIO,
                                              samples_per_epoch=cfg.DATA.VAL.SAMPLE_PER_EPOCH,
                                              max_gap=cfg.DATA.MAX_SAMPLE_INTERVAL, num_search_frames=settings.num_search,
                                              num_template_frames=settings.num_template, processing=data_processing_val,
                                              frame_sample_mode=sampler_mode, train_cls=train_cls)
        val_sampler = DistributedSampler(dataset_val) if settings.local_rank != -1 else None
        loader_val = LTRLoader('val', dataset_val, training=False, batch_size=cfg.TRAIN.BATCH_SIZE,
                               num_workers=cfg.TRAIN.NUM_WORKER, drop_last=True, stack_dim=1, sampler=val_sampler,
                               epoch_interval=cfg.TRAIN.VAL_EPOCH_INTERVAL)
    
        return loader_train, loader_val
    
    
    def get_optimizer_scheduler(net, cfg):
        train_cls = getattr(cfg.TRAIN, "TRAIN_CLS", False)
        if train_cls:
            print("Only training classification head. Learnable parameters are shown below.")
            param_dicts = [
                {"params": [p for n, p in net.named_parameters() if "cls" in n and p.requires_grad]}
            ]
    
            for n, p in net.named_parameters():
                if "cls" not in n:
                    p.requires_grad = False
                else:
                    print(n)
        else:
            param_dicts = [
                {"params": [p for n, p in net.named_parameters() if "backbone" not in n and p.requires_grad]},
                {
                    "params": [p for n, p in net.named_parameters() if "backbone" in n and p.requires_grad],
                    "lr": cfg.TRAIN.LR * cfg.TRAIN.BACKBONE_MULTIPLIER,
                },
            ]
            if is_main_process():
                print("Learnable parameters are shown below.")
                for n, p in net.named_parameters():
                    if p.requires_grad:
                        print(n)
    
        if cfg.TRAIN.OPTIMIZER == "ADAMW":
            optimizer = torch.optim.AdamW(param_dicts, lr=cfg.TRAIN.LR,
                                          weight_decay=cfg.TRAIN.WEIGHT_DECAY)
        else:
            raise ValueError("Unsupported Optimizer")
        if cfg.TRAIN.SCHEDULER.TYPE == 'step':
            lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, cfg.TRAIN.LR_DROP_EPOCH)
        elif cfg.TRAIN.SCHEDULER.TYPE == "Mstep":
            lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
                                                                milestones=cfg.TRAIN.SCHEDULER.MILESTONES,
                                                                gamma=cfg.TRAIN.SCHEDULER.GAMMA)
        else:
            raise ValueError("Unsupported scheduler")
        return optimizer, lr_scheduler
    

    通过这些调整,你可以确保在处理不同数据集时应用正确的图像变换和数据处理过程。注释中的信息被转换为实际的代码逻辑,在处理特定数据集时改变数据变换的行为。