在# --coding:utf-8-- import torch from utils.loadingutils import loadmodel, getdevice…
- User
在# --coding:utf-8-- import torch from utils.loading_utils import load_model, get_device import numpy as np import argparse import pandas as pd import os import cv2 import pdb
the following functions are borrowed from: https://github.com/uzh-rpg/rpg_e2vid
from utils.event_readers import FixedSizeEventReader, FixedDurationEventReader from utils.inference_utils import events_to_voxel_grid, events_to_voxel_grid_pytorch from utils.timers import Timer
import time from image_reconstructor import ImageReconstructor from options.inference_options import set_inference_options from PIL import Image from dv import AedatFile
def extract_davis(aedat_file_path, dvs_img_interval): frame_no = 0 frame_all = [] frame_exposure_time = [] frame_interval_time = [] # use_mode = 'frame_exposure_time' #### default mode use_mode = 'frame_interval_time' with AedatFile(aedat_file_path) as f: # list all the names of streams in the file print(f.names) # extract timestamps of each frame for frame in f['frames']: frame_all.append(frame.image) frame_exposure_time.append([frame.timestamp_start_of_exposure, frame.timestamp_end_of_exposure]) ## [1607928583397102, 1607928583401102] frame_interval_time.append( [frame.timestamp_start_of_frame, frame.timestamp_end_of_frame]) ## [1607928583387944, 1607928583410285]
# pdb.set_trace() if use_mode == 'frame_exposure_time': frame_timestamp = frame_exposure_time elif use_mode == 'frame_interval_time': frame_timestamp = frame_interval_time frame_num = len(frame_timestamp) # Access dimensions of the event stream height, width = f['events'].size # 260 346 event_frame = 255 * np.ones((height, width, 3), dtype=np.uint8) # loop through the "events" stream # events = np.hstack([packet for packet in f['events'].numpy()]) # # Access information of all events by type # timestamps, x, y, polarities = events['timestamp'], events['x'], events['y'], events['polarity'] # pdb.set_trace() # save event_img idx = np.round(np.linspace(0, len(frame_timestamp) - 1, int(frame_num / dvs_img_interval))).astype( int) ## frame index [0, 1, 2, ... , 3847] frame_timestamp = np.array(frame_timestamp)[idx] ## (3848, 2) # another reader # speed up # events will be a named numpy array events = np.hstack([packet for packet in f['events'].numpy()]) timestamps, x, y, polarities = events['timestamp'], events['x'], events['y'], events['polarity'] return timestamps, x, y, polarities, frame_num, frame_timestamp #######################################################################################################Main Function
####################################################################################################### if name == "main": device = torch.device("cuda:0") # data_path = r"/home/ioe/xxxxx/EventTracking/COESOT/train" data_path = r"/data2/local_userdata/heyuting/datasets/COESOT/test" save_path = r"/data2/local_userdata/heyuting/datasets/COESOT/test" video_files = os.listdir(data_path) # 由于数据集中有txt文件,因此需要排除 video_files = [file for file in video_files if file not in ('list.txt', 'val.txt')] for videoID in range(len(video_files)): foldName = video_files[videoID] print("==>> foldName: ", foldName) if foldName in ['dvSave-2022_01_26_20_01_46', 'dvSave-2022_03_21_11_09_50', 'dvSave-2022_03_21_11_12_27', 'dvSave-2022_03_21_11_19_37', 'dvSave-2022_02_25_13_21_41', 'dvSave-2022_02_25_13_31_18', 'dvSave-2022_03_21_09_05_49']: print("==>> pass it: ", foldName) continue # if foldName not in ['dvSave-2021_12_21_16_26_09']: # print("==>> pass it: ", foldName) # continue
# if foldName not in ['dvSave-2022_08_21_13_21_14']: # continue aedat_file_path = os.path.join(data_path, foldName, foldName+'.aedat4') save_img_path = os.path.join(save_path, foldName, foldName+'_recon') if not os.path.exists(save_img_path): os.makedirs(save_img_path) # dvs_img_interval = 1 # for root, dirs, files in os.walk(aedat_path): # for file in files: # (filename, extension) = os.path.splitext(file) # print("==>> filename: ", filename) # if (extension == '.aedat4'): # if filename + '_aps' in os.listdir(root) or filename + '_dvs' in os.listdir(root): # print("==>> Skip this video ... ") # continue # save_path_dvs = os.path.join(root, filename, filename+'_dvs') # save_path_aps = os.path.join(root, filename, filename+'_aps') # aedat_file_path = os.path.join(root, filename + '.aedat4') # filename_txt_path = os.path.join(root, filename+'_timestamp.txt') # croped_events_path = os.path.join(root, filename+'_events.txt') # if not os.path.exists(save_path_dvs): # os.makedirs(save_path_dvs) # if not os.path.exists(save_path_aps): # os.makedirs(save_path_aps) timestamps, x, y, polarities, frame_num, frame_timestamp = extract_davis(aedat_file_path, 1) ## (4391964,) # all_events = [timestamps, x, y, polarities] all_events = np.zeros((timestamps.shape[0], 4)) all_events[:, 0] = timestamps all_events[:, 1] = x all_events[:, 2] = y all_events[:, 3] = polarities # pdb.set_trace() parser = argparse.ArgumentParser(description='Evaluating a trained network') parser.add_argument('-c', '--path_to_model', required=True, type=str, help='path to model weights') parser.add_argument('-i', '--input_file', required=False, type=str) parser.add_argument('--fixed_duration', dest='fixed_duration', action='store_true') parser.set_defaults(fixed_duration=False) parser.add_argument('-N', '--window_size', default=500000, type=int, help="Size of each event window, in number of events. Ignored if --fixed_duration=True") parser.add_argument('-T', '--window_duration', default=33.33, type=float, help="Duration of each event window, in milliseconds. Ignored if --fixed_duration=False") parser.add_argument('--num_events_per_pixel', default=0.35, type=float, help='in case N (window size) is not specified, it will be automatically computed as N = width * height * num_events_per_pixel') parser.add_argument('--skipevents', default=0, type=int) parser.add_argument('--suboffset', default=0, type=int) parser.add_argument('--compute_voxel_grid_on_cpu', dest='compute_voxel_grid_on_cpu', action='store_true') parser.set_defaults(compute_voxel_grid_on_cpu=False) set_inference_options(parser) args = parser.parse_args() # Load model device = get_device(args.use_gpu) model = load_model(args.path_to_model) model = model.to(device) model.eval() # pdb.set_trace() height, width = 260, 346 reconstructor = ImageReconstructor(model, height, width, model.num_bins, args) start_s = 0 """ Read chunks of events using Pandas """ for frame_no in range(0, frame_num-1): # Loop through the events and reconstruct images N = frame_timestamp[frame_no][1] - frame_timestamp[frame_no][0] # if not args.fixed_duration: # if N is None: # N = int(width * height * args.num_events_per_pixel) # print( # 'Will use {} events per tensor (automatically estimated with num_events_per_pixel={:0.2f}).'.format( # N, args.num_events_per_pixel)) # else: # print('Will use {} events per tensor (user-specified)'.format(N)) # mean_num_events_per_pixel = float(N) / float(width * height) # if mean_num_events_per_pixel < 0.1: # print('!!Warning!! the number of events used ({}) seems to be low compared to the sensor size. \ # The reconstruction results might be suboptimal.'.format(N)) # elif mean_num_events_per_pixel > 1.5: # print('!!Warning!! the number of events used ({}) seems to be high compared to the sensor size. \ # The reconstruction results might be suboptimal.'.format(N)) initial_offset = args.skipevents sub_offset = args.suboffset start_index = initial_offset + sub_offset if args.compute_voxel_grid_on_cpu: print('Will compute voxel grid on CPU.') with Timer('Processing entire dataset'): # event_window = all_events[:N, :] # event_window = all_events[start_s: start_s+N, :] # last_timestamp = event_window[-1, 0] # start_s += N # start_idx = np.where(all_events[:, 0] >= frame_timestamp[frame_no][0])[0][0] # end_idx = np.where(all_events[:, 0] >= frame_timestamp[frame_no][1])[0][0] # event_window = all_events[start_idx:end_idx] # last_timestamp = event_window[-1, 0] ################################# start_idx_array = np.where(all_events[:, 0] >= frame_timestamp[frame_no][0])[0] end_idx_array = np.where(all_events[:, 0] < frame_timestamp[frame_no][1])[0] # 注意对终止条件的修改 if start_idx_array.size > 0 and end_idx_array.size > 0: start_idx = start_idx_array[0] end_idx = end_idx_array[-1] # 使用 [-1] 是为了获取这个条件下最后一个满足条件的事件的索引 # 确保 start_idx 小于 end_idx,这样提取出的窗口才有意义 if start_idx < end_idx: event_window = all_events[start_idx:end_idx] if event_window.size > 0: last_timestamp = event_window[-1, 0] # 进行后续处理… else: print(f'No events found between indices {start_idx} and {end_idx}') continue else: print(f'Start index {start_idx} is not less than end index {end_idx}') continue else: print(f'No valid start or end indices for frame {frame_no}, skipping…') continue ################################## with Timer('Building event tensor'): if args.compute_voxel_grid_on_cpu: # pdb.set_trace() event_tensor = events_to_voxel_grid(event_window, num_bins=model.num_bins, width=width, height=height) event_tensor = torch.from_numpy(event_tensor) else: # pdb.set_trace() event_tensor = events_to_voxel_grid_pytorch(event_window, num_bins=model.num_bins, width=width, height=height, device=device) num_events_in_window = event_window.shape[0] reconstructor.update_reconstruction(event_tensor, start_index + num_events_in_window, last_timestamp, frame_no, save_img_path) # reconstructor.update_reconstruction(event_tensor, start_index + num_events_in_window, last_timestamp) start_index += num_events_in_window运行中,生成的recon目录下,很多空文件。 - Assistant
在处理生成的重建目录时,发现有很多空文件,这可能是由于以下几个原因导致的:
- 事件窗口为空:在提取事件窗口时,如果没有符合条件的事件,或者提取出的事件窗口为空,则不会有数据传给
reconstructor.update_reconstruction函数,从而可能导致空文件生成。 - 重建失败:在调用
update_reconstruction函数时,如果重建的过程失败,或者函数内部未能正确处理输入的事件数据,则同样可能导致空文件生成。 - 文件写入问题:在保存重建图像时,若过程中出现文件写入错误,也可能导致空文件生成。
为了解决这些问题,可以从以下几个方面入手:
1. 确认事件窗口的提取过程
确保提取出的事件窗口不是空的。如果为空,可以直接跳过该帧的处理。
event_window_size = event_window.size if event_window_size > 0: reconstructor.update_reconstruction(event_tensor, start_index + event_window.shape[0], last_timestamp, frame_no, save_img_path) else: print(f'Event window for frame {frame_no} is empty, skipping...') continue2. 检查和调试
update_reconstruction函数确保在
update_reconstruction函数中,输入的数据被正确处理。如果其中出现错误,则需要在函数内部添加更多的检测和日志输出,以便识别并解决问题。3. 验证文件写入过程
确保在保存文件时没有出错,可以通过在写入文件前检查内容的有效性:
# Example inside update_reconstruction function output_image = self.reconstruct_image() # replace with actual reconstruction function if output_image is not None and output_image.size > 0: output_path = os.path.join(save_img_path, f'{frame_no}.png') Image.fromarray(output_image).save(output_path) else: print(f'No valid output for frame {frame_no}, skipping file save...')修改后的代码段
在主循环中,添加对
event_window的大小检查,并确保在update_reconstruction调用前确认输入有效:if __name__ == "__main__": # ...[省略部分代码]... for frame_no in range(0, frame_num-1): # ...[省略部分代码]... start_idx_array = np.where(all_events[:, 0] >= frame_timestamp[frame_no][0])[0] end_idx_array = np.where(all_events[:, 0] < frame_timestamp[frame_no][1])[0] if start_idx_array.size > 0 and end_idx_array.size > 0: start_idx = start_idx_array[0] end_idx = end_idx_array[-1] if start_idx < end_idx: event_window = all_events[start_idx:end_idx] if event_window.size > 0: last_timestamp = event_window[-1, 0] else: print(f'No events found between indices {start_idx} and {end_idx}') continue else: print(f'Start index {start_idx} is not less than end index {end_idx}') continue else: print(f'No valid start or end indices for frame {frame_no}, skipping...') continue with Timer('Building event tensor'): if args.compute_voxel_grid_on_cpu: event_tensor = events_to_voxel_grid(event_window, num_bins=model.num_bins, width=width, height=height) event_tensor = torch.from_numpy(event_tensor) else: event_tensor = events_to_voxel_grid_pytorch(event_window, num_bins=model.num_bins, width=width, height=height, device=device) if event_window.shape[0] > 0: num_events_in_window = event_window.shape[0] reconstructor.update_reconstruction(event_tensor, start_index + num_events_in_window, last_timestamp, frame_no, save_img_path) start_index += num_events_in_window else: print(f'Event window is empty for frame {frame_no}, skipping update_reconstruction...') continue以上改动确保只有在事件窗口有实际内容时,才会调用
update_reconstruction函数。另外需要在update_reconstruction函数中仔细检查输入数据,并确保数据处理和文件保存过程的正常进行。 - 事件窗口为空:在提取事件窗口时,如果没有符合条件的事件,或者提取出的事件窗口为空,则不会有数据传给
- User
但是缺少文件,那么在测评的时候就会缺少结果
- Assistant
为了防止生成空文件却导致测评时数据缺失的问题,我们需要确保每个帧都能生成一个有效的重建输出。为此,我们可以在主处理循环中减少添加更严格的检查,并在缺少事件数据或生成失败时生成一个占位图像(如全黑或全白图像)以确保文件存在。
修改后在没有有效数据时生成占位图的代码段
在主循环中添加对于空事件窗口的处理,并生成占位图以确保每个帧都有输出文件:
import numpy as np def create_placeholder_image(height, width): # 创建一个占位图像,此处为全黑图像 return np.zeros((height, width), dtype=np.uint8) if __name__ == "__main__": # ...[省略部分代码]... for frame_no in range(0, frame_num-1): # ...[省略部分代码]... start_idx_array = np.where(all_events[:, 0] >= frame_timestamp[frame_no][0])[0] end_idx_array = np.where(all_events[:, 0] < frame_timestamp[frame_no][1])[0] if start_idx_array.size > 0 and end_idx_array.size > 0: start_idx = start_idx_array[0] end_idx = end_idx_array[-1] if start_idx < end_idx: event_window = all_events[start_idx:end_idx] if event_window.size > 0: last_timestamp = event_window[-1, 0] else: print(f'No events found between indices {start_idx} and {end_idx}') event_window = None # 标记事件窗口为空 else: print(f'Start index {start_idx} is not less than end index {end_idx}') event_window = None # 标记事件窗口为空 else: print(f'No valid start or end indices for frame {frame_no}, skipping...') event_window = None # 标记事件窗口为空 if event_window is not None and event_window.size > 0: with Timer('Building event tensor'): if args.compute_voxel_grid_on_cpu: event_tensor = events_to_voxel_grid(event_window, num_bins=model.num_bins, width=width, height=height) event_tensor = torch.from_numpy(event_tensor) else: event_tensor = events_to_voxel_grid_pytorch(event_window, num_bins=model.num_bins, width=width, height=height, device=device) num_events_in_window = event_window.shape[0] reconstructor.update_reconstruction(event_tensor, start_index + num_events_in_window, last_timestamp, frame_no, save_img_path) start_index += num_events_in_window else: # 生成占位图像并保存 placeholder_image = create_placeholder_image(height, width) placeholder_image_path = os.path.join(save_img_path, f'{frame_no}.png') Image.fromarray(placeholder_image).save(placeholder_image_path) print(f'Generated placeholder image for frame {frame_no}')update_reconstruction函数内部调整如果
update_reconstruction函数内部有额外的步骤也可能失败或生成空文件,可以添加更多的错误处理逻辑,保证即使在失败的情况下,也能生成占位图像。完整示例
这是针对整个主处理循环添加占位图逻辑,以及在
update_reconstruction中进行应对的完整示例代码:主循环代码:
if __name__ == "__main__": # ...[省略部分代码]... for frame_no in range(0, frame_num-1): try: # ...[事件窗口提取省略部分代码]... if event_window is not None and event_window.size > 0: with Timer('Building event tensor'): if args.compute_voxel_grid_on_cpu: event_tensor = events_to_voxel_grid(event_window, num_bins=model.num_bins, width=width, height=height) event_tensor = torch.from_numpy(event_tensor) else: event_tensor = events_to_voxel_grid_pytorch(event_window, num_bins=model.num_bins, width=width, height=height, device=device) num_events_in_window = event_window.shape[0] reconstructor.update_reconstruction(event_tensor, start_index + num_events_in_window, last_timestamp, frame_no, save_img_path) start_index += num_events_in_window else: raise ValueError(f'No events found for frame {frame_no}') except Exception as e: print(f"Exception occurred while processing frame {frame_no}: {e}") placeholder_image = create_placeholder_image(height, width) placeholder_image_path = os.path.join(save_img_path, f'{frame_no}.png') Image.fromarray(placeholder_image).save(placeholder_image_path) print(f'Generated placeholder image for frame {frame_no} due to error.')update_reconstruction函数内部的调整(如果需要处理可能导致失败的步骤):class ImageReconstructor: # ... [原有代码保留] def update_reconstruction(self, event_tensor, start_index, last_timestamp, frame_no, save_img_path): try: # ... [图像重建和写入代码] output_image = self.reconstruct_image() # replace with actual reconstruction function output_path = os.path.join(save_img_path, f'{frame_no}.png') if output_image is not None and output_image.size > 0: Image.fromarray(output_image).save(output_path) else: raise ValueError(f'Reconstructed image is empty for frame {frame_no}') except Exception as e: print(f"Failed to reconstruct or save image for frame {frame_no}: {e}") placeholder_image = create_placeholder_image(self.height, self.width) Image.fromarray(placeholder_image).save(output_path)这样,通过以上调整,即便在提取事件或图像重建过程中发生问题也不会生成空文件,而是确保所有帧都有一个有效(即使是占位)的输出文件,从而在后续测评过程中不会缺失数据。
- User
在import torch from torch.utils.data.distributed import DistributedSampler
datasets related
from lib.train.dataset import Coesot, Fe108, VisEvent from lib.train.data import sampler, opencv_loader, processing, LTRLoader import lib.train.data.transforms as tfm from lib.utils.misc import is_main_process
def update_settings(settings, cfg): settings.print_interval = cfg.TRAIN.PRINT_INTERVAL settings.search_area_factor = {'template': cfg.DATA.TEMPLATE.FACTOR, 'search': cfg.DATA.SEARCH.FACTOR} settings.output_sz = {'template': cfg.DATA.TEMPLATE.SIZE, 'search': cfg.DATA.SEARCH.SIZE} settings.center_jitter_factor = {'template': cfg.DATA.TEMPLATE.CENTER_JITTER, 'search': cfg.DATA.SEARCH.CENTER_JITTER} settings.scale_jitter_factor = {'template': cfg.DATA.TEMPLATE.SCALE_JITTER, 'search': cfg.DATA.SEARCH.SCALE_JITTER} settings.grad_clip_norm = cfg.TRAIN.GRAD_CLIP_NORM settings.print_stats = None settings.batchsize = cfg.TRAIN.BATCH_SIZE settings.scheduler_type = cfg.TRAIN.SCHEDULER.TYPE
def names2datasets(name_list: list, settings, image_loader): assert isinstance(name_list, list) datasets = [] for name in name_list: assert name in ["LASOT", "GOT10K_vottrain", "GOT10K_votval", "GOT10K_train_full", "GOT10K_official_val", "COCO17", "VID", "TRACKINGNET", "COESOT", "COESOT_VAL", "FE108", "FE108_VAL", "VisEvent", "VisEvent_VAL"] if name == "LASOT": if settings.use_lmdb: print("Building lasot dataset from lmdb") datasets.append(Lasot_lmdb(settings.env.lasot_lmdb_dir, split='train', image_loader=image_loader)) else: datasets.append(Lasot(settings.env.lasot_dir, split='train', image_loader=image_loader)) if name == "GOT10K_vottrain": if settings.use_lmdb: print("Building got10k from lmdb") datasets.append(Got10k_lmdb(settings.env.got10k_lmdb_dir, split='vottrain', image_loader=image_loader)) else: datasets.append(Got10k(settings.env.got10k_dir, split='vottrain', image_loader=image_loader)) if name == "GOT10K_train_full": if settings.use_lmdb: print("Building got10k_train_full from lmdb") datasets.append(Got10k_lmdb(settings.env.got10k_lmdb_dir, split='train_full', image_loader=image_loader)) else: datasets.append(Got10k(settings.env.got10k_dir, split='train_full', image_loader=image_loader)) if name == "GOT10K_votval": if settings.use_lmdb: print("Building got10k from lmdb") datasets.append(Got10k_lmdb(settings.env.got10k_lmdb_dir, split='votval', image_loader=image_loader)) else: datasets.append(Got10k(settings.env.got10k_dir, split='votval', image_loader=image_loader)) if name == "GOT10K_official_val": if settings.use_lmdb: raise ValueError("Not implement") else: datasets.append(Got10k(settings.env.got10k_val_dir, split=None, image_loader=image_loader)) if name == "COCO17": if settings.use_lmdb: print("Building COCO2017 from lmdb") datasets.append(MSCOCOSeq_lmdb(settings.env.coco_lmdb_dir, version="2017", image_loader=image_loader)) else: datasets.append(MSCOCOSeq(settings.env.coco_dir, version="2017", image_loader=image_loader)) if name == "VID": if settings.use_lmdb: print("Building VID from lmdb") datasets.append(ImagenetVID_lmdb(settings.env.imagenet_lmdb_dir, image_loader=image_loader)) else: datasets.append(ImagenetVID(settings.env.imagenet_dir, image_loader=image_loader)) if name == "TRACKINGNET": if settings.use_lmdb: print("Building TrackingNet from lmdb") datasets.append(TrackingNet_lmdb(settings.env.trackingnet_lmdb_dir, image_loader=image_loader)) else: # raise ValueError("NOW WE CAN ONLY USE TRACKINGNET FROM LMDB") datasets.append(TrackingNet(settings.env.trackingnet_dir, image_loader=image_loader)) if name == "COESOT": datasets.append(Coesot(settings.env.coesot_dir, split='train', image_loader=image_loader)) if name == "COESOT_VAL": datasets.append(Coesot(settings.env.coesot_val_dir, split='val', image_loader=image_loader)) if name == "FE108": datasets.append(Fe108(settings.env.fe108_dir, split='train', image_loader=image_loader)) if name == "FE108_VAL": datasets.append(Fe108(settings.env.fe108_val_dir, split='val', image_loader=image_loader)) if name == "VisEvent": datasets.append(VisEvent(settings.env.visevent_dir, split='train', image_loader=image_loader)) if name == "VisEvent_VAL": datasets.append(VisEvent(settings.env.visevent_val_dir, split='val', image_loader=image_loader))
return datasetsdef build_dataloaders(cfg, settings): # Data transform # transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05), tfm.RandomHorizontalFlip(0.5)) transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05)) # for FE108 p=0.5 else 0.05
transform_train = tfm.Transform(tfm.ToTensor(), # tfm.RandomHorizontalFlip_Norm(0.5), # FE108, not normalize. tfm.Normalize(mean=cfg.DATA.MEAN, std=cfg.DATA.STD) ) transform_val = tfm.Transform(tfm.ToTensor(), tfm.Normalize(mean=cfg.DATA.MEAN, std=cfg.DATA.STD) ) # The tracking pairs processing module output_sz = settings.output_sz search_area_factor = settings.search_area_factor data_processing_train = processing.STARKProcessing(search_area_factor=search_area_factor, output_sz=output_sz, center_jitter_factor=settings.center_jitter_factor, scale_jitter_factor=settings.scale_jitter_factor, mode='sequence', transform=transform_train, joint_transform=transform_joint, settings=settings) data_processing_val = processing.STARKProcessing(search_area_factor=search_area_factor, output_sz=output_sz, center_jitter_factor=settings.center_jitter_factor, scale_jitter_factor=settings.scale_jitter_factor, mode='sequence', transform=transform_val, joint_transform=transform_joint, settings=settings) # Train sampler and loader settings.num_template = getattr(cfg.DATA.TEMPLATE, "NUMBER", 1) settings.num_search = getattr(cfg.DATA.SEARCH, "NUMBER", 1) sampler_mode = getattr(cfg.DATA, "SAMPLER_MODE", "causal") train_cls = getattr(cfg.TRAIN, "TRAIN_CLS", False) print("sampler_mode", sampler_mode) dataset_train = sampler.TrackingSampler(datasets=names2datasets(cfg.DATA.TRAIN.DATASETS_NAME, settings, opencv_loader), p_datasets=cfg.DATA.TRAIN.DATASETS_RATIO, samples_per_epoch=cfg.DATA.TRAIN.SAMPLE_PER_EPOCH, max_gap=cfg.DATA.MAX_SAMPLE_INTERVAL, num_search_frames=settings.num_search, num_template_frames=settings.num_template, processing=data_processing_train, frame_sample_mode=sampler_mode, train_cls=train_cls) train_sampler = DistributedSampler(dataset_train) if settings.local_rank != -1 else None shuffle = False if settings.local_rank != -1 else True loader_train = LTRLoader('train', dataset_train, training=True, batch_size=cfg.TRAIN.BATCH_SIZE, shuffle=shuffle, num_workers=cfg.TRAIN.NUM_WORKER, drop_last=True, stack_dim=1, sampler=train_sampler) # Validation samplers and loaders dataset_val = sampler.TrackingSampler(datasets=names2datasets(cfg.DATA.VAL.DATASETS_NAME, settings, opencv_loader), p_datasets=cfg.DATA.VAL.DATASETS_RATIO, samples_per_epoch=cfg.DATA.VAL.SAMPLE_PER_EPOCH, max_gap=cfg.DATA.MAX_SAMPLE_INTERVAL, num_search_frames=settings.num_search, num_template_frames=settings.num_template, processing=data_processing_val, frame_sample_mode=sampler_mode, train_cls=train_cls) val_sampler = DistributedSampler(dataset_val) if settings.local_rank != -1 else None loader_val = LTRLoader('val', dataset_val, training=False, batch_size=cfg.TRAIN.BATCH_SIZE, num_workers=cfg.TRAIN.NUM_WORKER, drop_last=True, stack_dim=1, sampler=val_sampler, epoch_interval=cfg.TRAIN.VAL_EPOCH_INTERVAL) return loader_train, loader_valdef get_optimizer_scheduler(net, cfg): train_cls = getattr(cfg.TRAIN, "TRAIN_CLS", False) if train_cls: print("Only training classification head. Learnable parameters are shown below.") param_dicts = [ {"params": [p for n, p in net.named_parameters() if "cls" in n and p.requires_grad]} ]
for n, p in net.named_parameters(): if "cls" not in n: p.requires_grad = False else: print(n) else: param_dicts = [ {"params": [p for n, p in net.named_parameters() if "backbone" not in n and p.requires_grad]}, { "params": [p for n, p in net.named_parameters() if "backbone" in n and p.requires_grad], "lr": cfg.TRAIN.LR * cfg.TRAIN.BACKBONE_MULTIPLIER, }, ] if is_main_process(): print("Learnable parameters are shown below.") for n, p in net.named_parameters(): if p.requires_grad: print(n) if cfg.TRAIN.OPTIMIZER == "ADAMW": optimizer = torch.optim.AdamW(param_dicts, lr=cfg.TRAIN.LR, weight_decay=cfg.TRAIN.WEIGHT_DECAY) else: raise ValueError("Unsupported Optimizer") if cfg.TRAIN.SCHEDULER.TYPE == 'step': lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, cfg.TRAIN.LR_DROP_EPOCH) elif cfg.TRAIN.SCHEDULER.TYPE == "Mstep": lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=cfg.TRAIN.SCHEDULER.MILESTONES, gamma=cfg.TRAIN.SCHEDULER.GAMMA) else: raise ValueError("Unsupported scheduler") return optimizer, lr_scheduler中 # transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05), tfm.RandomHorizontalFlip(0.5)) transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05)) # for FE108 p=0.5 else 0.05 transform_train = tfm.Transform(tfm.ToTensor(), # tfm.RandomHorizontalFlip_Norm(0.5), # FE108, not normalize.的注释中,对FE108的备注是什么意思? - Assistant
在注释中提到“for FE108 p=0.5 else 0.05”和“FE108, not normalize.”,这是指对数据集 FE108 的特定数据预处理和增强操作的设置。这些注释的目的是提醒开发者,在处理 FE108 数据集时需要特别注意的数据变换设置。
注释解释:
1.
transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05))注释:
# for FE108 p=0.5 else 0.05这条注释的意思是,对于 FE108 数据集,
ToGrayscale转换的概率应设置为 0.5,而对于其他数据集,概率应设置为 0.05。当前的代码设置将ToGrayscale转换的概率设置为 0.05。要符合注释所提到的不同数据集的概率要求,可以根据数据集的名称进行条件分支调整,例如:
if 'FE108' in cfg.DATA.TRAIN.DATASETS_NAME or 'FE108_VAL' in cfg.DATA.VAL.DATASETS_NAME: transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.5)) else: transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05))2.
transform_train = tfm.Transform(tfm.ToTensor())注释:
# FE108, not normalize.这里的注释表明,对于 FE108 数据集,在
train模式下不进行归一化(Normalizing)操作。然而,为了满足其他数据集的需求,通常还是需要进行归一化操作。要根据数据集的需求进行条件处理,可以像这样编码:
if 'FE108' in cfg.DATA.TRAIN.DATASETS_NAME or 'FE108_VAL' in cfg.DATA.VAL.DATASETS_NAME: transform_train = tfm.Transform(tfm.ToTensor()) else: transform_train = tfm.Transform( tfm.ToTensor(), tfm.Normalize(mean=cfg.DATA.MEAN, std=cfg.DATA.STD) )完整调整代码示例:
下面是根据上述注释提供的建议,对代码进行调整后的示例:
import torch from torch.utils.data.distributed import DistributedSampler # datasets related from lib.train.dataset import Coesot, Fe108, VisEvent from lib.train.data import sampler, opencv_loader, processing, LTRLoader import lib.train.data.transforms as tfm from lib.utils.misc import is_main_process def update_settings(settings, cfg): settings.print_interval = cfg.TRAIN.PRINT_INTERVAL settings.search_area_factor = {'template': cfg.DATA.TEMPLATE.FACTOR, 'search': cfg.DATA.SEARCH.FACTOR} settings.output_sz = {'template': cfg.DATA.TEMPLATE.SIZE, 'search': cfg.DATA.SEARCH.SIZE} settings.center_jitter_factor = {'template': cfg.DATA.TEMPLATE.CENTER_JITTER, 'search': cfg.DATA.SEARCH.CENTER_JITTER} settings.scale_jitter_factor = {'template': cfg.DATA.TEMPLATE.SCALE_JITTER, 'search': cfg.DATA.SEARCH.SCALE_JITTER} settings.grad_clip_norm = cfg.TRAIN.GRAD_CLIP_NORM settings.print_stats = None settings.batchsize = cfg.TRAIN.BATCH_SIZE settings.scheduler_type = cfg.TRAIN.SCHEDULER.TYPE def names2datasets(name_list: list, settings, image_loader): assert isinstance(name_list, list) datasets = [] for name in name_list: assert name in ["LASOT", "GOT10K_vottrain", "GOT10K_votval", "GOT10K_train_full", "GOT10K_official_val", "COCO17", "VID", "TRACKINGNET", "COESOT", "COESOT_VAL", "FE108", "FE108_VAL", "VisEvent", "VisEvent_VAL"] if name == "LASOT": if settings.use_lmdb: print("Building lasot dataset from lmdb") datasets.append(Lasot_lmdb(settings.env.lasot_lmdb_dir, split='train', image_loader=image_loader)) else: datasets.append(Lasot(settings.env.lasot_dir, split='train', image_loader=image_loader)) if name == "GOT10K_vottrain": if settings.use_lmdb: print("Building got10k from lmdb") datasets.append(Got10k_lmdb(settings.env.got10k_lmdb_dir, split='vottrain', image_loader=image_loader)) else: datasets.append(Got10k(settings.env.got10k_dir, split='vottrain', image_loader=image_loader)) if name == "GOT10K_train_full": if settings.use_lmdb: print("Building got10k_train_full from lmdb") datasets.append(Got10k_lmdb(settings.env.got10k_lmdb_dir, split='train_full', image_loader=image_loader)) else: datasets.append(Got10k(settings.env.got10k_dir, split='train_full', image_loader=image_loader)) if name == "GOT10K_votval": if settings.use_lmdb: print("Building got10k from lmdb") datasets.append(Got10k_lmdb(settings.env.got10k_lmdb_dir, split='votval', image_loader=image_loader)) else: datasets.append(Got10k(settings.env.got10k_dir, split='votval', image_loader=image_loader)) if name == "GOT10K_official_val": if settings.use_lmdb: raise ValueError("Not implement") else: datasets.append(Got10k(settings.env.got10k_val_dir, split=None, image_loader=image_loader)) if name == "COCO17": if settings.use_lmdb: print("Building COCO2017 from lmdb") datasets.append(MSCOCOSeq_lmdb(settings.env.coco_lmdb_dir, version="2017", image_loader=image_loader)) else: datasets.append(MSCOCOSeq(settings.env.coco_dir, version="2017", image_loader=image_loader)) if name == "VID": if settings.use_lmdb: print("Building VID from lmdb") datasets.append(ImagenetVID_lmdb(settings.env.imagenet_lmdb_dir, image_loader=image_loader)) else: datasets.append(ImagenetVID(settings.env.imagenet_dir, image_loader=image_loader)) if name == "TRACKINGNET": if settings.use_lmdb: print("Building TrackingNet from lmdb") datasets.append(TrackingNet_lmdb(settings.env.trackingnet_lmdb_dir, image_loader=image_loader)) else: # raise ValueError("NOW WE CAN ONLY USE TRACKINGNET FROM LMDB") datasets.append(TrackingNet(settings.env.trackingnet_dir, image_loader=image_loader)) if name == "COESOT": datasets.append(Coesot(settings.env.coesot_dir, split='train', image_loader=image_loader)) if name == "COESOT_VAL": datasets.append(Coesot(settings.env.coesot_val_dir, split='val', image_loader=image_loader)) if name == "FE108": datasets.append(Fe108(settings.env.fe108_dir, split='train', image_loader=image_loader)) if name == "FE108_VAL": datasets.append(Fe108(settings.env.fe108_val_dir, split='val', image_loader=image_loader)) if name == "VisEvent": datasets.append(VisEvent(settings.env.visevent_dir, split='train', image_loader=image_loader)) if name == "VisEvent_VAL": datasets.append(VisEvent(settings.env.visevent_val_dir, split='val', image_loader=image_loader)) return datasets def build_dataloaders(cfg, settings): # Data transform if 'FE108' in cfg.DATA.TRAIN.DATASETS_NAME or 'FE108_VAL' in cfg.DATA.VAL.DATASETS_NAME: transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.5)) transform_train = tfm.Transform(tfm.ToTensor()) else: transform_joint = tfm.Transform(tfm.ToGrayscale(probability=0.05)) transform_train = tfm.Transform( tfm.ToTensor(), tfm.Normalize(mean=cfg.DATA.MEAN, std=cfg.DATA.STD) ) transform_val = tfm.Transform( tfm.ToTensor(), tfm.Normalize(mean=cfg.DATA.MEAN, std=cfg.DATA.STD) ) # The tracking pairs processing module output_sz = settings.output_sz search_area_factor = settings.search_area_factor data_processing_train = processing.STARKProcessing(search_area_factor=search_area_factor, output_sz=output_sz, center_jitter_factor=settings.center_jitter_factor, scale_jitter_factor=settings.scale_jitter_factor, mode='sequence', transform=transform_train, joint_transform=transform_joint, settings=settings) data_processing_val = processing.STARKProcessing(search_area_factor=search_area_factor, output_sz=output_sz, center_jitter_factor=settings.center_jitter_factor, scale_jitter_factor=settings.scale_jitter_factor, mode='sequence', transform=transform_val, joint_transform=transform_joint, settings=settings) # Train sampler and loader settings.num_template = getattr(cfg.DATA.TEMPLATE, "NUMBER", 1) settings.num_search = getattr(cfg.DATA.SEARCH, "NUMBER", 1) sampler_mode = getattr(cfg.DATA, "SAMPLER_MODE", "causal") train_cls = getattr(cfg.TRAIN, "TRAIN_CLS", False) print("sampler_mode", sampler_mode) dataset_train = sampler.TrackingSampler(datasets=names2datasets(cfg.DATA.TRAIN.DATASETS_NAME, settings, opencv_loader), p_datasets=cfg.DATA.TRAIN.DATASETS_RATIO, samples_per_epoch=cfg.DATA.TRAIN.SAMPLE_PER_EPOCH, max_gap=cfg.DATA.MAX_SAMPLE_INTERVAL, num_search_frames=settings.num_search, num_template_frames=settings.num_template, processing=data_processing_train, frame_sample_mode=sampler_mode, train_cls=train_cls) train_sampler = DistributedSampler(dataset_train) if settings.local_rank != -1 else None shuffle = False if settings.local_rank != -1 else True loader_train = LTRLoader('train', dataset_train, training=True, batch_size=cfg.TRAIN.BATCH_SIZE, shuffle=shuffle, num_workers=cfg.TRAIN.NUM_WORKER, drop_last=True, stack_dim=1, sampler=train_sampler) # Validation samplers and loaders dataset_val = sampler.TrackingSampler(datasets=names2datasets(cfg.DATA.VAL.DATASETS_NAME, settings, opencv_loader), p_datasets=cfg.DATA.VAL.DATASETS_RATIO, samples_per_epoch=cfg.DATA.VAL.SAMPLE_PER_EPOCH, max_gap=cfg.DATA.MAX_SAMPLE_INTERVAL, num_search_frames=settings.num_search, num_template_frames=settings.num_template, processing=data_processing_val, frame_sample_mode=sampler_mode, train_cls=train_cls) val_sampler = DistributedSampler(dataset_val) if settings.local_rank != -1 else None loader_val = LTRLoader('val', dataset_val, training=False, batch_size=cfg.TRAIN.BATCH_SIZE, num_workers=cfg.TRAIN.NUM_WORKER, drop_last=True, stack_dim=1, sampler=val_sampler, epoch_interval=cfg.TRAIN.VAL_EPOCH_INTERVAL) return loader_train, loader_val def get_optimizer_scheduler(net, cfg): train_cls = getattr(cfg.TRAIN, "TRAIN_CLS", False) if train_cls: print("Only training classification head. Learnable parameters are shown below.") param_dicts = [ {"params": [p for n, p in net.named_parameters() if "cls" in n and p.requires_grad]} ] for n, p in net.named_parameters(): if "cls" not in n: p.requires_grad = False else: print(n) else: param_dicts = [ {"params": [p for n, p in net.named_parameters() if "backbone" not in n and p.requires_grad]}, { "params": [p for n, p in net.named_parameters() if "backbone" in n and p.requires_grad], "lr": cfg.TRAIN.LR * cfg.TRAIN.BACKBONE_MULTIPLIER, }, ] if is_main_process(): print("Learnable parameters are shown below.") for n, p in net.named_parameters(): if p.requires_grad: print(n) if cfg.TRAIN.OPTIMIZER == "ADAMW": optimizer = torch.optim.AdamW(param_dicts, lr=cfg.TRAIN.LR, weight_decay=cfg.TRAIN.WEIGHT_DECAY) else: raise ValueError("Unsupported Optimizer") if cfg.TRAIN.SCHEDULER.TYPE == 'step': lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, cfg.TRAIN.LR_DROP_EPOCH) elif cfg.TRAIN.SCHEDULER.TYPE == "Mstep": lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=cfg.TRAIN.SCHEDULER.MILESTONES, gamma=cfg.TRAIN.SCHEDULER.GAMMA) else: raise ValueError("Unsupported scheduler") return optimizer, lr_scheduler通过这些调整,你可以确保在处理不同数据集时应用正确的图像变换和数据处理过程。注释中的信息被转换为实际的代码逻辑,在处理特定数据集时改变数据变换的行为。