在import os import argparse import random def parseargs(): """ args for training. """…
- User
在import os import argparse import random
def parse_args(): """ args for training. """ parser = argparse.ArgumentParser(description='Parse args for training') # for train parser.add_argument('--script', type=str, help='training script name') parser.add_argument('--config', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--save_dir', type=str, help='root directory to save checkpoints, logs, and tensorboard') parser.add_argument('--mode', type=str, choices=["single", "multiple", "multi_node"], default="multiple", help="train on single gpu or multiple gpus") parser.add_argument('--nproc_per_node', type=int, help="number of GPUs per node") # specify when mode is multiple parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, help='training script name') parser.add_argument('--config_prv', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')
# for multiple machines parser.add_argument('--rank', type=int, help='Rank of the current process.') parser.add_argument('--world-size', type=int, help='Number of processes participating in the job.') parser.add_argument('--ip', type=str, default='127.0.0.1', help='IP of the current rank 0.') parser.add_argument('--port', type=int, default='20000', help='Port of the current rank 0.') # 增加时间步 parser.add_argument('--T', default=16, type=int, help='simulation steps') args = parser.parse_args() return argsdef main(): args = parse_args() if args.mode == "single": train_cmd = "python lib/train/run_training.py --script %s --config %s --save_dir %s --use_lmdb %d "
"--script_prv %s --config_prv %s --distill %d --script_teacher %s --config_teacher %s --use_wandb %d"
% (args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.distill, args.script_teacher, args.config_teacher, args.use_wandb) elif args.mode == "multiple": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_port %d lib/train/run_training.py "
"--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
"--distill %d --script_teacher %s --config_teacher %s"
% (args.nproc_per_node, random.randint(10000, 50000), args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) elif args.mode == "multi_node": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_addr %s --master_port %d --nnodes %d --node_rank %d lib/train/run_training.py "
"--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
"--distill %d --script_teacher %s --config_teacher %s"
% (args.nproc_per_node, args.ip, args.port, args.world_size, args.rank, args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) else: raise ValueError("mode should be 'single' or 'multiple'.") os.system(train_cmd)if name == "main": main() 中将T引入进来,但是需要在def main(): args = parse_args() if args.mode == "single": train_cmd = "python lib/train/run_training.py --script %s --config %s --save_dir %s --use_lmdb %d "
"--script_prv %s --config_prv %s --distill %d --script_teacher %s --config_teacher %s --use_wandb %d"
% (args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.distill, args.script_teacher, args.config_teacher, args.use_wandb) elif args.mode == "multiple": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_port %d lib/train/run_training.py "
"--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
"--distill %d --script_teacher %s --config_teacher %s"
% (args.nproc_per_node, random.randint(10000, 50000), args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) elif args.mode == "multi_node": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_addr %s --master_port %d --nnodes %d --node_rank %d lib/train/run_training.py "
"--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
"--distill %d --script_teacher %s --config_teacher %s"
% (args.nproc_per_node, args.ip, args.port, args.world_size, args.rank, args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) else: raise ValueError("mode should be 'single' or 'multiple'.") os.system(train_cmd) 中将T加进去。 - Assistant
为了将参数
T引入训练命令中,可以在三个不同模式的命令中都加入--T %d。具体如下:import os import argparse import random def parse_args(): """ args for training. """ parser = argparse.ArgumentParser(description='Parse args for training') # for train parser.add_argument('--script', type=str, help='training script name') parser.add_argument('--config', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--save_dir', type=str, help='root directory to save checkpoints, logs, and tensorboard') parser.add_argument('--mode', type=str, choices=["single", "multiple", "multi_node"], default="multiple", help="train on single gpu or multiple gpus") parser.add_argument('--nproc_per_node', type=int, help="number of GPUs per node") # specify when mode is multiple parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, help='training script name') parser.add_argument('--config_prv', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name') # for multiple machines parser.add_argument('--rank', type=int, help='Rank of the current process.') parser.add_argument('--world-size', type=int, help='Number of processes participating in the job.') parser.add_argument('--ip', type=str, default='127.0.0.1', help='IP of the current rank 0.') parser.add_argument('--port', type=int, default='20000', help='Port of the current rank 0.') # 增加时间步 parser.add_argument('--T', default=16, type=int, help='simulation steps') args = parser.parse_args() return args def main(): args = parse_args() if args.mode == "single": train_cmd = "python lib/train/run_training.py --script %s --config %s --save_dir %s --use_lmdb %d " \ "--script_prv %s --config_prv %s --distill %d --script_teacher %s --config_teacher %s --use_wandb %d --T %d" \ % (args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.distill, args.script_teacher, args.config_teacher, args.use_wandb, args.T) elif args.mode == "multiple": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_port %d lib/train/run_training.py " \ "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d " \ "--distill %d --script_teacher %s --config_teacher %s --T %d" \ % (args.nproc_per_node, random.randint(10000, 50000), args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher, args.T) elif args.mode == "multi_node": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_addr %s --master_port %d --nnodes %d --node_rank %d lib/train/run_training.py " \ "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d " \ "--distill %d --script_teacher %s --config_teacher %s --T %d" \ % (args.nproc_per_node, args.ip, args.port, args.world_size, args.rank, args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher, args.T) else: raise ValueError("mode should be 'single', 'multiple' or 'multi_node'.") os.system(train_cmd) if __name__ == "__main__": main()这样,你的时间步参数
T就会在调用训练命令时传递给 Python 脚本。确保你的lib/train/run_training.py脚本能够处理此参数。 - User
训练报错:usage: run_training.py [-h] --script SCRIPT --config CONFIG [--cudnn_benchmark CUDNN_BENCHMARK] [--local_rank LOCAL_RANK] [--save_dir SAVE_DIR] [--seed SEED] [--use_lmdb {0,1}] [--script_prv SCRIPT_PRV] [--config_prv CONFIG_PRV] [--use_wandb {0,1}] [--distill {0,1}] [--script_teacher SCRIPT_TEACHER] [--config_teacher CONFIG_TEACHER] run_training.py: error: unrecognized arguments: --T 16,其中import os import sys import argparse import importlib import cv2 as cv import torch.backends.cudnn import torch.distributed as dist
import random import numpy as np torch.backends.cudnn.benchmark = False
import _init_paths import lib.train.admin.settings as ws_settings
def init_seeds(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False
def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None, use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False, distill=None, script_teacher=None, config_teacher=None): """Run the train script. args: script_name: Name of emperiment in the "experiments/" folder. config_name: Name of the yaml file in the "experiments/<script_name>". cudnn_benchmark: Use cudnn benchmark or not (default is True). """ if save_dir is None: print("save_dir dir is not given. Use the default dir instead.") # This is needed to avoid strange crashes related to opencv cv.setNumThreads(0)
torch.backends.cudnn.benchmark = cudnn_benchmark print('script_name: {}.py config_name: {}.yaml'.format(script_name, config_name)) '''2021.1.5 set seed for different process''' if base_seed is not None: if local_rank != -1: init_seeds(base_seed + local_rank) else: init_seeds(base_seed) settings = ws_settings.Settings() settings.script_name = script_name settings.config_name = config_name settings.project_path = 'train/{}/{}'.format(script_name, config_name) if script_name_prv is not None and config_name_prv is not None: settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv) settings.local_rank = local_rank settings.save_dir = os.path.abspath(save_dir) settings.use_lmdb = use_lmdb prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name)) settings.use_wandb = use_wandb if distill: settings.distill = distill settings.script_teacher = script_teacher settings.config_teacher = config_teacher if script_teacher is not None and config_teacher is not None: settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher) settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher)) expr_module = importlib.import_module('lib.train.train_script_distill') else: expr_module = importlib.import_module('lib.train.train_script') expr_func = getattr(expr_module, 'run') expr_func(settings)def main(): parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.') parser.add_argument('--script', type=str, required=True, help='Name of the train script.') parser.add_argument('--config', type=str, required=True, help="Name of the config file.") parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).') parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training') parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs') parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42') parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.') parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.") parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')
args = parser.parse_args() if args.local_rank != -1: dist.init_process_group(backend='nccl') torch.cuda.set_device(args.local_rank) else: torch.cuda.set_device(0) run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark, local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed, use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv, use_wandb=args.use_wandb, distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher)if name == 'main': # os.environ['CUDA_VISIBLE_DEVICES'] = "4" main(),应该如何修改?
- Assistant
你遇到的错误是因为
T参数没有在run_training.py中定义。所以,当你传递T参数时,Python argparse 无法识别它。你需要在
run_training.py中添加对T参数的支持。以下为如何进行相应修改的步骤:-
在
run_training.py中添加T参数:在
def main()函数中的 argparse 定义部分添加T参数。 -
在
run_training函数中传递T参数:你需要确保将
T参数传递到你的run_training函数中。
以下为修改过的
run_training.py:import os import sys import argparse import importlib import cv2 as cv import torch.backends.cudnn import torch.distributed as dist import random import numpy as np torch.backends.cudnn.benchmark = False import _init_paths import lib.train.admin.settings as ws_settings def init_seeds(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None, use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False, distill=None, script_teacher=None, config_teacher=None, T=16): # Add T parameter here """Run the train script. args: script_name: Name of emperiment in the "experiments/" folder. config_name: Name of the yaml file in the "experiments/<script_name>". cudnn_benchmark: Use cudnn benchmark or not (default is True). """ if save_dir is None: print("save_dir dir is not given. Use the default dir instead.") # This is needed to avoid strange crashes related to opencv cv.setNumThreads(0) torch.backends.cudnn.benchmark = cudnn_benchmark print('script_name: {}.py config_name: {}.yaml'.format(script_name, config_name)) '''2021.1.5 set seed for different process''' if base_seed is not None: if local_rank != -1: init_seeds(base_seed + local_rank) else: init_seeds(base_seed) settings = ws_settings.Settings() settings.script_name = script_name settings.config_name = config_name settings.project_path = 'train/{}/{}'.format(script_name, config_name) if script_name_prv is not None and config_name_prv is not None: settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv) settings.local_rank = local_rank settings.save_dir = os.path.abspath(save_dir) settings.use_lmdb = use_lmdb prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name)) settings.use_wandb = use_wandb settings.T = T # Pass T parameter to settings if distill: settings.distill = distill settings.script_teacher = script_teacher settings.config_teacher = config_teacher if script_teacher is not None and config_teacher is not None: settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher) settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher)) expr_module = importlib.import_module('lib.train.train_script_distill') else: expr_module = importlib.import_module('lib.train.train_script') expr_func = getattr(expr_module, 'run') expr_func(settings) def main(): parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.') parser.add_argument('--script', type=str, required=True, help='Name of the train script.') parser.add_argument('--config', type=str, required=True, help="Name of the config file.") parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).') parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training') parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs') parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42') parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.') parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.") parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name') parser.add_argument('--T', type=int, default=16, help='simulation steps') # Add T parameter here args = parser.parse_args() if args.local_rank != -1: dist.init_process_group(backend='nccl') torch.cuda.set_device(args.local_rank) else: torch.cuda.set_device(0) run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark, local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed, use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv, use_wandb=args.use_wandb, distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher, T=args.T) # Pass T to run_training if __name__ == '__main__': # os.environ['CUDA_VISIBLE_DEVICES'] = "4" main() -
- User
在import os import os.path import numpy as np import torch import csv import pandas import random from collections import OrderedDict from .base_video_dataset import BaseVideoDataset from lib.train.data import jpeg4py_loader from lib.train.admin import env_settings import scipy.io as scio
class Coesot(BaseVideoDataset): def init(self, root=None, image_loader=jpeg4py_loader, split=None, seq_ids=None, data_fraction=None):
root = env_settings().got10k_dir if root is None else root super().__init__('Coesot', root, image_loader) self.sequence_list = self._get_sequence_list() # seq_id is the index of the folder inside the got10k root path if split is not None: if seq_ids is not None: raise ValueError('Cannot set both split_name and seq_ids.') if split == 'train': file_path = os.path.join(self.root, 'train.txt') elif split == 'val': file_path = os.path.join(self.root, 'val.txt') else: raise ValueError('Unknown split name') seq_ids = pandas.read_csv(file_path, header=None, dtype=np.int64).squeeze("columns").values.tolist() elif seq_ids is None: seq_ids = list(range(0, len(self.sequence_list))) self.sequence_list = [self.sequence_list[i] for i in seq_ids] def get_name(self): return 'coesot' def _get_sequence_list(self): with open(os.path.join(self.root, 'list.txt')) as f: dir_list = list(csv.reader(f)) dir_list = [dir_name[0] for dir_name in dir_list] return dir_list def _read_bb_anno(self, seq_path): bb_anno_file = os.path.join(seq_path, "groundtruth.txt") gt = pandas.read_csv(bb_anno_file, delimiter=',', header=None, dtype=np.float32, na_filter=False, low_memory=False).values return torch.tensor(gt) def _get_sequence_path(self, seq_id): return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_aps") # RGB模态 def _get_event_img_sequence_path(self, seq_id): return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_dvs") # 事件模态 - 表征方式 event frame # return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_timesurface") # 事件模态 - 表征方式 event timesurface # return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_recon") # 事件模态 - 表征方式 event timesurface def _get_grountgruth_path(self, seq_id): return os.path.join(self.root, self.sequence_list[seq_id]) def get_sequence_info(self, seq_id): bbox_path = self._get_grountgruth_path(seq_id) bbox = self._read_bb_anno(bbox_path) valid = (bbox[:, 2] > 0) & (bbox[:, 3] > 0) visible = valid.clone().byte() # return {'bbox': bbox, 'valid': valid, 'visible': visible, 'visible_ratio': visible_ratio} return {'bbox': bbox, 'valid': valid, 'visible': visible, } def _get_frame_path(self, seq_path, frame_id): if os.path.exists(os.path.join(seq_path, 'frame{:04}.png'.format(frame_id))): return os.path.join(seq_path, 'frame{:04}.png'.format(frame_id)) # frames start from 0 else: return os.path.join(seq_path, 'frame{:04}.bmp'.format(frame_id)) # some image is bmp def _get_frame(self, seq_path, frame_id): return self.image_loader(self._get_frame_path(seq_path, frame_id)) def _get_event_sequence_path(self, seq_id): ## get evemts' frames return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_voxel") # 事件模态 - 表征方式 event voxel def _get_event_frame(self, seq_path, frame_id): frame_event_list = [] for f_id in frame_id: event_frame_file = os.path.join(seq_path, 'frame{:04}.mat'.format(f_id)) if os.path.getsize(event_frame_file) == 0: event_features = np.zeros(4096, 19) # need_data = [np.zeros([4096, 3]), np.zeros([4096, 16])] else: mat_data = scio.loadmat(event_frame_file) # need_data = [mat_data['coor'], mat_data['features']] event_features = np.concatenate((mat_data['coor'], mat_data['features']), axis=1) # concat coorelate and features (x,y,z, feauture32/16) if np.isnan(event_features).any(): event_features = np.zeros(4096, 19) print(event_frame_file, 'exist nan value in voxel.') frame_event_list.append(event_features) return frame_event_list def get_frames(self, seq_id, frame_ids, anno=None): # RGB模态 seq_path = self._get_sequence_path(seq_id) # obj_meta = self.sequence_meta_info[self.sequence_list[seq_id]] frame_list = [self._get_frame(seq_path, f_id) for f_id in frame_ids] # 事件模态 seq_event_path = self._get_event_img_sequence_path(seq_id) frame_event_img_list = [self._get_frame(seq_event_path, f_id) for f_id in frame_ids] if anno is None: anno = self.get_sequence_info(seq_id) anno_frames = {} for key, value in anno.items(): anno_frames[key] = [value[f_id, ...].clone() for f_id in frame_ids] object_meta = OrderedDict({'object_class_name': None, 'motion_class': None, 'major_class': None, 'root_class': None, 'motion_adverb': None}) # 读取 event voxel # seq_event_path = self._get_event_sequence_path(seq_id) # frame_event_list = self._get_event_frame(seq_event_path, frame_ids) # 现在的目的是:将voxel直接换掉 → event frame(即dvs格式) frame_event_list = frame_event_img_list # 读入的就是 dvs下的新的表征形式了 return frame_list, anno_frames, object_meta, frame_event_list, frame_event_img_list中我们读取的_dvs目录下文件都是frame0000.bmp或png,现在我们需要读取的是_tensor目录,然后该目录下的文件是 frame0000.npz格式,那么现在应该如何修改?
- Assistant
为了实现从
_tensor目录读取.npz格式的文件,需要对现有代码进行以下几项修改:- 新增方法
_get_tensor_sequence_path用于获取_tensor目录的路径。 - 新增方法
_get_tensor_frame用于读取.npz文件。 - 修改现有代码中的
_get_event_img_sequence_path和_get_event_frame方法以适配新的.npz格式读取路径和文件格式。
下面是修改后的代码:
import os import os.path import numpy as np import torch import csv import pandas import random from collections import OrderedDict from .base_video_dataset import BaseVideoDataset from lib.train.data import jpeg4py_loader from lib.train.admin import env_settings import scipy.io as scio class Coesot(BaseVideoDataset): def __init__(self, root=None, image_loader=jpeg4py_loader, split=None, seq_ids=None, data_fraction=None): root = env_settings().got10k_dir if root is None else root super().__init__('Coesot', root, image_loader) self.sequence_list = self._get_sequence_list() # seq_id is the index of the folder inside the got10k root path if split is not None: if seq_ids is not None: raise ValueError('Cannot set both split_name and seq_ids.') if split == 'train': file_path = os.path.join(self.root, 'train.txt') elif split == 'val': file_path = os.path.join(self.root, 'val.txt') else: raise ValueError('Unknown split name') seq_ids = pandas.read_csv(file_path, header=None, dtype=np.int64).squeeze("columns").values.tolist() elif seq_ids is None: seq_ids = list(range(0, len(self.sequence_list))) self.sequence_list = [self.sequence_list[i] for i in seq_ids] def get_name(self): return 'coesot' def _get_sequence_list(self): with open(os.path.join(self.root, 'list.txt')) as f: dir_list = list(csv.reader(f)) dir_list = [dir_name[0] for dir_name in dir_list] return dir_list def _read_bb_anno(self, seq_path): bb_anno_file = os.path.join(seq_path, "groundtruth.txt") gt = pandas.read_csv(bb_anno_file, delimiter=',', header=None, dtype=np.float32, na_filter=False, low_memory=False).values return torch.tensor(gt) def _get_sequence_path(self, seq_id): return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_aps") # RGB模态 def _get_event_img_sequence_path(self, seq_id): return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_dvs") # 事件模态 - 表征方式 event frame def _get_tensor_sequence_path(self, seq_id): return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_tensor") # 新的事件模态 - 表征方式 event tensor def _get_grountgruth_path(self, seq_id): return os.path.join(self.root, self.sequence_list[seq_id]) def get_sequence_info(self, seq_id): bbox_path = self._get_grountgruth_path(seq_id) bbox = self._read_bb_anno(bbox_path) valid = (bbox[:, 2] > 0) & (bbox[:, 3] > 0) visible = valid.clone().byte() # return {'bbox': bbox, 'valid': valid, 'visible': visible, 'visible_ratio': visible_ratio} return {'bbox': bbox, 'valid': valid, 'visible': visible, } def _get_frame_path(self, seq_path, frame_id): if os.path.exists(os.path.join(seq_path, 'frame{:04}.png'.format(frame_id))): return os.path.join(seq_path, 'frame{:04}.png'.format(frame_id)) # frames start from 0 else: return os.path.join(seq_path, 'frame{:04}.bmp'.format(frame_id)) # some image is bmp def _get_frame(self, seq_path, frame_id): return self.image_loader(self._get_frame_path(seq_path, frame_id)) def _get_event_sequence_path(self, seq_id): ## get evemts' frames return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_voxel") # 事件模态 - 表征方式 event voxel def _get_event_frame(self, seq_path, frame_id): frame_event_list = [] for f_id in frame_id: event_frame_file = os.path.join(seq_path, 'frame{:04}.mat'.format(f_id)) if os.path.getsize(event_frame_file) == 0: event_features = np.zeros((4096, 19)) # need_data = [np.zeros([4096, 3]), np.zeros([4096, 16])] else: mat_data = scio.loadmat(event_frame_file) # need_data = [mat_data['coor'], mat_data['features']] event_features = np.concatenate((mat_data['coor'], mat_data['features']), axis=1) # concat coorelate and features (x,y,z, feauture32/16) if np.isnan(event_features).any(): event_features = np.zeros((4096, 19)) print(event_frame_file, 'exist nan value in voxel.') frame_event_list.append(event_features) return frame_event_list def _get_tensor_frame(self, seq_path, frame_id): frame_tensor_list = [] for f_id in frame_id: tensor_frame_file = os.path.join(seq_path, 'frame{:04}.npz'.format(f_id)) if os.path.exists(tensor_frame_file): data = np.load(tensor_frame_file) frame_tensor_list.append(data) else: print(f"File {tensor_frame_file} does not exist.") frame_tensor_list.append(None) return frame_tensor_list def get_frames(self, seq_id, frame_ids, anno=None): # RGB模态 seq_path = self._get_sequence_path(seq_id) # obj_meta = self.sequence_meta_info[self.sequence_list[seq_id]] frame_list = [self._get_frame(seq_path, f_id) for f_id in frame_ids] # 事件模态 seq_event_path = self._get_event_img_sequence_path(seq_id) frame_event_img_list = [self._get_frame(seq_event_path, f_id) for f_id in frame_ids] if anno is None: anno = self.get_sequence_info(seq_id) anno_frames = {} for key, value in anno.items(): anno_frames[key] = [value[f_id, ...].clone() for f_id in frame_ids] object_meta = OrderedDict({'object_class_name': None, 'motion_class': None, 'major_class': None, 'root_class': None, 'motion_adverb': None}) # 读取 event tensor seq_tensor_path = self._get_tensor_sequence_path(seq_id) frame_tensor_list = self._get_tensor_frame(seq_tensor_path, frame_ids) return frame_list, anno_frames, object_meta, frame_tensor_list, frame_event_img_list在这个修改后的代码中,新添的方法
_get_tensor_sequence_path和_get_tensor_frame专门处理_tensor目录和读取.npz文件。get_frames方法的返回值中也包含了frame_tensor_list,即从_tensor目录读取的.npz数据。这样,代码便能够读取指定目录下的.npz文件并进行处理。 - 新增方法
- User
pip install wandb清华镜像源
- Assistant
你可以通过使用国内镜像源来安装 pip 包以加速下载。对于清华镜像源,你可以使用如下命令来安装
wandb包:pip install -i https://pypi.tuna.tsinghua.edu.cn/simple wandb上述命令指定了
pip使用清华大学的镜像源来进行包的安装。备用镜像源
如果清华的镜像源不可用或速度不理想,你也可以尝试其他国内常用的镜像源,例如:
-
阿里云镜像源:
pip install -i https://mirrors.aliyun.com/pypi/simple wandb -
中国科学技术大学镜像源:
pip install -i https://pypi.mirrors.ustc.edu.cn/simple wandb -
豆瓣镜像源:
pip install -i https://pypi.douban.com/simple wandb
配置全局镜像源
如果你希望每次使用 pip 时都默认使用某个镜像源,可以配置 pip 的全局镜像,一劳永逸。
-
编辑或创建
pip.conf(Linux 或 macOS) 或pip.ini(Windows):-
Linux/macOS:
mkdir -p ~/.pip echo "[global] index-url = https://pypi.tuna.tsinghua.edu.cn/simple" > ~/.pip/pip.conf -
Windows: 在
C:\Users\你的用户名\目录下创建一个名为pip的文件夹,然后在里面创建一个pip.ini文件,内容如下:[global] index-url = https://pypi.tuna.tsinghua.edu.cn/simple
-
这样,你在使用 pip 安装包时,就会默认使用指定的镜像源。希望对你有所帮助!
-
- User
网络是:import torch import torch.nn as nn from spikingjelly.clock_driven.neuron import MultiStepParametricLIFNode, MultiStepLIFNode from spikingjelly.clock_driven import layer from timm.models.layers import to_2tuple, trunc_normal_, DropPath from timm.models.registry import register_model from timm.models.vision_transformer import _cfg from einops.layers.torch import Rearrange import torch.nn.functional as F from functools import partial from timm.models import create_model import pdb from .Bottleneck import BottleneckTransformer
from mmengine.model.weight_init import kaiming_init
from mmcv.cnn import kaiming_init
all = ['Spikingformer'] tau_thr = 1.5
class MLP(nn.Module): def init(self, in_features, hidden_features=None, out_features=None, drop=0.): super().init() out_features = out_features or in_features hidden_features = hidden_features or in_features self.mlp1_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True) #, backend='cupy' self.mlp1_conv = nn.Conv2d(in_features, hidden_features, kernel_size=1, stride=1) self.mlp1_bn = nn.BatchNorm2d(hidden_features)
self.mlp2_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True) self.mlp2_conv = nn.Conv2d(hidden_features, out_features, kernel_size=1, stride=1) self.mlp2_bn = nn.BatchNorm2d(out_features) self.c_hidden = hidden_features self.c_output = out_features def forward(self, x): T, B, C, H, W = x.shape x = self.mlp1_lif(x) x = self.mlp1_conv(x.flatten(0, 1)) x = self.mlp1_bn(x).reshape(T, B, self.c_hidden, H, W) x = self.mlp2_lif(x) x = self.mlp2_conv(x.flatten(0, 1)) x = self.mlp2_bn(x).reshape(T, B, C, H, W) return xclass SpikingSelfAttention(nn.Module): def init(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., sr_ratio=1): super().init() assert dim % num_heads == 0, f"dim {dim} should be divided by num_heads {num_heads}."
self.dim = dim self.num_heads = num_heads self.proj_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True) self.q_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1, bias=False) self.q_bn = nn.BatchNorm1d(dim) self.q_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True) self.k_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1, bias=False) self.k_bn = nn.BatchNorm1d(dim) self.k_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True) self.v_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1, bias=False) self.v_bn = nn.BatchNorm1d(dim) self.v_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True) self.attn_lif = MultiStepLIFNode(tau=tau_thr, v_threshold=0.5, detach_reset=True) self.proj_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1) self.proj_bn = nn.BatchNorm1d(dim) def forward(self, x): T, B, C, H, W = x.shape x = self.proj_lif(x) x = x.flatten(3) T, B, C, N = x.shape x_for_qkv = x.flatten(0, 1) q_conv_out = self.q_conv(x_for_qkv) q_conv_out = self.q_bn(q_conv_out).reshape(T, B, C, N) q_conv_out = self.q_lif(q_conv_out) q = q_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C // self.num_heads).permute(0, 1, 3, 2, 4) #torch.Size([4, 2, 16, 336, 16]) k_conv_out = self.k_conv(x_for_qkv) k_conv_out = self.k_bn(k_conv_out).reshape(T, B, C, N) k_conv_out = self.k_lif(k_conv_out) k = k_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C // self.num_heads).permute(0, 1, 3, 2, 4)#torch.Size([4, 2, 16, 336, 16]) v_conv_out = self.v_conv(x_for_qkv) v_conv_out = self.v_bn(v_conv_out).reshape(T, B, C, N) v_conv_out = self.v_lif(v_conv_out) v = v_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C // self.num_heads).permute(0, 1, 3, 2, 4) #torch.Size([4, 2, 16, 336, 16]) attn = (q @ k.transpose(-2, -1)) #torch.Size([4, 2, 16, 336, 336]) x = (attn @ v) * 0.125 #torch.Size([4, 2, 16, 336, 16]) x = x.transpose(3, 4).reshape(T, B, C, N) x = self.attn_lif(x) x = x.flatten(0, 1) x = self.proj_bn(self.proj_conv(x)).reshape(T, B, C, H, W) #torch.Size([4, 2, 256, 21, 16]) return xclass SpikingTransformer(nn.Module): def init(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., drop_path=0., norm_layer=nn.LayerNorm, sr_ratio=1): super().init() self.norm1 = norm_layer(dim) self.attn = SpikingSelfAttention(dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop, sr_ratio=sr_ratio) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() self.norm2 = norm_layer(dim) mlp_hidden_dim = int(dim * mlp_ratio) self.mlp = MLP(in_features=dim, hidden_features=mlp_hidden_dim, drop=drop)
def forward(self, x): x = x + self.attn(x) x = x + self.mlp(x) return xclass SpikingTokenizer(nn.Module): def init(self, img_size_h=128, img_size_w=128, patch_size=4, in_channels=2, embed_dims=256): super().init() self.image_size = [img_size_h, img_size_w] patch_size = to_2tuple(patch_size) self.patch_size = patch_size self.C = in_channels self.H, self.W = self.image_size[0] // patch_size[0], self.image_size[1] // patch_size[1] self.num_patches = self.H * self.W
self.proj_conv = nn.Conv2d(in_channels, embed_dims//8, kernel_size=3, stride=1, padding=1, bias=False) self.proj_bn = nn.BatchNorm2d(embed_dims//8) self.proj1_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True) self.maxpool1 = torch.nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False) self.proj1_conv = nn.Conv2d(embed_dims//8, embed_dims//4, kernel_size=3, stride=1, padding=1, bias=False) self.proj1_bn = nn.BatchNorm2d(embed_dims//4) self.proj2_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True) self.maxpool2 = torch.nn.MaxPool2d(kernel_size=(3,2), stride=2, padding=0, dilation=1, ceil_mode=False) self.proj2_conv = nn.Conv2d(embed_dims//4, embed_dims//2, kernel_size=3, stride=1, padding=1, bias=False) self.proj2_bn = nn.BatchNorm2d(embed_dims//2) self.proj3_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True) self.maxpool3 = torch.nn.MaxPool2d(kernel_size=(2,3), stride=2, padding=0, dilation=1, ceil_mode=False) self.proj3_conv = nn.Conv2d(embed_dims//2, embed_dims, kernel_size=3, stride=1, padding=1, bias=False) self.proj3_bn = nn.BatchNorm2d(embed_dims) self.proj4_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True) self.maxpool4 = torch.nn.MaxPool2d(kernel_size=(3,2), stride=2, padding=0, dilation=1, ceil_mode=False) self.proj4_conv = nn.Conv2d(embed_dims, embed_dims, kernel_size=3, stride=1, padding=1, bias=False) self.proj4_bn = nn.BatchNorm2d(embed_dims) def forward(self, x): T, B, C, H, W = x.shape x = self.proj_conv(x.flatten(0, 1)) x = self.proj_bn(x).reshape(T,B,-1,H,W).contiguous() x = self.proj1_lif(x).flatten(0,1).contiguous() x = self.maxpool1(x) x = self.proj1_conv(x) x = self.proj1_bn(x).reshape(T, B, -1, H//2, W//2).contiguous() x = self.proj2_lif(x).flatten(0, 1).contiguous() x = self.maxpool2(x) x = self.proj2_conv(x) x = self.proj2_bn(x).reshape(T, B, -1, H//4, W//4).contiguous() x = self.proj3_lif(x).flatten(0, 1).contiguous() x = self.maxpool3(x) x = self.proj3_conv(x) x = self.proj3_bn(x).reshape(T, B, -1, H//8, W//8).contiguous() x = self.proj4_lif(x).flatten(0, 1).contiguous() x = self.maxpool4(x) x = self.proj4_conv(x) x = self.proj4_bn(x).reshape(T, B, -1, H//16, W//16).contiguous() return x, (None, None)class vit_snn(nn.Module): def init(self, img_size_h=128, img_size_w=128, patch_size=16, in_channels=2, num_classes=11, embed_dims=[64, 128, 256], num_heads=[1, 2, 4], mlp_ratios=[4, 4, 4], qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=nn.LayerNorm, depths=[6, 8, 6], sr_ratios=[8, 4, 2], T=4, pretrained_cfg=None, ): super().init() self.num_classes = num_classes self.depths = depths self.T = T dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depths)]
patch_embed = SpikingTokenizer(img_size_h=img_size_h, img_size_w=img_size_w, patch_size=patch_size, in_channels=in_channels, embed_dims=embed_dims) num_patches = patch_embed.num_patches block = nn.ModuleList([SpikingTransformer( dim=embed_dims, num_heads=num_heads, mlp_ratio=mlp_ratios, qkv_bias=qkv_bias, qk_scale=qk_scale, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[j], norm_layer=norm_layer, sr_ratio=sr_ratios) for j in range(depths)]) self.Bottleneck_block = nn.ModuleList([BottleneckTransformer( dim=embed_dims, num_heads=4, mlp_ratio=mlp_ratios, qkv_bias=qkv_bias, drop=0., attn_drop=0., init_values=None, drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm) for j in range(2)]) setattr(self, f"patch_embed", patch_embed) setattr(self, f"block", block) # classification head self.head = nn.Linear(8192, num_classes) if num_classes > 0 else nn.Identity() self.apply(self._init_weights) self.bottleneck_initfeature = nn.Parameter(torch.randn(1, 64, 256)) kaiming_init(self.bottleneck_initfeature, mode='fan_out') self.pos_embed = nn.Parameter(torch.zeros(1, 400, 256)) self.pos_drop = nn.Dropout(p=0.) def _init_weights(self, m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.LayerNorm): nn.init.constant_(m.bias, 0) nn.init.constant_(m.weight, 1.0) def forward_features(self, x): block = getattr(self, f"block") patch_embed = getattr(self, f"patch_embed") x, (H, W) = patch_embed(x) #x--torch.Size([4, 2, 256, 21, 16]) for blk in block: x = blk(x) #torch.Size([16, 8, 256, 21, 16]) #torch.Size([4, 2, 256, 21, 16]) T,B,C,H,W =x.shape botlk_x = x.flatten(3) #torch.Size([4, 2, 256, 336]) T, B, C, N botlk_x = botlk_x.flatten(0, 1).permute(0,2,1) #torch.Size([8, 336, 256]) bottleneck_feature = self.bottleneck_initfeature.expand(T*B,64,C) fusion_feature_agg = torch.cat((bottleneck_feature,botlk_x),1) #BT, 672,256 fusion_feature_agg = self.pos_drop(fusion_feature_agg + self.pos_embed) # for bottleneck_blk in self.Bottleneck_block: fusion_feature_agg = bottleneck_blk(fusion_feature_agg) bottleneck_feature = fusion_feature_agg[:,0:336,:].mean(1).reshape(B,1,-1) #torch.Size([16, 2, 336, 256]) return x.flatten(3).mean(3), bottleneck_feature #torch.Size([4, 2, 256]) def forward(self, event_tensor): B, N, C, H, W = event_tensor.shape event_tensor = event_tensor.permute(1, 0, 2, 3, 4) # [T, N, 2, *, *] torch.Size([16, 2, 12, 346, 260]) event_tensor, bottleneck_feature = self.forward_features(event_tensor) #torch.Size([16, 2, 256, 21, 16]) torch.Size([16, 2, 336, 256]) event_tensor = event_tensor.permute(1,0,2).reshape(B,-1) x1 = self.head(event_tensor) return x1@register_model def Spikingformer(pretrained=False, **kwargs): model = vit_snn( patch_size=16, embed_dims=256, num_heads=8, mlp_ratios=4, in_channels=12, num_classes=300, qkv_bias=False, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=2, sr_ratios=1, **kwargs ) model.default_cfg = _cfg() return model
,集成到vit_ce中,但是报错:sampler_mode causal Traceback (most recent call last): File "lib/train/run_training.py", line 112, in main() File "lib/train/run_training.py", line 103, in main run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark, File "lib/train/run_training.py", line 76, in run_training expr_func(settings) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/train/train_script.py", line 55, in run net = build_ceutrack(cfg) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/ceutrack.py", line 124, in build_ceutrack backbone = vit_base_patch16_224_ce(pretrained, drop_path_rate=cfg.TRAIN.DROP_PATH_RATE, File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 193, in vit_base_patch16_224_ce model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 175, in _create_vision_transformer model = VisionTransformerCE(**kwargs) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 58, in init self.vit_snn_model = vit_snn(img_size_h=128, img_size_w=128, patch_size=16, in_channels=12, num_classes=11, File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_snn.py", line 200, in init dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depths)]
TypeError: linspace(): argument 'steps' (position 3) must be int, not list - Assistant
你遇到的错误是由于
torch.linspace的第三个参数steps传递了一个列表而不是一个整数。depths是一个列表,因此必须处理它以确保torch.linspace接受整数。在你的
vit_snn类的构造函数中修改生成dpr列表的方式即可解决这个问题。以下是修改后的__init__方法:class vit_snn(nn.Module): def __init__(self, img_size_h=128, img_size_w=128, patch_size=16, in_channels=2, num_classes=11, embed_dims=[64, 128, 256], num_heads=[1, 2, 4], mlp_ratios=[4, 4, 4], qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=nn.LayerNorm, depths=[6, 8, 6], sr_ratios=[8, 4, 2], T=4, pretrained_cfg=None): super().__init__() self.num_classes = num_classes self.depths = depths self.T = T dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] # Modify here patch_embed = SpikingTokenizer(img_size_h=img_size_h, img_size_w=img_size_w, patch_size=patch_size, in_channels=in_channels, embed_dims=embed_dims) num_patches = patch_embed.num_patches block = nn.ModuleList([SpikingTransformer( dim=embed_dims[i], num_heads=num_heads[i], mlp_ratio=mlp_ratios[i], qkv_bias=qkv_bias, qk_scale=qk_scale, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[j], norm_layer=norm_layer, sr_ratio=sr_ratios[i]) for i in range(len(embed_dims)) for j in range(depths[i])]) self.Bottleneck_block = nn.ModuleList([BottleneckTransformer( dim=embed_dims[-1], num_heads=4, mlp_ratio=mlp_ratios[-1], qkv_bias=qkv_bias, drop=0., attn_drop=0., init_values=None, drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm) for j in range(2)]) setattr(self, f"patch_embed", patch_embed) setattr(self, f"block", block) # classification head self.head = nn.Linear(8192, num_classes) if num_classes > 0 else nn.Identity() self.apply(self._init_weights) self.bottleneck_initfeature = nn.Parameter(torch.randn(1, 64, 256)) kaiming_init(self.bottleneck_initfeature, mode='fan_out') self.pos_embed = nn.Parameter(torch.zeros(1, 400, 256)) self.pos_drop = nn.Dropout(p=0.) def _init_weights(self, m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.LayerNorm): nn.init.constant_(m.bias, 0) nn.init.constant_(m.weight, 1.0) def forward_features(self, x): block = getattr(self, f"block") patch_embed = getattr(self, f"patch_embed") x, (H, W) = patch_embed(x) for blk in block: x = blk(x) T, B, C, H, W = x.shape botlk_x = x.flatten(3) botlk_x = botlk_x.flatten(0, 1).permute(0, 2, 1) bottleneck_feature = self.bottleneck_initfeature.expand(T * B, 64, C) fusion_feature_agg = torch.cat((bottleneck_feature, botlk_x), 1) fusion_feature_agg = self.pos_drop(fusion_feature_agg + self.pos_embed) for bottleneck_blk in self.Bottleneck_block: fusion_feature_agg = bottleneck_blk(fusion_feature_agg) bottleneck_feature = fusion_feature_agg[:, 0:336, :].mean(1).reshape(B, 1, -1) return x.flatten(3).mean(3), bottleneck_feature def forward(self, event_tensor): B, N, C, H, W = event_tensor.shape event_tensor = event_tensor.permute(1, 0, 2, 3, 4) event_tensor, bottleneck_feature = self.forward_features(event_tensor) event_tensor = event_tensor.permute(1, 0, 2).reshape(B, -1) x1 = self.head(event_tensor) return x1以上修改中,
dpr列表的生成方式中torch.linspace()的第三个参数被修改为sum(depths),它将返回一个包含depths元素总数个数值的列表。这确保了drop_path_rate正确地按比例分配到每个块。请确保修改后的代码片段与
SpikingTokenizer和SpikingTransformer类适当地整合。另外,确保你正确地将vit_snn类实例化并用于你的主模型。如果有其他模型和逻辑依赖于这个参数调整,请一并检查和测试。 - User
只把修改的部分写出来
- Assistant
好的,只写出需要修改的部分。
# 修改 vit_snn 类中的以下几行 class vit_snn(nn.Module): def __init__(self, # ... 省略其他参数 depths=[6, 8, 6], # 确保这个默认值和你调整的相匹配 # ... 省略其他参数 ): # 省略其他代码 # 修改 linspace 函数的调用 dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] # 修改这行 # 下面的 block 定义部分也需要稍微调整,以适配不同深度的 block block = nn.ModuleList([ SpikingTransformer( dim=embed_dims[i], num_heads=num_heads[i], mlp_ratio=mlp_ratios[i], qkv_bias=qkv_bias, qk_scale=qk_scale, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[j], norm_layer=norm_layer, sr_ratio=sr_ratios[i]) for i in range(len(embed_dims)) for j in range(depths[i]) ]) # 省略其他代码说明
-
线性插值 (
linspace) 部分: 将原来torch.linspace(0, drop_path_rate, depths)修改为torch.linspace(0, drop_path_rate, sum(depths))。 -
block 的创建部分: 在创建
block的时候,外层循环for i in range(len(embed_dims))遍历embed_dims,内层循环for j in range(depths[i])遍历每个深度列表中的项。这样确保dpr列表中每个元素都正确分配到每个 Transformer 块。
这些修改会确保
dpr的元素数量与所有 Transformer 块的总数匹配,从而避免错误。 -
- User
报错:Traceback (most recent call last): File "lib/train/run_training.py", line 112, in main() File "lib/train/run_training.py", line 103, in main run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark, File "lib/train/run_training.py", line 76, in run_training expr_func(settings) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/train/train_script.py", line 55, in run net = build_ceutrack(cfg) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/ceutrack.py", line 124, in build_ceutrack backbone = vit_base_patch16_224_ce(pretrained, drop_path_rate=cfg.TRAIN.DROP_PATH_RATE, File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 193, in vit_base_patch16_224_ce model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 175, in _create_vision_transformer model = VisionTransformerCE(**kwargs) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 58, in init self.vit_snn_model = vit_snn(img_size_h=128, img_size_w=128, patch_size=16, in_channels=12, num_classes=11, File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_snn.py", line 202, in init patch_embed = SpikingTokenizer(img_size_h=img_size_h, File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_snn.py", line 139, in init self.proj_conv = nn.Conv2d(in_channels, embed_dims//8, kernel_size=3, stride=1, padding=1, bias=False) TypeError: unsupported operand type(s) for //: 'list' and 'int'
- Assistant
报错信息指出
embed_dims是一个列表,但在初始化SpikingTokenizer类时尝试对其进行整数除法 (//) 操作。这显然是不支持的,因为你不能对列表进行整数除法。你需要确保
embed_dims是单一整数值而不是列表,或者如果确实需要传递列表,则需要对代码进行相应的调整以处理此列表。在你的案例中,假设你希望
patch_embed、block中包含多个维度(一个维度为每个块),那么你需要分别处理这些维度。请将以下部分代码进行调整:
在
SpikingTokenizer中,确保embed_dims是单一整数在调用
SpikingTokenizer时传递正确的参数。在vit_snn构造函数中,请确保传递的是单个整数,而不是列表。class vit_snn(nn.Module): def __init__(self, img_size_h=128, img_size_w=128, patch_size=16, in_channels=2, num_classes=11, embed_dims=[64, 128, 256], num_heads=[1, 2, 4], mlp_ratios=[4, 4, 4], qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=nn.LayerNorm, depths=[6, 8, 6], sr_ratios=[8, 4, 2], T=4, pretrained_cfg=None): super().__init__() self.num_classes = num_classes self.depths = depths self.T = T # 合并 depth 以计算 drop path rate total_depth = sum(depths) dpr = [x.item() for x in torch.linspace(0, drop_path_rate, total_depth)] # 创建每个 stage 的 SpikingTokenizer 和 SpikingTransformer block self.patch_embeds = nn.ModuleList([SpikingTokenizer( img_size_h=img_size_h if i == 0 else img_size_h // (2 ** i), img_size_w=img_size_w if i == 0 else img_size_w // (2 ** i), patch_size=patch_size, in_channels=in_channels if i == 0 else embed_dims[i - 1], embed_dims=embed_dims[i] ) for i in range(len(embed_dims))]) self.blocks = nn.ModuleList([ nn.ModuleList([ SpikingTransformer( dim=embed_dims[i], num_heads=num_heads[i], mlp_ratio=mlp_ratios[i], qkv_bias=qkv_bias, qk_scale=qk_scale, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[idx], norm_layer=norm_layer, sr_ratio=sr_ratios[i] ) for idx in range(sum(depths[:i]), sum(depths[:i+1])) ]) for i in range(len(embed_dims)) ]) self.bottleneck_blocks = nn.ModuleList([ BottleneckTransformer( dim=embed_dims[-1], num_heads=4, mlp_ratio=mlp_ratios[-1], qkv_bias=qkv_bias, drop=0., attn_drop=0., init_values=None, drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm ) for _ in range(2) ]) # classification head self.head = nn.Linear(embed_dims[-1] * (img_size_h // (patch_size * 2 ** (len(embed_dims) - 1))) * (img_size_w // (patch_size * 2 ** (len(embed_dims) - 1))), num_classes) if num_classes > 0 else nn.Identity() self.apply(self._init_weights) self.bottleneck_initfeature = nn.Parameter(torch.randn(1, 64, embed_dims[-1])) kaiming_init(self.bottleneck_initfeature, mode='fan_out') self.pos_embed = nn.Parameter(torch.zeros(1, 400, embed_dims[-1])) self.pos_drop = nn.Dropout(p=0.) def _init_weights(self, m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.LayerNorm): nn.init.constant_(m.bias, 0) nn.init.constant_(m.weight, 1.0) def forward_features(self, x): for i, (patch_embed, blocks) in enumerate(zip(self.patch_embeds, self.blocks)): x, (H, W) = patch_embed(x) for block in blocks: x = block(x) x = x.flatten(3).mean(3) T, B, C = x.shape botlk_x = x.view(T * B, -1, C) bottleneck_feature = self.bottleneck_initfeature.expand(T * B, 64, -1) fusion_feature_agg = torch.cat((bottleneck_feature, botlk_x), 1) fusion_feature_agg = self.pos_drop(fusion_feature_agg + self.pos_embed) for bottleneck_blk in self.bottleneck_blocks: fusion_feature_agg = bottleneck_blk(fusion_feature_agg) bottleneck_feature = fusion_feature_agg[:, 0:336, :].mean(1).view(B, T, -1) return x.mean(1), bottleneck_feature def forward(self, event_tensor): B, N, C, H, W = event_tensor.shape event_tensor = event_tensor.permute(1, 0, 2, 3, 4) event_tensor, bottleneck_feature = self.forward_features(event_tensor) event_tensor = event_tensor.permute(1, 0, 2).reshape(B, -1) x1 = self.head(event_tensor) return x1说明
- 修改
dpr:使用sum(depths)计算总的阶段,并调整dpr的生成。 - 调整各阶段的
patch_embed和block:使用nn.ModuleList分别定义每个 Tokenizer 和 Transformer block。 - 注意: 确保
BottleneckTransformer和SpikingTransformer的参数与更改后的网络结构匹配。
通过这些修改,模型应会正确处理
embed_dims列表,并且每个阶段的patch_embed和block将适应不同的embed_dims。 - 修改