Skip to content
The Chat Archive
gpt-4o-2024-05-13· OpenAI WildChat-4.8M (AI2)

在import os import argparse import random def parseargs(): """ args for training. """…

  1. User

    在import os import argparse import random

    def parse_args(): """ args for training. """ parser = argparse.ArgumentParser(description='Parse args for training') # for train parser.add_argument('--script', type=str, help='training script name') parser.add_argument('--config', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--save_dir', type=str, help='root directory to save checkpoints, logs, and tensorboard') parser.add_argument('--mode', type=str, choices=["single", "multiple", "multi_node"], default="multiple", help="train on single gpu or multiple gpus") parser.add_argument('--nproc_per_node', type=int, help="number of GPUs per node") # specify when mode is multiple parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, help='training script name') parser.add_argument('--config_prv', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')

    # for multiple machines
    parser.add_argument('--rank', type=int, help='Rank of the current process.')
    parser.add_argument('--world-size', type=int, help='Number of processes participating in the job.')
    parser.add_argument('--ip', type=str, default='127.0.0.1', help='IP of the current rank 0.')
    parser.add_argument('--port', type=int, default='20000', help='Port of the current rank 0.')
    
    
    # 增加时间步
    parser.add_argument('--T', default=16, type=int, help='simulation steps')
    args = parser.parse_args()
    
    return args
    

    def main(): args = parse_args() if args.mode == "single": train_cmd = "python lib/train/run_training.py --script %s --config %s --save_dir %s --use_lmdb %d "
    "--script_prv %s --config_prv %s --distill %d --script_teacher %s --config_teacher %s --use_wandb %d"
    % (args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.distill, args.script_teacher, args.config_teacher, args.use_wandb) elif args.mode == "multiple": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_port %d lib/train/run_training.py "
    "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
    "--distill %d --script_teacher %s --config_teacher %s"
    % (args.nproc_per_node, random.randint(10000, 50000), args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) elif args.mode == "multi_node": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_addr %s --master_port %d --nnodes %d --node_rank %d lib/train/run_training.py "
    "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
    "--distill %d --script_teacher %s --config_teacher %s"
    % (args.nproc_per_node, args.ip, args.port, args.world_size, args.rank, args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) else: raise ValueError("mode should be 'single' or 'multiple'.") os.system(train_cmd)

    if name == "main": main() 中将T引入进来,但是需要在def main(): args = parse_args() if args.mode == "single": train_cmd = "python lib/train/run_training.py --script %s --config %s --save_dir %s --use_lmdb %d "
    "--script_prv %s --config_prv %s --distill %d --script_teacher %s --config_teacher %s --use_wandb %d"
    % (args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.distill, args.script_teacher, args.config_teacher, args.use_wandb) elif args.mode == "multiple": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_port %d lib/train/run_training.py "
    "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
    "--distill %d --script_teacher %s --config_teacher %s"
    % (args.nproc_per_node, random.randint(10000, 50000), args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) elif args.mode == "multi_node": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_addr %s --master_port %d --nnodes %d --node_rank %d lib/train/run_training.py "
    "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
    "--distill %d --script_teacher %s --config_teacher %s"
    % (args.nproc_per_node, args.ip, args.port, args.world_size, args.rank, args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) else: raise ValueError("mode should be 'single' or 'multiple'.") os.system(train_cmd) 中将T加进去。

  2. Assistant

    为了将参数 T 引入训练命令中,可以在三个不同模式的命令中都加入 --T %d。具体如下:

    import os
    import argparse
    import random
    
    
    def parse_args():
        """
        args for training.
        """
        parser = argparse.ArgumentParser(description='Parse args for training')
        # for train
        parser.add_argument('--script', type=str, help='training script name')
        parser.add_argument('--config', type=str, default='baseline', help='yaml configure file name')
        parser.add_argument('--save_dir', type=str, help='root directory to save checkpoints, logs, and tensorboard')
        parser.add_argument('--mode', type=str, choices=["single", "multiple", "multi_node"], default="multiple",
                            help="train on single gpu or multiple gpus")
        parser.add_argument('--nproc_per_node', type=int, help="number of GPUs per node")  # specify when mode is multiple
        parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0)  # whether datasets are in lmdb format
        parser.add_argument('--script_prv', type=str, help='training script name')
        parser.add_argument('--config_prv', type=str, default='baseline', help='yaml configure file name')
        parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0)  # whether to use wandb
        # for knowledge distillation
        parser.add_argument('--distill', type=int, choices=[0, 1], default=0)  # whether to use knowledge distillation
        parser.add_argument('--script_teacher', type=str, help='teacher script name')
        parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')
    
        # for multiple machines
        parser.add_argument('--rank', type=int, help='Rank of the current process.')
        parser.add_argument('--world-size', type=int, help='Number of processes participating in the job.')
        parser.add_argument('--ip', type=str, default='127.0.0.1', help='IP of the current rank 0.')
        parser.add_argument('--port', type=int, default='20000', help='Port of the current rank 0.')
    
        
        # 增加时间步
        parser.add_argument('--T', default=16, type=int, help='simulation steps')
        args = parser.parse_args()
    
        return args
    
    
    def main():
        args = parse_args()
        if args.mode == "single":
            train_cmd = "python lib/train/run_training.py --script %s --config %s --save_dir %s --use_lmdb %d " \
                        "--script_prv %s --config_prv %s --distill %d --script_teacher %s --config_teacher %s --use_wandb %d --T %d" \
                        % (args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv,
                           args.distill, args.script_teacher, args.config_teacher, args.use_wandb, args.T)
        elif args.mode == "multiple":
            train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_port %d lib/train/run_training.py " \
                        "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d " \
                        "--distill %d --script_teacher %s --config_teacher %s --T %d" \
                        % (args.nproc_per_node, random.randint(10000, 50000), args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb,
                           args.distill, args.script_teacher, args.config_teacher, args.T)
        elif args.mode == "multi_node":
            train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_addr %s --master_port %d --nnodes %d --node_rank %d lib/train/run_training.py " \
                        "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d " \
                        "--distill %d --script_teacher %s --config_teacher %s --T %d" \
                        % (args.nproc_per_node, args.ip, args.port, args.world_size, args.rank, args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb,
                           args.distill, args.script_teacher, args.config_teacher, args.T)
        else:
            raise ValueError("mode should be 'single', 'multiple' or 'multi_node'.")
        os.system(train_cmd)
    
    
    if __name__ == "__main__":
        main()
    

    这样,你的时间步参数 T 就会在调用训练命令时传递给 Python 脚本。确保你的 lib/train/run_training.py 脚本能够处理此参数。

  3. User

    训练报错:usage: run_training.py [-h] --script SCRIPT --config CONFIG [--cudnn_benchmark CUDNN_BENCHMARK] [--local_rank LOCAL_RANK] [--save_dir SAVE_DIR] [--seed SEED] [--use_lmdb {0,1}] [--script_prv SCRIPT_PRV] [--config_prv CONFIG_PRV] [--use_wandb {0,1}] [--distill {0,1}] [--script_teacher SCRIPT_TEACHER] [--config_teacher CONFIG_TEACHER] run_training.py: error: unrecognized arguments: --T 16,其中import os import sys import argparse import importlib import cv2 as cv import torch.backends.cudnn import torch.distributed as dist

    import random import numpy as np torch.backends.cudnn.benchmark = False

    import _init_paths import lib.train.admin.settings as ws_settings

    def init_seeds(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False

    def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None, use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False, distill=None, script_teacher=None, config_teacher=None): """Run the train script. args: script_name: Name of emperiment in the "experiments/" folder. config_name: Name of the yaml file in the "experiments/<script_name>". cudnn_benchmark: Use cudnn benchmark or not (default is True). """ if save_dir is None: print("save_dir dir is not given. Use the default dir instead.") # This is needed to avoid strange crashes related to opencv cv.setNumThreads(0)

    torch.backends.cudnn.benchmark = cudnn_benchmark
    
    print('script_name: {}.py  config_name: {}.yaml'.format(script_name, config_name))
    
    '''2021.1.5 set seed for different process'''
    if base_seed is not None:
        if local_rank != -1:
            init_seeds(base_seed + local_rank)
        else:
            init_seeds(base_seed)
    
    settings = ws_settings.Settings()
    settings.script_name = script_name
    settings.config_name = config_name
    settings.project_path = 'train/{}/{}'.format(script_name, config_name)
    if script_name_prv is not None and config_name_prv is not None:
        settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv)
    settings.local_rank = local_rank
    settings.save_dir = os.path.abspath(save_dir)
    settings.use_lmdb = use_lmdb
    prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
    settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name))
    settings.use_wandb = use_wandb
    if distill:
        settings.distill = distill
        settings.script_teacher = script_teacher
        settings.config_teacher = config_teacher
        if script_teacher is not None and config_teacher is not None:
            settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher)
        settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher))
        expr_module = importlib.import_module('lib.train.train_script_distill')
    else:
        expr_module = importlib.import_module('lib.train.train_script')
    expr_func = getattr(expr_module, 'run')
    
    expr_func(settings)
    

    def main(): parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.') parser.add_argument('--script', type=str, required=True, help='Name of the train script.') parser.add_argument('--config', type=str, required=True, help="Name of the config file.") parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).') parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training') parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs') parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42') parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.') parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.") parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')

    args = parser.parse_args()
    if args.local_rank != -1:
        dist.init_process_group(backend='nccl')
        torch.cuda.set_device(args.local_rank)
    else:
        torch.cuda.set_device(0)
    run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark,
                 local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed,
                 use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv,
                 use_wandb=args.use_wandb,
                 distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher)
    

    if name == 'main': # os.environ['CUDA_VISIBLE_DEVICES'] = "4" main(),应该如何修改?

  4. Assistant

    你遇到的错误是因为 T 参数没有在 run_training.py 中定义。所以,当你传递 T 参数时,Python argparse 无法识别它。

    你需要在 run_training.py 中添加对 T 参数的支持。以下为如何进行相应修改的步骤:

    1. 在 run_training.py 中添加 T 参数:

      在 def main() 函数中的 argparse 定义部分添加 T 参数。

    2. 在 run_training 函数中传递 T 参数:

      你需要确保将 T 参数传递到你的 run_training 函数中。

    以下为修改过的 run_training.py:

    import os
    import sys
    import argparse
    import importlib
    import cv2 as cv
    import torch.backends.cudnn
    import torch.distributed as dist
    
    import random
    import numpy as np
    torch.backends.cudnn.benchmark = False
    
    import _init_paths
    import lib.train.admin.settings as ws_settings
    
    
    def init_seeds(seed):
        random.seed(seed)
        np.random.seed(seed)
        torch.manual_seed(seed)
        torch.cuda.manual_seed(seed)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
    
    
    def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None,
                     use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False,
                     distill=None, script_teacher=None, config_teacher=None, T=16):  # Add T parameter here
        """Run the train script.
        args:
            script_name: Name of emperiment in the "experiments/" folder.
            config_name: Name of the yaml file in the "experiments/<script_name>".
            cudnn_benchmark: Use cudnn benchmark or not (default is True).
        """
        if save_dir is None:
            print("save_dir dir is not given. Use the default dir instead.")
        # This is needed to avoid strange crashes related to opencv
        cv.setNumThreads(0)
    
        torch.backends.cudnn.benchmark = cudnn_benchmark
    
        print('script_name: {}.py  config_name: {}.yaml'.format(script_name, config_name))
    
        '''2021.1.5 set seed for different process'''
        if base_seed is not None:
            if local_rank != -1:
                init_seeds(base_seed + local_rank)
            else:
                init_seeds(base_seed)
    
        settings = ws_settings.Settings()
        settings.script_name = script_name
        settings.config_name = config_name
        settings.project_path = 'train/{}/{}'.format(script_name, config_name)
        if script_name_prv is not None and config_name_prv is not None:
            settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv)
        settings.local_rank = local_rank
        settings.save_dir = os.path.abspath(save_dir)
        settings.use_lmdb = use_lmdb
        prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
        settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name))
        settings.use_wandb = use_wandb
        settings.T = T  # Pass T parameter to settings
        if distill:
            settings.distill = distill
            settings.script_teacher = script_teacher
            settings.config_teacher = config_teacher
            if script_teacher is not None and config_teacher is not None:
                settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher)
            settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher))
            expr_module = importlib.import_module('lib.train.train_script_distill')
        else:
            expr_module = importlib.import_module('lib.train.train_script')
        expr_func = getattr(expr_module, 'run')
    
        expr_func(settings)
    
    
    def main():
        parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.')
        parser.add_argument('--script', type=str, required=True, help='Name of the train script.')
        parser.add_argument('--config', type=str, required=True, help="Name of the config file.")
        parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).')
        parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training')
        parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs')
        parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42')
        parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0)  # whether datasets are in lmdb format
        parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.')
        parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.")
        parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0)  # whether to use wandb
        # for knowledge distillation
        parser.add_argument('--distill', type=int, choices=[0, 1], default=0)  # whether to use knowledge distillation
        parser.add_argument('--script_teacher', type=str, help='teacher script name')
        parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')
        parser.add_argument('--T', type=int, default=16, help='simulation steps')  # Add T parameter here
    
        args = parser.parse_args()
        if args.local_rank != -1:
            dist.init_process_group(backend='nccl')
            torch.cuda.set_device(args.local_rank)
        else:
            torch.cuda.set_device(0)
        run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark,
                     local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed,
                     use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv,
                     use_wandb=args.use_wandb,
                     distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher,
                     T=args.T)  # Pass T to run_training
    
    
    if __name__ == '__main__':
        # os.environ['CUDA_VISIBLE_DEVICES'] = "4"
        main()
    
  5. User

    在import os import os.path import numpy as np import torch import csv import pandas import random from collections import OrderedDict from .base_video_dataset import BaseVideoDataset from lib.train.data import jpeg4py_loader from lib.train.admin import env_settings import scipy.io as scio

    class Coesot(BaseVideoDataset): def init(self, root=None, image_loader=jpeg4py_loader, split=None, seq_ids=None, data_fraction=None):

        root = env_settings().got10k_dir if root is None else root
        super().__init__('Coesot', root, image_loader)
    
        self.sequence_list = self._get_sequence_list()
    
        # seq_id is the index of the folder inside the got10k root path
        if split is not None:
            if seq_ids is not None:
                raise ValueError('Cannot set both split_name and seq_ids.')
            if split == 'train':
                file_path = os.path.join(self.root, 'train.txt')
            elif split == 'val':
                file_path = os.path.join(self.root, 'val.txt')
            else:
                raise ValueError('Unknown split name')
            seq_ids = pandas.read_csv(file_path, header=None, dtype=np.int64).squeeze("columns").values.tolist()
        elif seq_ids is None:
            seq_ids = list(range(0, len(self.sequence_list)))
    
        self.sequence_list = [self.sequence_list[i] for i in seq_ids]
    
    def get_name(self):
        return 'coesot'
    
    def _get_sequence_list(self):
        with open(os.path.join(self.root, 'list.txt')) as f:
            dir_list = list(csv.reader(f))
        dir_list = [dir_name[0] for dir_name in dir_list]
        return dir_list
    
    def _read_bb_anno(self, seq_path):
        bb_anno_file = os.path.join(seq_path, "groundtruth.txt")
        gt = pandas.read_csv(bb_anno_file, delimiter=',', header=None, dtype=np.float32, na_filter=False, low_memory=False).values
        return torch.tensor(gt)
    
    def _get_sequence_path(self, seq_id):
        return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_aps") # RGB模态
    
    def _get_event_img_sequence_path(self, seq_id): 
        return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_dvs") # 事件模态 - 表征方式 event frame 
        # return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_timesurface") # 事件模态 - 表征方式 event timesurface 
        # return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_recon") # 事件模态 - 表征方式 event timesurface 
    def _get_grountgruth_path(self, seq_id):
        return os.path.join(self.root, self.sequence_list[seq_id])
    
    def get_sequence_info(self, seq_id):
        bbox_path = self._get_grountgruth_path(seq_id)
        bbox = self._read_bb_anno(bbox_path)
    
        valid = (bbox[:, 2] > 0) & (bbox[:, 3] > 0)
        visible = valid.clone().byte()
        # return {'bbox': bbox, 'valid': valid, 'visible': visible, 'visible_ratio': visible_ratio}
        return {'bbox': bbox, 'valid': valid, 'visible': visible, }
    
    def _get_frame_path(self, seq_path, frame_id):
        if os.path.exists(os.path.join(seq_path, 'frame{:04}.png'.format(frame_id))):
            return os.path.join(seq_path, 'frame{:04}.png'.format(frame_id))    # frames start from 0
        else:
            return os.path.join(seq_path, 'frame{:04}.bmp'.format(frame_id))    # some image is bmp
    
    def _get_frame(self, seq_path, frame_id):
        return self.image_loader(self._get_frame_path(seq_path, frame_id))
    
    def _get_event_sequence_path(self, seq_id):        ## get evemts' frames
        return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_voxel") # 事件模态 - 表征方式 event voxel
    
    def _get_event_frame(self, seq_path, frame_id):
        frame_event_list = []
        for f_id in frame_id:
            event_frame_file = os.path.join(seq_path, 'frame{:04}.mat'.format(f_id))
            if os.path.getsize(event_frame_file) == 0:
                event_features = np.zeros(4096, 19)
                # need_data = [np.zeros([4096, 3]), np.zeros([4096, 16])]
            else:
                mat_data = scio.loadmat(event_frame_file)
                # need_data = [mat_data['coor'], mat_data['features']]
                event_features = np.concatenate((mat_data['coor'], mat_data['features']), axis=1)        # concat coorelate and features (x,y,z, feauture32/16)
                if np.isnan(event_features).any():
                    event_features = np.zeros(4096, 19)
                    print(event_frame_file, 'exist nan value in voxel.')
            frame_event_list.append(event_features)
        return frame_event_list
    
    def get_frames(self, seq_id, frame_ids, anno=None):
        # RGB模态
        seq_path = self._get_sequence_path(seq_id)
        # obj_meta = self.sequence_meta_info[self.sequence_list[seq_id]]
    
        frame_list = [self._get_frame(seq_path, f_id) for f_id in frame_ids]
        # 事件模态
        seq_event_path = self._get_event_img_sequence_path(seq_id)
        frame_event_img_list = [self._get_frame(seq_event_path, f_id) for f_id in frame_ids]
        if anno is None:
            anno = self.get_sequence_info(seq_id)
    
        anno_frames = {}
        for key, value in anno.items():
            anno_frames[key] = [value[f_id, ...].clone() for f_id in frame_ids]
    
        object_meta = OrderedDict({'object_class_name': None,
                                   'motion_class': None,
                                   'major_class': None,
                                   'root_class': None,
                                   'motion_adverb': None})
        # 读取 event voxel 
        # seq_event_path = self._get_event_sequence_path(seq_id)
        # frame_event_list = self._get_event_frame(seq_event_path, frame_ids)
        # 现在的目的是:将voxel直接换掉 → event frame(即dvs格式)
        frame_event_list = frame_event_img_list # 读入的就是 dvs下的新的表征形式了
        return frame_list, anno_frames, object_meta, frame_event_list, frame_event_img_list
    

    中我们读取的_dvs目录下文件都是frame0000.bmp或png,现在我们需要读取的是_tensor目录,然后该目录下的文件是 frame0000.npz格式,那么现在应该如何修改?

  6. Assistant

    为了实现从 _tensor 目录读取 .npz 格式的文件,需要对现有代码进行以下几项修改:

    1. 新增方法 _get_tensor_sequence_path 用于获取 _tensor 目录的路径。
    2. 新增方法 _get_tensor_frame 用于读取 .npz 文件。
    3. 修改现有代码中的 _get_event_img_sequence_path 和 _get_event_frame 方法以适配新的 .npz 格式读取路径和文件格式。

    下面是修改后的代码:

    import os
    import os.path
    import numpy as np
    import torch
    import csv
    import pandas
    import random
    from collections import OrderedDict
    from .base_video_dataset import BaseVideoDataset
    from lib.train.data import jpeg4py_loader
    from lib.train.admin import env_settings
    import scipy.io as scio
    
    class Coesot(BaseVideoDataset):
        def __init__(self, root=None, image_loader=jpeg4py_loader, split=None, seq_ids=None, data_fraction=None):
    
            root = env_settings().got10k_dir if root is None else root
            super().__init__('Coesot', root, image_loader)
    
            self.sequence_list = self._get_sequence_list()
    
            # seq_id is the index of the folder inside the got10k root path
            if split is not None:
                if seq_ids is not None:
                    raise ValueError('Cannot set both split_name and seq_ids.')
                if split == 'train':
                    file_path = os.path.join(self.root, 'train.txt')
                elif split == 'val':
                    file_path = os.path.join(self.root, 'val.txt')
                else:
                    raise ValueError('Unknown split name')
                seq_ids = pandas.read_csv(file_path, header=None, dtype=np.int64).squeeze("columns").values.tolist()
            elif seq_ids is None:
                seq_ids = list(range(0, len(self.sequence_list)))
    
            self.sequence_list = [self.sequence_list[i] for i in seq_ids]
    
        def get_name(self):
            return 'coesot'
    
        def _get_sequence_list(self):
            with open(os.path.join(self.root, 'list.txt')) as f:
                dir_list = list(csv.reader(f))
            dir_list = [dir_name[0] for dir_name in dir_list]
            return dir_list
    
        def _read_bb_anno(self, seq_path):
            bb_anno_file = os.path.join(seq_path, "groundtruth.txt")
            gt = pandas.read_csv(bb_anno_file, delimiter=',', header=None, dtype=np.float32, na_filter=False, low_memory=False).values
            return torch.tensor(gt)
    
        def _get_sequence_path(self, seq_id):
            return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_aps") # RGB模态
    
        def _get_event_img_sequence_path(self, seq_id): 
            return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_dvs") # 事件模态 - 表征方式 event frame 
    
        def _get_tensor_sequence_path(self, seq_id): 
            return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_tensor") # 新的事件模态 - 表征方式 event tensor
    
        def _get_grountgruth_path(self, seq_id):
            return os.path.join(self.root, self.sequence_list[seq_id])
    
        def get_sequence_info(self, seq_id):
            bbox_path = self._get_grountgruth_path(seq_id)
            bbox = self._read_bb_anno(bbox_path)
    
            valid = (bbox[:, 2] > 0) & (bbox[:, 3] > 0)
            visible = valid.clone().byte()
            # return {'bbox': bbox, 'valid': valid, 'visible': visible, 'visible_ratio': visible_ratio}
            return {'bbox': bbox, 'valid': valid, 'visible': visible, }
    
        def _get_frame_path(self, seq_path, frame_id):
            if os.path.exists(os.path.join(seq_path, 'frame{:04}.png'.format(frame_id))):
                return os.path.join(seq_path, 'frame{:04}.png'.format(frame_id))    # frames start from 0
            else:
                return os.path.join(seq_path, 'frame{:04}.bmp'.format(frame_id))    # some image is bmp
    
        def _get_frame(self, seq_path, frame_id):
            return self.image_loader(self._get_frame_path(seq_path, frame_id))
    
        def _get_event_sequence_path(self, seq_id):        ## get evemts' frames
            return os.path.join(self.root, self.sequence_list[seq_id], self.sequence_list[seq_id] + "_voxel") # 事件模态 - 表征方式 event voxel
    
        def _get_event_frame(self, seq_path, frame_id):
            frame_event_list = []
            for f_id in frame_id:
                event_frame_file = os.path.join(seq_path, 'frame{:04}.mat'.format(f_id))
                if os.path.getsize(event_frame_file) == 0:
                    event_features = np.zeros((4096, 19))
                    # need_data = [np.zeros([4096, 3]), np.zeros([4096, 16])]
                else:
                    mat_data = scio.loadmat(event_frame_file)
                    # need_data = [mat_data['coor'], mat_data['features']]
                    event_features = np.concatenate((mat_data['coor'], mat_data['features']), axis=1)        # concat coorelate and features (x,y,z, feauture32/16)
                    if np.isnan(event_features).any():
                        event_features = np.zeros((4096, 19))
                        print(event_frame_file, 'exist nan value in voxel.')
                frame_event_list.append(event_features)
            return frame_event_list
    
        def _get_tensor_frame(self, seq_path, frame_id):
            frame_tensor_list = []
            for f_id in frame_id:
                tensor_frame_file = os.path.join(seq_path, 'frame{:04}.npz'.format(f_id))
                if os.path.exists(tensor_frame_file):
                    data = np.load(tensor_frame_file)
                    frame_tensor_list.append(data)
                else:
                    print(f"File {tensor_frame_file} does not exist.")
                    frame_tensor_list.append(None)
            return frame_tensor_list
    
        def get_frames(self, seq_id, frame_ids, anno=None):
            # RGB模态
            seq_path = self._get_sequence_path(seq_id)
            # obj_meta = self.sequence_meta_info[self.sequence_list[seq_id]]
    
            frame_list = [self._get_frame(seq_path, f_id) for f_id in frame_ids]
            # 事件模态
            seq_event_path = self._get_event_img_sequence_path(seq_id)
            frame_event_img_list = [self._get_frame(seq_event_path, f_id) for f_id in frame_ids]
            if anno is None:
                anno = self.get_sequence_info(seq_id)
    
            anno_frames = {}
            for key, value in anno.items():
                anno_frames[key] = [value[f_id, ...].clone() for f_id in frame_ids]
    
            object_meta = OrderedDict({'object_class_name': None,
                                       'motion_class': None,
                                       'major_class': None,
                                       'root_class': None,
                                       'motion_adverb': None})
            # 读取 event tensor
            seq_tensor_path = self._get_tensor_sequence_path(seq_id)
            frame_tensor_list = self._get_tensor_frame(seq_tensor_path, frame_ids)
            return frame_list, anno_frames, object_meta, frame_tensor_list, frame_event_img_list
    

    在这个修改后的代码中,新添的方法 _get_tensor_sequence_path 和 _get_tensor_frame 专门处理 _tensor 目录和读取 .npz 文件。get_frames 方法的返回值中也包含了 frame_tensor_list,即从 _tensor 目录读取的 .npz 数据。这样,代码便能够读取指定目录下的 .npz 文件并进行处理。

  7. User

    pip install wandb清华镜像源

  8. Assistant

    你可以通过使用国内镜像源来安装 pip 包以加速下载。对于清华镜像源,你可以使用如下命令来安装 wandb 包:

    pip install -i https://pypi.tuna.tsinghua.edu.cn/simple wandb
    

    上述命令指定了 pip 使用清华大学的镜像源来进行包的安装。

    备用镜像源

    如果清华的镜像源不可用或速度不理想,你也可以尝试其他国内常用的镜像源,例如:

    1. 阿里云镜像源:

      pip install -i https://mirrors.aliyun.com/pypi/simple wandb
      
    2. 中国科学技术大学镜像源:

      pip install -i https://pypi.mirrors.ustc.edu.cn/simple wandb
      
    3. 豆瓣镜像源:

      pip install -i https://pypi.douban.com/simple wandb
      

    配置全局镜像源

    如果你希望每次使用 pip 时都默认使用某个镜像源,可以配置 pip 的全局镜像,一劳永逸。

    1. 编辑或创建 pip.conf (Linux 或 macOS) 或 pip.ini (Windows):

      • Linux/macOS:

        mkdir -p ~/.pip
        echo "[global]
        index-url = https://pypi.tuna.tsinghua.edu.cn/simple" > ~/.pip/pip.conf
        
      • Windows: 在 C:\Users\你的用户名\ 目录下创建一个名为 pip 的文件夹,然后在里面创建一个 pip.ini 文件,内容如下:

        [global]
        index-url = https://pypi.tuna.tsinghua.edu.cn/simple
        

    这样,你在使用 pip 安装包时,就会默认使用指定的镜像源。希望对你有所帮助!

  9. User

    网络是:import torch import torch.nn as nn from spikingjelly.clock_driven.neuron import MultiStepParametricLIFNode, MultiStepLIFNode from spikingjelly.clock_driven import layer from timm.models.layers import to_2tuple, trunc_normal_, DropPath from timm.models.registry import register_model from timm.models.vision_transformer import _cfg from einops.layers.torch import Rearrange import torch.nn.functional as F from functools import partial from timm.models import create_model import pdb from .Bottleneck import BottleneckTransformer

    from mmengine.model.weight_init import kaiming_init

    from mmcv.cnn import kaiming_init

    all = ['Spikingformer'] tau_thr = 1.5

    class MLP(nn.Module): def init(self, in_features, hidden_features=None, out_features=None, drop=0.): super().init() out_features = out_features or in_features hidden_features = hidden_features or in_features self.mlp1_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True) #, backend='cupy' self.mlp1_conv = nn.Conv2d(in_features, hidden_features, kernel_size=1, stride=1) self.mlp1_bn = nn.BatchNorm2d(hidden_features)

        self.mlp2_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
        self.mlp2_conv = nn.Conv2d(hidden_features, out_features, kernel_size=1, stride=1)
        self.mlp2_bn = nn.BatchNorm2d(out_features)
    
        self.c_hidden = hidden_features
        self.c_output = out_features
    
    def forward(self, x):
        T, B, C, H, W = x.shape
    
        x = self.mlp1_lif(x)
        x = self.mlp1_conv(x.flatten(0, 1))
        x = self.mlp1_bn(x).reshape(T, B, self.c_hidden, H, W)
    
        x = self.mlp2_lif(x)
        x = self.mlp2_conv(x.flatten(0, 1))
        x = self.mlp2_bn(x).reshape(T, B, C, H, W)
        return x
    

    class SpikingSelfAttention(nn.Module): def init(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., sr_ratio=1): super().init() assert dim % num_heads == 0, f"dim {dim} should be divided by num_heads {num_heads}."

        self.dim = dim
        self.num_heads = num_heads
    
        self.proj_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
    
        self.q_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1, bias=False)
        self.q_bn = nn.BatchNorm1d(dim)
        self.q_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
    
        self.k_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1, bias=False)
        self.k_bn = nn.BatchNorm1d(dim)
        self.k_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
    
        self.v_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1, bias=False)
        self.v_bn = nn.BatchNorm1d(dim)
        self.v_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
    
        self.attn_lif = MultiStepLIFNode(tau=tau_thr, v_threshold=0.5, detach_reset=True)
    
        self.proj_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1)
        self.proj_bn = nn.BatchNorm1d(dim)
    
    
    def forward(self, x):
        T, B, C, H, W = x.shape 
        x = self.proj_lif(x)
    
        x = x.flatten(3)   
        T, B, C, N = x.shape
        x_for_qkv = x.flatten(0, 1) 
    
        q_conv_out = self.q_conv(x_for_qkv)
        q_conv_out = self.q_bn(q_conv_out).reshape(T, B, C, N)
        q_conv_out = self.q_lif(q_conv_out)
        q = q_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C // self.num_heads).permute(0, 1, 3, 2, 4) #torch.Size([4, 2, 16, 336, 16])
    
        k_conv_out = self.k_conv(x_for_qkv)
        k_conv_out = self.k_bn(k_conv_out).reshape(T, B, C, N)
        k_conv_out = self.k_lif(k_conv_out)
        k = k_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C // self.num_heads).permute(0, 1, 3, 2, 4)#torch.Size([4, 2, 16, 336, 16])
    
        v_conv_out = self.v_conv(x_for_qkv)
        v_conv_out = self.v_bn(v_conv_out).reshape(T, B, C, N)
        v_conv_out = self.v_lif(v_conv_out)
        v = v_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C // self.num_heads).permute(0, 1, 3, 2, 4)  #torch.Size([4, 2, 16, 336, 16])
    
        attn = (q @ k.transpose(-2, -1))  #torch.Size([4, 2, 16, 336, 336])
        x = (attn @ v) * 0.125  #torch.Size([4, 2, 16, 336, 16])
    
        x = x.transpose(3, 4).reshape(T, B, C, N)
        x = self.attn_lif(x)
        x = x.flatten(0, 1)
        x = self.proj_bn(self.proj_conv(x)).reshape(T, B, C, H, W)  #torch.Size([4, 2, 256, 21, 16])
    
        return x
    

    class SpikingTransformer(nn.Module): def init(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., drop_path=0., norm_layer=nn.LayerNorm, sr_ratio=1): super().init() self.norm1 = norm_layer(dim) self.attn = SpikingSelfAttention(dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop, sr_ratio=sr_ratio) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() self.norm2 = norm_layer(dim) mlp_hidden_dim = int(dim * mlp_ratio) self.mlp = MLP(in_features=dim, hidden_features=mlp_hidden_dim, drop=drop)

    def forward(self, x):
        x = x + self.attn(x)
        x = x + self.mlp(x)
        return x
    

    class SpikingTokenizer(nn.Module): def init(self, img_size_h=128, img_size_w=128, patch_size=4, in_channels=2, embed_dims=256): super().init() self.image_size = [img_size_h, img_size_w] patch_size = to_2tuple(patch_size) self.patch_size = patch_size self.C = in_channels self.H, self.W = self.image_size[0] // patch_size[0], self.image_size[1] // patch_size[1] self.num_patches = self.H * self.W

        self.proj_conv = nn.Conv2d(in_channels, embed_dims//8, kernel_size=3, stride=1, padding=1, bias=False)
        self.proj_bn = nn.BatchNorm2d(embed_dims//8)
    
        self.proj1_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
        self.maxpool1 = torch.nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
        self.proj1_conv = nn.Conv2d(embed_dims//8, embed_dims//4, kernel_size=3, stride=1, padding=1, bias=False)
        self.proj1_bn = nn.BatchNorm2d(embed_dims//4)
    
        self.proj2_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
        self.maxpool2 = torch.nn.MaxPool2d(kernel_size=(3,2), stride=2, padding=0, dilation=1, ceil_mode=False)
        self.proj2_conv = nn.Conv2d(embed_dims//4, embed_dims//2, kernel_size=3, stride=1, padding=1, bias=False)
        self.proj2_bn = nn.BatchNorm2d(embed_dims//2)
    
        self.proj3_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
        self.maxpool3 = torch.nn.MaxPool2d(kernel_size=(2,3), stride=2, padding=0, dilation=1, ceil_mode=False)
        self.proj3_conv = nn.Conv2d(embed_dims//2, embed_dims, kernel_size=3, stride=1, padding=1, bias=False)
        self.proj3_bn = nn.BatchNorm2d(embed_dims)
    
        self.proj4_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
        self.maxpool4 = torch.nn.MaxPool2d(kernel_size=(3,2), stride=2, padding=0, dilation=1, ceil_mode=False)
        self.proj4_conv = nn.Conv2d(embed_dims, embed_dims, kernel_size=3, stride=1, padding=1, bias=False)
        self.proj4_bn = nn.BatchNorm2d(embed_dims)
    
    def forward(self, x):
        T, B, C, H, W = x.shape  
        x = self.proj_conv(x.flatten(0, 1)) 
        x = self.proj_bn(x).reshape(T,B,-1,H,W).contiguous()  
    
        x = self.proj1_lif(x).flatten(0,1).contiguous()  
        x = self.maxpool1(x)                             
        x = self.proj1_conv(x)                           
        x = self.proj1_bn(x).reshape(T, B, -1, H//2, W//2).contiguous() 
    
        x = self.proj2_lif(x).flatten(0, 1).contiguous() 
        x = self.maxpool2(x)                               
        x = self.proj2_conv(x)                             
        x = self.proj2_bn(x).reshape(T, B, -1, H//4, W//4).contiguous() 
    
        x = self.proj3_lif(x).flatten(0, 1).contiguous()    
        x = self.maxpool3(x)                            
        x = self.proj3_conv(x)                          
        x = self.proj3_bn(x).reshape(T, B, -1, H//8, W//8).contiguous() 
    
        x = self.proj4_lif(x).flatten(0, 1).contiguous() 
        x = self.maxpool4(x)                            
        x = self.proj4_conv(x)                          
        x = self.proj4_bn(x).reshape(T, B, -1, H//16, W//16).contiguous() 
        return x, (None, None)
    

    class vit_snn(nn.Module): def init(self, img_size_h=128, img_size_w=128, patch_size=16, in_channels=2, num_classes=11, embed_dims=[64, 128, 256], num_heads=[1, 2, 4], mlp_ratios=[4, 4, 4], qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=nn.LayerNorm, depths=[6, 8, 6], sr_ratios=[8, 4, 2], T=4, pretrained_cfg=None, ): super().init() self.num_classes = num_classes self.depths = depths self.T = T dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depths)]

        patch_embed = SpikingTokenizer(img_size_h=img_size_h,
                          img_size_w=img_size_w,
                          patch_size=patch_size,
                          in_channels=in_channels,
                          embed_dims=embed_dims)
        num_patches = patch_embed.num_patches
        block = nn.ModuleList([SpikingTransformer(
            dim=embed_dims, num_heads=num_heads, mlp_ratio=mlp_ratios, qkv_bias=qkv_bias,
            qk_scale=qk_scale, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[j],
            norm_layer=norm_layer, sr_ratio=sr_ratios)
            for j in range(depths)])
    
        self.Bottleneck_block = nn.ModuleList([BottleneckTransformer(
            dim=embed_dims, num_heads=4, mlp_ratio=mlp_ratios,
            qkv_bias=qkv_bias, drop=0., attn_drop=0., init_values=None,
            drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm)
            for j in range(2)])
    
        setattr(self, f"patch_embed", patch_embed)
        setattr(self, f"block", block)
    
        # classification head
        self.head = nn.Linear(8192, num_classes) if num_classes > 0 else nn.Identity()
        self.apply(self._init_weights)
        
        self.bottleneck_initfeature = nn.Parameter(torch.randn(1, 64, 256))
        kaiming_init(self.bottleneck_initfeature, mode='fan_out')  
        self.pos_embed = nn.Parameter(torch.zeros(1, 400, 256))
        self.pos_drop = nn.Dropout(p=0.)
    
    def _init_weights(self, m):
        if isinstance(m, nn.Linear):
            trunc_normal_(m.weight, std=.02)
            if isinstance(m, nn.Linear) and m.bias is not None:
                nn.init.constant_(m.bias, 0)
        elif isinstance(m, nn.LayerNorm):
            nn.init.constant_(m.bias, 0)
            nn.init.constant_(m.weight, 1.0)
    
    def forward_features(self, x):
        block = getattr(self, f"block")
        patch_embed = getattr(self, f"patch_embed")
    
        x, (H, W) = patch_embed(x)   #x--torch.Size([4, 2, 256, 21, 16])
        for blk in block:
            x = blk(x)     #torch.Size([16, 8, 256, 21, 16])     #torch.Size([4, 2, 256, 21, 16])  
        T,B,C,H,W =x.shape
    
        botlk_x = x.flatten(3)   #torch.Size([4, 2, 256, 336])   T, B, C, N
        botlk_x = botlk_x.flatten(0, 1).permute(0,2,1)  #torch.Size([8, 336, 256])
    
        bottleneck_feature = self.bottleneck_initfeature.expand(T*B,64,C)
        fusion_feature_agg = torch.cat((bottleneck_feature,botlk_x),1)  #BT, 672,256
        fusion_feature_agg = self.pos_drop(fusion_feature_agg + self.pos_embed)  # 
    
        for bottleneck_blk in self.Bottleneck_block:
            fusion_feature_agg = bottleneck_blk(fusion_feature_agg)
        bottleneck_feature = fusion_feature_agg[:,0:336,:].mean(1).reshape(B,1,-1) #torch.Size([16, 2, 336, 256])
        return x.flatten(3).mean(3), bottleneck_feature   #torch.Size([4, 2, 256])
    
    def forward(self, event_tensor):
        B, N, C, H, W = event_tensor.shape
        event_tensor = event_tensor.permute(1, 0, 2, 3, 4)  # [T, N, 2, *, *]    torch.Size([16, 2, 12, 346, 260])
        event_tensor, bottleneck_feature = self.forward_features(event_tensor) #torch.Size([16, 2, 256, 21, 16])   torch.Size([16, 2, 336, 256])
        event_tensor = event_tensor.permute(1,0,2).reshape(B,-1)
        x1 = self.head(event_tensor)
    
        return x1
    

    @register_model def Spikingformer(pretrained=False, **kwargs): model = vit_snn( patch_size=16, embed_dims=256, num_heads=8, mlp_ratios=4, in_channels=12, num_classes=300, qkv_bias=False, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=2, sr_ratios=1, **kwargs ) model.default_cfg = _cfg() return model

    ,集成到vit_ce中,但是报错:sampler_mode causal Traceback (most recent call last): File "lib/train/run_training.py", line 112, in main() File "lib/train/run_training.py", line 103, in main run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark, File "lib/train/run_training.py", line 76, in run_training expr_func(settings) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/train/train_script.py", line 55, in run net = build_ceutrack(cfg) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/ceutrack.py", line 124, in build_ceutrack backbone = vit_base_patch16_224_ce(pretrained, drop_path_rate=cfg.TRAIN.DROP_PATH_RATE, File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 193, in vit_base_patch16_224_ce model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 175, in _create_vision_transformer model = VisionTransformerCE(**kwargs) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 58, in init self.vit_snn_model = vit_snn(img_size_h=128, img_size_w=128, patch_size=16, in_channels=12, num_classes=11, File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_snn.py", line 200, in init dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depths)]
    TypeError: linspace(): argument 'steps' (position 3) must be int, not list

  10. Assistant

    你遇到的错误是由于 torch.linspace 的第三个参数 steps 传递了一个列表而不是一个整数。depths 是一个列表,因此必须处理它以确保 torch.linspace 接受整数。

    在你的 vit_snn 类的构造函数中修改生成 dpr 列表的方式即可解决这个问题。以下是修改后的 __init__ 方法:

    class vit_snn(nn.Module):
        def __init__(self,
                     img_size_h=128, img_size_w=128, patch_size=16, in_channels=2, num_classes=11,
                     embed_dims=[64, 128, 256], num_heads=[1, 2, 4], mlp_ratios=[4, 4, 4], qkv_bias=False, qk_scale=None,
                     drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=nn.LayerNorm,
                     depths=[6, 8, 6], sr_ratios=[8, 4, 2], T=4, pretrained_cfg=None):
            super().__init__()
            self.num_classes = num_classes
            self.depths = depths
            self.T = T
    
            dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]  # Modify here
    
            patch_embed = SpikingTokenizer(img_size_h=img_size_h,
                                           img_size_w=img_size_w,
                                           patch_size=patch_size,
                                           in_channels=in_channels,
                                           embed_dims=embed_dims)
            num_patches = patch_embed.num_patches
            block = nn.ModuleList([SpikingTransformer(
                dim=embed_dims[i], num_heads=num_heads[i], mlp_ratio=mlp_ratios[i], qkv_bias=qkv_bias,
                qk_scale=qk_scale, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[j],
                norm_layer=norm_layer, sr_ratio=sr_ratios[i])
                for i in range(len(embed_dims))
                for j in range(depths[i])])
    
            self.Bottleneck_block = nn.ModuleList([BottleneckTransformer(
                    dim=embed_dims[-1], num_heads=4, mlp_ratio=mlp_ratios[-1], 
                    qkv_bias=qkv_bias, drop=0., attn_drop=0., init_values=None,
                    drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm)
                    for j in range(2)])
    
            setattr(self, f"patch_embed", patch_embed)
            setattr(self, f"block", block)
    
            # classification head
            self.head = nn.Linear(8192, num_classes) if num_classes > 0 else nn.Identity()
            self.apply(self._init_weights)
    
            self.bottleneck_initfeature = nn.Parameter(torch.randn(1, 64, 256))
            kaiming_init(self.bottleneck_initfeature, mode='fan_out')
            self.pos_embed = nn.Parameter(torch.zeros(1, 400, 256))
            self.pos_drop = nn.Dropout(p=0.)
    
        def _init_weights(self, m):
            if isinstance(m, nn.Linear):
                trunc_normal_(m.weight, std=.02)
                if isinstance(m, nn.Linear) and m.bias is not None:
                    nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.LayerNorm):
                nn.init.constant_(m.bias, 0)
                nn.init.constant_(m.weight, 1.0)
    
        def forward_features(self, x):
            block = getattr(self, f"block")
            patch_embed = getattr(self, f"patch_embed")
    
            x, (H, W) = patch_embed(x)
            for blk in block:
                x = blk(x)
            T, B, C, H, W = x.shape
    
            botlk_x = x.flatten(3)
            botlk_x = botlk_x.flatten(0, 1).permute(0, 2, 1)
    
            bottleneck_feature = self.bottleneck_initfeature.expand(T * B, 64, C)
            fusion_feature_agg = torch.cat((bottleneck_feature, botlk_x), 1)
            fusion_feature_agg = self.pos_drop(fusion_feature_agg + self.pos_embed)
    
            for bottleneck_blk in self.Bottleneck_block:
                fusion_feature_agg = bottleneck_blk(fusion_feature_agg)
            bottleneck_feature = fusion_feature_agg[:, 0:336, :].mean(1).reshape(B, 1, -1)
            return x.flatten(3).mean(3), bottleneck_feature
    
        def forward(self, event_tensor):
            B, N, C, H, W = event_tensor.shape
            event_tensor = event_tensor.permute(1, 0, 2, 3, 4)
            event_tensor, bottleneck_feature = self.forward_features(event_tensor)
            event_tensor = event_tensor.permute(1, 0, 2).reshape(B, -1)
            x1 = self.head(event_tensor)
    
            return x1
    

    以上修改中,dpr 列表的生成方式中 torch.linspace() 的第三个参数被修改为 sum(depths),它将返回一个包含 depths 元素总数个数值的列表。这确保了 drop_path_rate 正确地按比例分配到每个块。

    请确保修改后的代码片段与 SpikingTokenizer 和 SpikingTransformer 类适当地整合。另外,确保你正确地将 vit_snn 类实例化并用于你的主模型。如果有其他模型和逻辑依赖于这个参数调整,请一并检查和测试。

  11. User

    只把修改的部分写出来

  12. Assistant

    好的,只写出需要修改的部分。

    # 修改 vit_snn 类中的以下几行
    class vit_snn(nn.Module):
        def __init__(self,
                     # ... 省略其他参数
                     depths=[6, 8, 6],  # 确保这个默认值和你调整的相匹配
                     # ... 省略其他参数
                     ):
            # 省略其他代码
            # 修改 linspace 函数的调用
            dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]  # 修改这行
            
            # 下面的 block 定义部分也需要稍微调整,以适配不同深度的 block
            block = nn.ModuleList([
                SpikingTransformer(
                    dim=embed_dims[i], num_heads=num_heads[i], mlp_ratio=mlp_ratios[i], qkv_bias=qkv_bias,
                    qk_scale=qk_scale, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[j],
                    norm_layer=norm_layer, sr_ratio=sr_ratios[i])
                for i in range(len(embed_dims))
                for j in range(depths[i])
            ])
            # 省略其他代码
    

    说明

    1. 线性插值 (linspace) 部分: 将原来 torch.linspace(0, drop_path_rate, depths) 修改为 torch.linspace(0, drop_path_rate, sum(depths))。

    2. block 的创建部分: 在创建 block 的时候,外层循环 for i in range(len(embed_dims)) 遍历 embed_dims,内层循环 for j in range(depths[i]) 遍历每个深度列表中的项。这样确保 dpr 列表中每个元素都正确分配到每个 Transformer 块。

    这些修改会确保 dpr 的元素数量与所有 Transformer 块的总数匹配,从而避免错误。

  13. User

    报错:Traceback (most recent call last): File "lib/train/run_training.py", line 112, in main() File "lib/train/run_training.py", line 103, in main run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark, File "lib/train/run_training.py", line 76, in run_training expr_func(settings) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/train/train_script.py", line 55, in run net = build_ceutrack(cfg) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/ceutrack.py", line 124, in build_ceutrack backbone = vit_base_patch16_224_ce(pretrained, drop_path_rate=cfg.TRAIN.DROP_PATH_RATE, File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 193, in vit_base_patch16_224_ce model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 175, in _create_vision_transformer model = VisionTransformerCE(**kwargs) File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 58, in init self.vit_snn_model = vit_snn(img_size_h=128, img_size_w=128, patch_size=16, in_channels=12, num_classes=11, File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_snn.py", line 202, in init patch_embed = SpikingTokenizer(img_size_h=img_size_h, File "/data2/local_userdata/heyuting/REUTrack/CEUTrack/lib/train/../../lib/models/ceutrack/vit_snn.py", line 139, in init self.proj_conv = nn.Conv2d(in_channels, embed_dims//8, kernel_size=3, stride=1, padding=1, bias=False) TypeError: unsupported operand type(s) for //: 'list' and 'int'

  14. Assistant

    报错信息指出 embed_dims 是一个列表,但在初始化 SpikingTokenizer 类时尝试对其进行整数除法 (//) 操作。这显然是不支持的,因为你不能对列表进行整数除法。

    你需要确保 embed_dims 是单一整数值而不是列表,或者如果确实需要传递列表,则需要对代码进行相应的调整以处理此列表。

    在你的案例中,假设你希望 patch_embed、block 中包含多个维度(一个维度为每个块),那么你需要分别处理这些维度。

    请将以下部分代码进行调整:

    在 SpikingTokenizer 中,确保 embed_dims 是单一整数

    在调用 SpikingTokenizer 时传递正确的参数。在 vit_snn 构造函数中,请确保传递的是单个整数,而不是列表。

    class vit_snn(nn.Module):
        def __init__(self,
                     img_size_h=128, img_size_w=128, patch_size=16, in_channels=2, num_classes=11,
                     embed_dims=[64, 128, 256], num_heads=[1, 2, 4], mlp_ratios=[4, 4, 4], qkv_bias=False, qk_scale=None,
                     drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=nn.LayerNorm,
                     depths=[6, 8, 6], sr_ratios=[8, 4, 2], T=4, pretrained_cfg=None):
            super().__init__()
            self.num_classes = num_classes
            self.depths = depths
            self.T = T
            
            # 合并 depth 以计算 drop path rate
            total_depth = sum(depths)
            dpr = [x.item() for x in torch.linspace(0, drop_path_rate, total_depth)]
    
            # 创建每个 stage 的 SpikingTokenizer 和 SpikingTransformer block
            self.patch_embeds = nn.ModuleList([SpikingTokenizer(
                img_size_h=img_size_h if i == 0 else img_size_h // (2 ** i),
                img_size_w=img_size_w if i == 0 else img_size_w // (2 ** i),
                patch_size=patch_size,
                in_channels=in_channels if i == 0 else embed_dims[i - 1],
                embed_dims=embed_dims[i]
            ) for i in range(len(embed_dims))])
    
            self.blocks = nn.ModuleList([
                nn.ModuleList([
                    SpikingTransformer(
                        dim=embed_dims[i], num_heads=num_heads[i], mlp_ratio=mlp_ratios[i], qkv_bias=qkv_bias,
                        qk_scale=qk_scale, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[idx],
                        norm_layer=norm_layer, sr_ratio=sr_ratios[i]
                    ) for idx in range(sum(depths[:i]), sum(depths[:i+1]))
                ]) for i in range(len(embed_dims))
            ])
    
            self.bottleneck_blocks = nn.ModuleList([
                BottleneckTransformer(
                    dim=embed_dims[-1], num_heads=4, mlp_ratio=mlp_ratios[-1], 
                    qkv_bias=qkv_bias, drop=0., attn_drop=0., init_values=None,
                    drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm
                ) for _ in range(2)
            ])
    
            # classification head
            self.head = nn.Linear(embed_dims[-1] * (img_size_h // (patch_size * 2 ** (len(embed_dims) - 1))) *
                                  (img_size_w // (patch_size * 2 ** (len(embed_dims) - 1))), num_classes) if num_classes > 0 else nn.Identity()
            self.apply(self._init_weights)
    
            self.bottleneck_initfeature = nn.Parameter(torch.randn(1, 64, embed_dims[-1]))
            kaiming_init(self.bottleneck_initfeature, mode='fan_out')
            self.pos_embed = nn.Parameter(torch.zeros(1, 400, embed_dims[-1]))
            self.pos_drop = nn.Dropout(p=0.)
    
        def _init_weights(self, m):
            if isinstance(m, nn.Linear):
                trunc_normal_(m.weight, std=.02)
                if isinstance(m, nn.Linear) and m.bias is not None:
                    nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.LayerNorm):
                nn.init.constant_(m.bias, 0)
                nn.init.constant_(m.weight, 1.0)
    
        def forward_features(self, x):
            for i, (patch_embed, blocks) in enumerate(zip(self.patch_embeds, self.blocks)):
                x, (H, W) = patch_embed(x)
                for block in blocks:
                    x = block(x)
    
            x = x.flatten(3).mean(3)
            T, B, C = x.shape
    
            botlk_x = x.view(T * B, -1, C)
            bottleneck_feature = self.bottleneck_initfeature.expand(T * B, 64, -1)
            fusion_feature_agg = torch.cat((bottleneck_feature, botlk_x), 1)
            fusion_feature_agg = self.pos_drop(fusion_feature_agg + self.pos_embed)
    
            for bottleneck_blk in self.bottleneck_blocks:
                fusion_feature_agg = bottleneck_blk(fusion_feature_agg)
            
            bottleneck_feature = fusion_feature_agg[:, 0:336, :].mean(1).view(B, T, -1)
            return x.mean(1), bottleneck_feature
    
        def forward(self, event_tensor):
            B, N, C, H, W = event_tensor.shape
            event_tensor = event_tensor.permute(1, 0, 2, 3, 4)
            event_tensor, bottleneck_feature = self.forward_features(event_tensor)
            event_tensor = event_tensor.permute(1, 0, 2).reshape(B, -1)
            x1 = self.head(event_tensor)
    
            return x1
    

    说明

    1. 修改 dpr:使用 sum(depths) 计算总的阶段,并调整 dpr 的生成。
    2. 调整各阶段的 patch_embed 和 block:使用 nn.ModuleList 分别定义每个 Tokenizer 和 Transformer block。
    3. 注意: 确保 BottleneckTransformer 和 SpikingTransformer 的参数与更改后的网络结构匹配。

    通过这些修改,模型应会正确处理 embed_dims 列表,并且每个阶段的 patch_embed 和 block 将适应不同的 embed_dims。