Skip to content
The Chat Archive
gpt-4-turbo-2024-04-09· OpenAI WildChat-4.8M (AI2)

在import math import logging from functools import partial from collections import…

  1. User

    在import math import logging from functools import partial from collections import OrderedDict from copy import deepcopy

    import torch import torch.nn as nn import torch.nn.functional as F

    from timm.models.layers import to_2tuple

    from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise from .utils import combine_tokens, recover_tokens from .vit import VisionTransformer from ..layers.attn_blocks import CEBlock import random import numpy as np

    from .muti_cou import Counter_Guide_Enhanced_with_MultiHead, Cross_Attention

    from .cross_module import Counter_Guide_Enhanced, Cross_Attention

    _logger = logging.getLogger(name)

    class VisionTransformerCE(VisionTransformer): """ Vision Transformer with candidate elimination (CE) module

    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
        - https://arxiv.org/abs/2010.11929
    
    Includes distillation token & head support for `DeiT: Data-efficient Image Transformers`
        - https://arxiv.org/abs/2012.12877
    """
    def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
                 num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False,
                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None,
                 act_layer=None, weight_init='',
                 ce_loc=None, ce_keep_ratio=None):
        super().__init__()
        if isinstance(img_size, tuple):
            self.img_size = img_size
        else:
            self.img_size = to_2tuple(img_size)
        self.patch_size = patch_size
        self.in_chans = in_chans
    
        self.num_classes = num_classes
        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
        self.num_tokens = 2 if distilled else 1
        norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
        act_layer = act_layer or nn.GELU
    
        self.patch_embed = embed_layer(
            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
        num_patches = self.patch_embed.num_patches
    
        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
        self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None
        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
        self.pos_drop = nn.Dropout(p=drop_rate)
        self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4)
    
        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
        blocks = []
        ce_index = 0
        self.ce_loc = ce_loc
        for i in range(depth):
            ce_keep_ratio_i = 1.0
            if ce_loc is not None and i in ce_loc:
                ce_keep_ratio_i = ce_keep_ratio[ce_index]
                ce_index += 1
    
            blocks.append(
                CEBlock(
                    dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate,
                    attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer,
                    keep_ratio_search=ce_keep_ratio_i)
            )
    
        self.blocks = nn.Sequential(*blocks)
        self.norm = norm_layer(embed_dim)
    
        self.init_weights(weight_init)
    
        # 添加交互模块counter_guide-此处对模块已经做了更新
        self.counter_guide = Counter_Guide_Enhanced(768, 768)
        # self.counter_guide = Counter_Guide_Enhanced_with_MultiHead(768, 768, 12)
    
        # self.cross_attention = Cross_Attention(768, 768)
    
    # 新增随机遮蔽特征操作
    # def masking_fea(self,z, event_z, x, event_x, ratio=0.8):
    #     b,nz,c = z.shape
    #     b,nez,c = event_z.shape
    #     b,nx,c = x.shape
    #     b,nex,c = event_x.shape
    
    #     assert(nz == nez)
    #     assert(nx == nex)
    #     lenz_out = int(nz*ratio)
    #     lenx_out = int(nx*ratio)
    
    #     mask_nz = torch.rand(b,nz).float()
    #     mask_ez = torch.rand(b,nez).float()
    #     mask_nx = torch.rand(b,nx).float()
    #     mask_ex = torch.rand(b,nex).float()
    #     mask_nz = mask_nz>0.4
    #     mask_ez = mask_ez>0.4
    #     mask_ez = ~mask_nz + mask_ez
    #     mask_nz_idx = mask_nz.float().sort(1,descending=True)[-1].to(device = z.device)
    #     mask_ez_idx = mask_ez.float().sort(1,descending=True)[-1].to(device = z.device)
    #     mask_nx = mask_nx>0.4
    #     mask_ex = mask_ex>0.4
    #     mask_ex = ~mask_nx + mask_ex
    #     mask_nx_idx = mask_nx.float().sort(1,descending=True)[-1].to(device = z.device)
    #     mask_ex_idx = mask_ex.float().sort(1,descending=True)[-1].to(device = z.device)
    
    #     masked_z = torch.gather(z, 1, mask_nz_idx[:,:lenz_out,None].repeat([1,1,c]))
    #     masked_ez = torch.gather(event_z, 1, mask_ez_idx[:,:lenz_out,None].repeat([1,1,c]))
    #     masked_x = torch.gather(x, 1, mask_nx_idx[:,:lenx_out,None].repeat([1,1,c]))
    #     masked_ex = torch.gather(event_x, 1, mask_ex_idx[:,:lenx_out,None].repeat([1,1,c]))
    #     return masked_z, masked_ez, masked_x, masked_ex,{'x1':mask_nx_idx[:,:lenx_out],'x0':mask_nx_idx[:,lenx_out:],
    #                                                     'ex1':mask_ex_idx[:,:lenx_out],'ex0':mask_ex_idx[:,lenx_out:], }
    
    # 随机遮蔽特征操作完成
    # def forward_features(self, z, x, event_z, event_x,
    #                      mask_z=None, mask_x=None,
    #                      ce_template_mask=None, ce_keep_rate=None,
    #                      return_last_attn=False, Track=False
    #                      ):
    def forward_features(self, z, x, event_z, event_x,
                        mask_z=None, mask_x=None,
                        ce_template_mask=None, ce_keep_rate=None,
                        return_last_attn=False
                        ):
        # 分支1 处理流程
        B, H, W = x.shape[0], x.shape[2], x.shape[3]
        x = self.patch_embed(x) 
        z = self.patch_embed(z) 
        z += self.pos_embed_z
        x += self.pos_embed_x   
        
        # event_x = self.pos_embed_event(event_x)  
        # event_z = self.pos_embed_event(event_z) 
        # event_x += self.pos_embed_x
        # event_z += self.pos_embed_z
        # if Track == False:
        #     masked_z, masked_event_z, masked_x, masked_event_x, token_idx = self.masking_fea(z, event_z, x, event_x, ratio=0.98)
    
        # z = masked_z
        # x = masked_x
        # event_x = masked_event_x
        # event_z = masked_event_z  
    
        if mask_z is not None and mask_x is not None:
            mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_z = mask_z.flatten(1).unsqueeze(-1)
    
            mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_x = mask_x.flatten(1).unsqueeze(-1)
    
            mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode)
            mask_x = mask_x.squeeze(-1)
    
        if self.add_cls_token:
            cls_tokens = self.cls_token.expand(B, -1, -1)
            cls_tokens = cls_tokens + self.cls_pos_embed
        if self.add_sep_seg:
            x += self.search_segment_pos_embed
            z += self.template_segment_pos_embed
        x = combine_tokens(z, x, mode=self.cat_mode)
        if self.add_cls_token:
            x = torch.cat([cls_tokens, x], dim=1)
        x = self.pos_drop(x)
    
        lens_z = self.pos_embed_z.shape[1]
        lens_x = self.pos_embed_x.shape[1]
        global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device)
        global_index_t = global_index_t.repeat(B, 1)
        global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device)
        global_index_s = global_index_s.repeat(B, 1)
        removed_indexes_s = []
    
        # # 分支2 处理流程
        event_x = self.pos_embed_event(event_x)  
        event_z = self.pos_embed_event(event_z) 
        event_x += self.pos_embed_x
        event_z += self.pos_embed_z
        event_x = combine_tokens(event_z, event_x, mode=self.cat_mode)
    
        if self.add_cls_token:
            event_x = torch.cat([cls_tokens, event_x], dim=1)
    
        lens_z = self.pos_embed_z.shape[1]
        lens_x = self.pos_embed_x.shape[1]
    
        global_index_t1 = torch.linspace(0, lens_z - 1, lens_z).to(event_x.device)
        global_index_t1 = global_index_t1.repeat(B, 1)
        global_index_s1 = torch.linspace(0, lens_x - 1, lens_x).to(event_x.device)
        global_index_s1 = global_index_s1.repeat(B, 1)
    
        removed_indexes_s1 = []
    
        out_attn = []
        out_attn1 = []
        for i, blk in enumerate(self.blocks):
            # 第一个分支处理
            x, global_index_t, global_index_s, removed_index_s, attn = \
                blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate)
            # 第二个分支处理
            event_x, global_index_t1, global_index_s1, removed_index_s1, attn1 = \
                blk(event_x, global_index_t1, global_index_s1, mask_x, ce_template_mask, ce_keep_rate)
    
    
            if self.ce_loc is not None and i in self.ce_loc:
                removed_indexes_s.append(removed_index_s)
                removed_indexes_s1.append(removed_index_s1)
    
            out_attn.append(attn)
            out_attn1.append(attn1)
            
            if i == 0 :
                # 新增原始特征(在counter_guide之前的特征,便于后续的loss计算)
                A_original_x = x.clone() # 保存原始x特征 (RGB)
                A_original_event_x = event_x.clone() # 保存原始event特征 (event)
    
                # 引入counter_guide进行特征增强 此处RGB模态特征作为主特征,event_x作为补充特征
                enhanced_x, enhanced_event_x = self.counter_guide(x, event_x)
    
                # 保存经过counter_guide增强的特征
                A_enhanced_x = enhanced_x  # 保存增强后的特征
                A_enhanced_event_x = enhanced_event_x  # 保存增强后的特征
                # 将增强后的特征与原特征相加,这里类似残差操作
                x = x + enhanced_x
                event_x = event_x + enhanced_event_x
    
        out_attn_idx = random.choice(np.arange(len(out_attn)))
        out_attn = out_attn[out_attn_idx]
    
        out_attn_idx1 = random.choice(np.arange(len(out_attn1)))
        out_attn1 = out_attn1[out_attn_idx1]
    
        # 应用LayerNorm归一化处理
        x = self.norm(x) 
        event_x = self.norm(event_x)
    
        x_cat = torch.cat([event_x, x], dim=1) 
        x = x_cat # cat仅仅是做 rgb 和 event voxel拼接,数据增强采用的随机遮蔽特征策略。
        # x_cat = self.cross_attention(event_x, x)
        # x = x_cat
    
    
        # aux_dict = {
        #     "attn": out_attn,
        #     "attn1": out_attn1,
        #     "removed_indexes_s": removed_indexes_s,  # used for visualization
        #     'removed_indexes_s1': removed_indexes_s1,
        # }
        aux_dict = {
            "attn": out_attn,
            "attn1": out_attn1,
            "removed_indexes_s": removed_indexes_s,  # used for visualization
            'removed_indexes_s1': removed_indexes_s1,
        }
        return x, aux_dict 
    
    # def forward(self, z, x, event_z, event_x,
    #             ce_template_mask=None, ce_keep_rate=None,
    #             tnc_keep_rate=None,
    #             return_last_attn=False, Track=False):
    def forward(self, z, x, event_z, event_x,
                ce_template_mask=None, ce_keep_rate=None,
                tnc_keep_rate=None,
                return_last_attn=False):
        x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate)
        # x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,Track=Track)
    
        return x, aux_dict
    

    def _create_vision_transformer(pretrained=False, **kwargs): model = VisionTransformerCE(**kwargs)

    if pretrained:
        if 'npz' in pretrained:
            model.load_pretrained(pretrained, prefix='')
        else:
            checkpoint = torch.load(pretrained, map_location="cpu")
            missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False)
            print('Load pretrained model from: ' + pretrained)
    
    return model
    

    def vit_base_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model

    def vit_large_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model

    中,将分支2,即event数据分支的模型换为:import torch import torch.nn as nn from spikingjelly.clock_driven.neuron import MultiStepLIFNode from spikingjelly.clock_driven import layer, surrogate from timm.models.layers import to_2tuple, trunc_normal_, DropPath from timm.models.registry import register_model from timm.models.vision_transformer import _cfg from einops.layers.torch import Rearrange import torch.nn.functional as F from functools import partial

    all = ['spikformer']

    class MLP(nn.Module): def init(self, in_features, hidden_features=None, out_features=None, drop=0.): super().init() out_features = out_features or in_features hidden_features = hidden_features or in_features self.fc1_conv = nn.Conv1d(in_features, hidden_features, kernel_size=1, stride=1) self.fc1_bn = nn.BatchNorm1d(hidden_features) self.fc1_lif = MultiStepLIFNode(tau=2.0, detach_reset=True, backend='cupy')

        self.fc2_conv = nn.Conv1d(hidden_features, out_features, kernel_size=1, stride=1)
        self.fc2_bn = nn.BatchNorm1d(out_features)
        self.fc2_lif = MultiStepLIFNode(tau=2.0, detach_reset=True, backend='cupy')
        self.c_hidden = hidden_features
        self.c_output = out_features
    def forward(self, x):
        T,B,C,N = x.shape
        x = self.fc1_conv(x.flatten(0,1))
        x = self.fc1_bn(x).reshape(T,B,self.c_hidden,N).contiguous()
        x = self.fc1_lif(x)
    
        x = self.fc2_conv(x.flatten(0,1))
        x = self.fc2_bn(x).reshape(T,B,C,N).contiguous()
        x = self.fc2_lif(x)
        return x
    

    class SSA(nn.Module): def init(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., sr_ratio=1): super().init() assert dim % num_heads == 0, f"dim {dim} should be divided by num_heads {num_heads}." self.dim = dim self.num_heads = num_heads self.scale = 0.25

        self.q_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1,bias=False)
    
        self.q_bn = nn.BatchNorm1d(dim)
        self.q_lif = MultiStepLIFNode(tau=2.0, detach_reset=True, backend='cupy')
    
        self.k_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1,bias=False)
        self.k_bn = nn.BatchNorm1d(dim)
        self.k_lif = MultiStepLIFNode(tau=2.0, detach_reset=True, backend='cupy')
    
        self.v_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1,bias=False)
        self.v_bn = nn.BatchNorm1d(dim)
        self.v_lif = MultiStepLIFNode(tau=2.0, detach_reset=True, backend='cupy')
    
        self.attn_drop = nn.Dropout(0.2)
        self.res_lif = MultiStepLIFNode(tau=2.0, detach_reset=True, backend='cupy')
        self.attn_lif = MultiStepLIFNode(tau=2.0, v_threshold=0.5, detach_reset=True, backend='cupy')
    
        self.proj_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1)
        self.proj_bn = nn.BatchNorm1d(dim)
        self.proj_lif = MultiStepLIFNode(tau=2.0, detach_reset=True, backend='cupy')
    
    def forward(self, x):
        T, B, C, N = x.shape
        x_for_qkv = x.flatten(0, 1)
        q_conv_out = self.q_conv(x_for_qkv)
        q_conv_out = self.q_bn(q_conv_out).reshape(T,B,C,N).contiguous()
        q_conv_out = self.q_lif(q_conv_out)
        q = q_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C//self.num_heads).permute(0, 1, 3, 2, 4).contiguous()
    
        k_conv_out = self.k_conv(x_for_qkv)
        k_conv_out = self.k_bn(k_conv_out).reshape(T,B,C,N).contiguous()
        k_conv_out = self.k_lif(k_conv_out)
        k = k_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C//self.num_heads).permute(0, 1, 3, 2, 4).contiguous()
    
        v_conv_out = self.v_conv(x_for_qkv)
        v_conv_out = self.v_bn(v_conv_out).reshape(T,B,C,N).contiguous()
        v_conv_out = self.v_lif(v_conv_out)
        v = v_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C//self.num_heads).permute(0, 1, 3, 2, 4).contiguous()
    
        attn = (q @ k.transpose(-2, -1))
        x = (attn @ v) * self.scale
    
        x = x.transpose(3, 4).reshape(T, B, C, N).contiguous()
        x = self.attn_lif(x)
        x = x.flatten(0,1)
        x = self.proj_lif(self.proj_bn(self.proj_conv(x)).reshape(T,B,C,N))
    
        return x
    

    class Block(nn.Module): def init(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., drop_path=0., norm_layer=nn.LayerNorm, sr_ratio=1): super().init() self.norm1 = norm_layer(dim) self.attn = SSA(dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop, sr_ratio=sr_ratio) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() self.norm2 = norm_layer(dim) mlp_hidden_dim = int(dim * mlp_ratio) self.mlp = MLP(in_features=dim, hidden_features=mlp_hidden_dim, drop=drop)

    def forward(self, x):
        x = x + self.attn(x)
        x = x + (self.mlp(x))
        return x
    

    class SPS(nn.Module): def init(self, img_size_h=128, img_size_w=128, patch_size=4, in_channels=2, embed_dims=256): super().init() self.image_size = [img_size_h, img_size_w] patch_size = to_2tuple(patch_size) self.patch_size = patch_size self.C = in_channels self.H, self.W = self.image_size[0] // patch_size[0], self.image_size[1] // patch_size[1] self.num_patches = self.H * self.W self.proj_conv = nn.Conv2d(in_channels, embed_dims//8, kernel_size=3, stride=1, padding=1, bias=False) self.proj_bn = nn.BatchNorm2d(embed_dims//8) self.proj_lif = MultiStepLIFNode(tau=2.0, detach_reset=True, backend='cupy') self.maxpool = torch.nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)

        self.proj_conv1 = nn.Conv2d(embed_dims//8, embed_dims//4, kernel_size=3, stride=1, padding=1, bias=False)
        self.proj_bn1 = nn.BatchNorm2d(embed_dims//4)
        self.proj_lif1 = MultiStepLIFNode(tau=2.0, detach_reset=True, backend='cupy')
        self.maxpool1 = torch.nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
    
        self.proj_conv2 = nn.Conv2d(embed_dims//4, embed_dims//2, kernel_size=3, stride=1, padding=1, bias=False)
        self.proj_bn2 = nn.BatchNorm2d(embed_dims//2)
        self.proj_lif2 = MultiStepLIFNode(tau=2.0, detach_reset=True, backend='cupy')
        self.maxpool2 = torch.nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
    
        self.proj_conv3 = nn.Conv2d(embed_dims//2, embed_dims, kernel_size=3, stride=1, padding=1, bias=False)
        self.proj_bn3 = nn.BatchNorm2d(embed_dims)
        self.proj_lif3 = MultiStepLIFNode(tau=2.0, detach_reset=True, backend='cupy')
        self.maxpool3 = torch.nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
    
        self.rpe_conv = nn.Conv2d(embed_dims, embed_dims, kernel_size=3, stride=1, padding=1, bias=False)
        self.rpe_bn = nn.BatchNorm2d(embed_dims)
        self.rpe_lif = MultiStepLIFNode(tau=2.0, detach_reset=True, backend='cupy')
    def forward(self, x):
        T, B, C, H, W = x.shape
        x = self.proj_conv(x.flatten(0, 1)) # have some fire value
        x = self.proj_bn(x).reshape(T,B,-1,H,W).contiguous()
        x = self.proj_lif(x).flatten(0,1).contiguous()
        x = self.maxpool(x)
    
        x = self.proj_conv1(x)
        x = self.proj_bn1(x).reshape(T, B, -1, 64, 64).contiguous()
        x = self.proj_lif1(x).flatten(0, 1).contiguous()
        x = self.maxpool1(x)
    
        x = self.proj_conv2(x)
        x = self.proj_bn2(x).reshape(T, B, -1, 32, 32).contiguous()
        x = self.proj_lif2(x).flatten(0, 1).contiguous()
        x = self.maxpool2(x)
    
        x = self.proj_conv3(x)
        x = self.proj_bn3(x).reshape(T, B, -1, 16, 16).contiguous()
        x = self.proj_lif3(x).flatten(0, 1).contiguous()
        x = self.maxpool3(x)
    
        x_rpe = self.rpe_bn(self.rpe_conv(x)).reshape(T,B,-1,H//16,W//16).contiguous()
        x_rpe = self.rpe_lif(x_rpe).flatten(0,1)
        x = x + x_rpe
        x = x.reshape(T, B, -1, (H//16)*(H//16)).contiguous()
    
        return x
    

    class Spikformer(nn.Module): def init(self, img_size_h=128, img_size_w=128, patch_size=16, in_channels=2, num_classes=11, embed_dims=[64, 128, 256], num_heads=[1, 2, 4], mlp_ratios=[4, 4, 4], qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=nn.LayerNorm, depths=[6, 8, 6], sr_ratios=[8, 4, 2] ): super().init() self.num_classes = num_classes self.depths = depths

        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depths)]  # stochastic depth decay rule
    
        patch_embed = SPS(img_size_h=img_size_h,
                                 img_size_w=img_size_w,
                                 patch_size=patch_size,
                                 in_channels=in_channels,
                                 embed_dims=embed_dims)
        num_patches = patch_embed.num_patches
        pos_embed = nn.Parameter(torch.zeros(1, num_patches, embed_dims))
        block = nn.ModuleList([Block(
            dim=embed_dims, num_heads=num_heads, mlp_ratio=mlp_ratios, qkv_bias=qkv_bias,
            qk_scale=qk_scale, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[j],
            norm_layer=norm_layer, sr_ratio=sr_ratios)
            for j in range(depths)])
    
        setattr(self, f"patch_embed", patch_embed)
        setattr(self, f"pos_embed", pos_embed)
        setattr(self, f"block", block)
    
        self.norm = norm_layer(embed_dims)
        # classification head 这里不需要脉冲,因为输入的是在T时长平均发射值
        self.head = nn.Linear(embed_dims, num_classes) if num_classes > 0 else nn.Identity()
    
        pos_embed = getattr(self, f"pos_embed")
        trunc_normal_(pos_embed, std=.02)
        self.apply(self._init_weights)
    
    @torch.jit.ignore
    def _get_pos_embed(self, pos_embed, patch_embed, H, W):
        if H * W == self.patch_embed1.num_patches:
            return pos_embed
        else:
            return F.interpolate(
                pos_embed.reshape(1, patch_embed.H, patch_embed.W, -1).permute(0, 3, 1, 2),
                size=(H, W), mode="bilinear").reshape(1, -1, H * W).permute(0, 2, 1)
    
    def _init_weights(self, m):
        if isinstance(m, nn.Linear):
            trunc_normal_(m.weight, std=.02)
            if isinstance(m, nn.Linear) and m.bias is not None:
                nn.init.constant_(m.bias, 0)
        elif isinstance(m, nn.LayerNorm):
            nn.init.constant_(m.bias, 0)
            nn.init.constant_(m.weight, 1.0)
    
    def forward_features(self, x):
    
        block = getattr(self, f"block")
        patch_embed = getattr(self, f"patch_embed")
        x = patch_embed(x)
        for blk in block:
            x = blk(x)
        return x.mean(3)
    
    def forward(self, x):
        x = x.permute(1, 0, 2, 3, 4)  # [T, N, 2, *, *]
        x = self.forward_features(x)
        x = self.head(x.mean(0))
        return x
    

    @register_model def spikformer(pretrained=False, **kwargs): model = Spikformer( patch_size=16, embed_dims=256, num_heads=16, mlp_ratios=4, in_channels=2, num_classes=10, qkv_bias=False, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=2, sr_ratios=1, **kwargs ) model.default_cfg = _cfg() return model

  2. Assistant

    要将VisionTransformerCE模型中处理event数据的分支替换为使用Spikformer模型,我们首先需要确保两种模型可以交互适配。由于Spikformer是以脉冲神经网络构建的,它的输入输出与VisionTransformerCE中现有的处理方法会有所不同。在替换之前,需要确保event数据的格式可以被Spikformer正确处理,并且Spikformer的输出能够为VisionTransformerCE所接受。

    以下是集成Spikformer模型到VisionTransformerCE中的修改步骤,这些修改将仅针对处理event数据的部分:

    1. 引入Spikformer模型定义: 确保你已将Spikformer的类定义放入当前的工作环境中。

    2. 将Spikformer集成至VisionTransformerCE:

      • 在VisionTransformerCE类的初始化方法(init)中,初始化Spikformer模型。
      • 在forward_features方法中,用Spikformer模型替换原有处理event_data的逻辑。

    修改后的VisionTransformerCE类将包括对Spikformer模型的引用和数据流的重新指向。

    修改后的VisionTransformerCE代码

    class VisionTransformerCE(VisionTransformer):
        def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
                     num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False,
                     drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None,
                     act_layer=None, weight_init='',
                     ce_loc=None, ce_keep_ratio=None):
            super().__init__()
            # 初始化和配置代码保持不变
            # ...
    
            # 使用Spikformer模型处理event数据
            self.event_transformer = Spikformer(
                img_size_h=img_size, img_size_w=img_size, in_channels=in_chans,
                num_classes=num_classes, embed_dims=embed_dim, depth=depth,
                num_heads=num_heads)
        
        def forward_features(self, z, x, event_z, event_x, ...):
            # 处理RGB数据的代码保持原样
            # ...
            
            # 使用Spikformer处理事件数据
            event_x_transformed = self.event_transformer(event_x)
            
            # 整合脉冲神经网络输出与Vision Transformer的输出
            
            # 其他处理逻辑
            # ...
    
            return x, aux_dict
    

    注意事项

    • **数据格式:**需要确保event数据格式与Spikformer模型期望的输入格式一致。Spikformer模型通常处理的是多步脉冲序列,因此可能需要调整或转换数据。
    • **模型训练配置:**集成脉冲神经网络可能需要使用不同的训练策略,包括损失函数和优化方法,以适应孤立脉冲的特性。
    • **测试和验证:**在完成集成后,需要详尽测试新模型以验证其性能和稳定性。

    以上步骤提供了将Spikformer集成进VisionTransformerCE的基本框架,实际应用可能还需要进一步的调试和优化。