Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-05-13· OpenAIAi2 WildChat-4.8M (AI2)

现在将# 将 4输入分开,构建新的相同模态结合的2输入,2分支 import math import logging from functools import partial…

  1. User

    现在将# 将 4输入分开,构建新的相同模态结合的2输入,2分支 import math import logging from functools import partial from collections import OrderedDict from copy import deepcopy

    import torch import torch.nn as nn import torch.nn.functional as F

    from timm.models.layers import to_2tuple

    from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise from .utils import combine_tokens, recover_tokens from .vit import VisionTransformer from ..layers.attn_blocks import CEBlock

    _logger = logging.getLogger(name)

    class VisionTransformerCE(VisionTransformer): """ Vision Transformer with candidate elimination (CE) module

    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
        - https://arxiv.org/abs/2010.11929
    
    Includes distillation token & head support for `DeiT: Data-efficient Image Transformers`
        - https://arxiv.org/abs/2012.12877
    """
    def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
                 num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False,
                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None,
                 act_layer=None, weight_init='',
                 ce_loc=None, ce_keep_ratio=None):
        super().__init__()
        if isinstance(img_size, tuple):
            self.img_size = img_size
        else:
            self.img_size = to_2tuple(img_size)
        self.patch_size = patch_size
        self.in_chans = in_chans
    
        self.num_classes = num_classes
        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
        self.num_tokens = 2 if distilled else 1
        norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
        act_layer = act_layer or nn.GELU
    
        self.patch_embed = embed_layer(
            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
        num_patches = self.patch_embed.num_patches
    
        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
        self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None
        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
        self.pos_drop = nn.Dropout(p=drop_rate)
        self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4)
        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
        blocks = []
        ce_index = 0
        self.ce_loc = ce_loc
        for i in range(depth):
            ce_keep_ratio_i = 1.0
            if ce_loc is not None and i in ce_loc:
                ce_keep_ratio_i = ce_keep_ratio[ce_index]
                ce_index += 1
    
            blocks.append(
                CEBlock(
                    dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate,
                    attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer,
                    keep_ratio_search=ce_keep_ratio_i)
            )
    
        self.blocks = nn.Sequential(*blocks)
        self.norm = norm_layer(embed_dim)
    
        self.init_weights(weight_init)
    
    
    def forward_features(self, z, x, event_z, event_x,
                         mask_z=None, mask_x=None,
                         ce_template_mask=None, ce_keep_rate=None,
                         return_last_attn=False
                         ):
        # 分支1 处理流程
        B, H, W = x.shape[0], x.shape[2], x.shape[3]
    
        x = self.patch_embed(x)
        z = self.patch_embed(z)
        z += self.pos_embed_z
        x += self.pos_embed_x        
    
        if mask_z is not None and mask_x is not None:
            mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_z = mask_z.flatten(1).unsqueeze(-1)
    
            mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_x = mask_x.flatten(1).unsqueeze(-1)
    
            mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode)
            mask_x = mask_x.squeeze(-1)
    
        if self.add_cls_token:
            cls_tokens = self.cls_token.expand(B, -1, -1)
            cls_tokens = cls_tokens + self.cls_pos_embed
        if self.add_sep_seg:
            x += self.search_segment_pos_embed
            z += self.template_segment_pos_embed
        x = combine_tokens(z, x, mode=self.cat_mode)
        if self.add_cls_token:
            x = torch.cat([cls_tokens, x], dim=1)
        x = self.pos_drop(x)
    
        lens_z = self.pos_embed_z.shape[1]
        lens_x = self.pos_embed_x.shape[1]
        global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device)
        global_index_t = global_index_t.repeat(B, 1)
        global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device)
        global_index_s = global_index_s.repeat(B, 1)
        removed_indexes_s = []
    
        for i, blk in enumerate(self.blocks):
            x, global_index_t, global_index_s, removed_index_s, attn = \
                blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate)
    
    
            if self.ce_loc is not None and i in self.ce_loc:
                removed_indexes_s.append(removed_index_s)
        x = self.norm(x) # # [bs, n_patch, dim] = [bs, 320, 768] 320 = 64 + 256
       
    
        # # 分支2 处理流程
        event_x = self.pos_embed_event(event_x)
        event_z = self.pos_embed_event(event_z)
        event_x += self.pos_embed_x
        event_z += self.pos_embed_z
    
        event_x = combine_tokens(event_z, event_x, mode=self.cat_mode)
    
        if self.add_cls_token:
            event_x = torch.cat([cls_tokens, event_x], dim=1)
    
        lens_z = self.pos_embed_z.shape[1]
        lens_x = self.pos_embed_x.shape[1]
    
        global_index_t1 = torch.linspace(0, lens_z - 1, lens_z).to(event_x.device)
        global_index_t1 = global_index_t1.repeat(B, 1)
        global_index_s1 = torch.linspace(0, lens_x - 1, lens_x).to(event_x.device)
        global_index_s1 = global_index_s1.repeat(B, 1)
    
        removed_indexes_s1 = []
        for i, blk in enumerate(self.blocks):
            event_x, global_index_t1, global_index_s1, removed_index_s1, attn = \
                blk(event_x, global_index_t1, global_index_s1, mask_x, ce_template_mask, ce_keep_rate)
    
            if self.ce_loc is not None and i in self.ce_loc:
                removed_indexes_s1.append(removed_index_s1)
        event_x = self.norm(event_x)
        # print('x.shape: ',x.shape) #x.shape:  torch.Size([2, 320, 768])
        # print('event_x.shape: ',event_x.shape) # event_x.shape:  torch.Size([2, 320, 768])
        x_cat = torch.cat([x, event_x], dim=1) 
        # print('x_cat.shape: ',x_cat.shape) # x_cat.shape:  torch.Size([2, 640, 768])
        x = x_cat
    
        aux_dict = {
            "attn": attn,
            "removed_indexes_s": removed_indexes_s,  # used for visualization
        }
        
        return x, aux_dict 
    
    def forward(self, z, x, event_z, event_x,
                ce_template_mask=None, ce_keep_rate=None,
                tnc_keep_rate=None,
                return_last_attn=False):
    
        x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,)
    
        return x, aux_dict
    

    def _create_vision_transformer(pretrained=False, **kwargs): model = VisionTransformerCE(**kwargs)

    if pretrained:
        if 'npz' in pretrained:
            model.load_pretrained(pretrained, prefix='')
        else:
            checkpoint = torch.load(pretrained, map_location="cpu")
            missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False)
            print('Load pretrained model from: ' + pretrained)
    
    return model
    

    def vit_base_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model

    def vit_large_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) model = create_vision_transformer(pretrained=pretrained, **model_kwargs) return model中event部分的特征提取网络改为import torch import torch.nn as nn from spikingjelly.clock_driven.neuron import MultiStepParametricLIFNode, MultiStepLIFNode from spikingjelly.clock_driven import layer from timm.models.layers import to_2tuple, trunc_normal, DropPath from timm.models.registry import register_model from timm.models.vision_transformer import _cfg from einops.layers.torch import Rearrange import torch.nn.functional as F from functools import partial from timm.models import create_model import pdb from Bottleneck import BottleneckTransformer

    from mmengine.model.weight_init import kaiming_init

    from mmcv.cnn import kaiming_init

    all = ['Spikingformer'] tau_thr = 1.5

    class MLP(nn.Module): def init(self, in_features, hidden_features=None, out_features=None, drop=0.): super().init() out_features = out_features or in_features hidden_features = hidden_features or in_features self.mlp1_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True) #, backend='cupy' self.mlp1_conv = nn.Conv2d(in_features, hidden_features, kernel_size=1, stride=1) self.mlp1_bn = nn.BatchNorm2d(hidden_features)

        self.mlp2_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
        self.mlp2_conv = nn.Conv2d(hidden_features, out_features, kernel_size=1, stride=1)
        self.mlp2_bn = nn.BatchNorm2d(out_features)
    
        self.c_hidden = hidden_features
        self.c_output = out_features
    
    def forward(self, x):
        T, B, C, H, W = x.shape
    
        x = self.mlp1_lif(x)
        x = self.mlp1_conv(x.flatten(0, 1))
        x = self.mlp1_bn(x).reshape(T, B, self.c_hidden, H, W)
    
        x = self.mlp2_lif(x)
        x = self.mlp2_conv(x.flatten(0, 1))
        x = self.mlp2_bn(x).reshape(T, B, C, H, W)
        return x
    

    class SpikingSelfAttention(nn.Module): def init(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., sr_ratio=1): super().init() assert dim % num_heads == 0, f"dim {dim} should be divided by num_heads {num_heads}."

        self.dim = dim
        self.num_heads = num_heads
    
        self.proj_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
    
        self.q_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1, bias=False)
        self.q_bn = nn.BatchNorm1d(dim)
        self.q_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
    
        self.k_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1, bias=False)
        self.k_bn = nn.BatchNorm1d(dim)
        self.k_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
    
        self.v_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1, bias=False)
        self.v_bn = nn.BatchNorm1d(dim)
        self.v_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
    
        self.attn_lif = MultiStepLIFNode(tau=tau_thr, v_threshold=0.5, detach_reset=True)
    
        self.proj_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1)
        self.proj_bn = nn.BatchNorm1d(dim)
    
    
    def forward(self, x):
        T, B, C, H, W = x.shape 
        x = self.proj_lif(x)
    
        x = x.flatten(3)   
        T, B, C, N = x.shape
        x_for_qkv = x.flatten(0, 1) 
    
        q_conv_out = self.q_conv(x_for_qkv)
        q_conv_out = self.q_bn(q_conv_out).reshape(T, B, C, N)
        q_conv_out = self.q_lif(q_conv_out)
        q = q_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C // self.num_heads).permute(0, 1, 3, 2, 4) #torch.Size([4, 2, 16, 336, 16])
    
        k_conv_out = self.k_conv(x_for_qkv)
        k_conv_out = self.k_bn(k_conv_out).reshape(T, B, C, N)
        k_conv_out = self.k_lif(k_conv_out)
        k = k_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C // self.num_heads).permute(0, 1, 3, 2, 4)#torch.Size([4, 2, 16, 336, 16])
    
        v_conv_out = self.v_conv(x_for_qkv)
        v_conv_out = self.v_bn(v_conv_out).reshape(T, B, C, N)
        v_conv_out = self.v_lif(v_conv_out)
        v = v_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C // self.num_heads).permute(0, 1, 3, 2, 4)  #torch.Size([4, 2, 16, 336, 16])
    
        attn = (q @ k.transpose(-2, -1))  #torch.Size([4, 2, 16, 336, 336])
        x = (attn @ v) * 0.125  #torch.Size([4, 2, 16, 336, 16])
    
        x = x.transpose(3, 4).reshape(T, B, C, N)
        x = self.attn_lif(x)
        x = x.flatten(0, 1)
        x = self.proj_bn(self.proj_conv(x)).reshape(T, B, C, H, W)  #torch.Size([4, 2, 256, 21, 16])
    
        return x
    

    class SpikingTransformer(nn.Module): def init(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., drop_path=0., norm_layer=nn.LayerNorm, sr_ratio=1): super().init() self.norm1 = norm_layer(dim) self.attn = SpikingSelfAttention(dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop, sr_ratio=sr_ratio) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() self.norm2 = norm_layer(dim) mlp_hidden_dim = int(dim * mlp_ratio) self.mlp = MLP(in_features=dim, hidden_features=mlp_hidden_dim, drop=drop)

    def forward(self, x):
        x = x + self.attn(x)
        x = x + self.mlp(x)
        return x
    

    class SpikingTokenizer(nn.Module): def init(self, img_size_h=128, img_size_w=128, patch_size=4, in_channels=2, embed_dims=256): super().init() self.image_size = [img_size_h, img_size_w] patch_size = to_2tuple(patch_size) self.patch_size = patch_size self.C = in_channels self.H, self.W = self.image_size[0] // patch_size[0], self.image_size[1] // patch_size[1] self.num_patches = self.H * self.W

        self.proj_conv = nn.Conv2d(in_channels, embed_dims//8, kernel_size=3, stride=1, padding=1, bias=False)
        self.proj_bn = nn.BatchNorm2d(embed_dims//8)
    
        self.proj1_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
        self.maxpool1 = torch.nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
        self.proj1_conv = nn.Conv2d(embed_dims//8, embed_dims//4, kernel_size=3, stride=1, padding=1, bias=False)
        self.proj1_bn = nn.BatchNorm2d(embed_dims//4)
    
        self.proj2_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
        self.maxpool2 = torch.nn.MaxPool2d(kernel_size=(3,2), stride=2, padding=0, dilation=1, ceil_mode=False)
        self.proj2_conv = nn.Conv2d(embed_dims//4, embed_dims//2, kernel_size=3, stride=1, padding=1, bias=False)
        self.proj2_bn = nn.BatchNorm2d(embed_dims//2)
    
        self.proj3_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
        self.maxpool3 = torch.nn.MaxPool2d(kernel_size=(2,3), stride=2, padding=0, dilation=1, ceil_mode=False)
        self.proj3_conv = nn.Conv2d(embed_dims//2, embed_dims, kernel_size=3, stride=1, padding=1, bias=False)
        self.proj3_bn = nn.BatchNorm2d(embed_dims)
    
        self.proj4_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
        self.maxpool4 = torch.nn.MaxPool2d(kernel_size=(3,2), stride=2, padding=0, dilation=1, ceil_mode=False)
        self.proj4_conv = nn.Conv2d(embed_dims, embed_dims, kernel_size=3, stride=1, padding=1, bias=False)
        self.proj4_bn = nn.BatchNorm2d(embed_dims)
    
    def forward(self, x):
        T, B, C, H, W = x.shape  
        x = self.proj_conv(x.flatten(0, 1)) 
        x = self.proj_bn(x).reshape(T,B,-1,H,W).contiguous()  
    
        x = self.proj1_lif(x).flatten(0,1).contiguous()  
        x = self.maxpool1(x)                             
        x = self.proj1_conv(x)                           
        x = self.proj1_bn(x).reshape(T, B, -1, H//2, W//2).contiguous() 
    
        x = self.proj2_lif(x).flatten(0, 1).contiguous() 
        x = self.maxpool2(x)                               
        x = self.proj2_conv(x)                             
        x = self.proj2_bn(x).reshape(T, B, -1, H//4, W//4).contiguous() 
    
        x = self.proj3_lif(x).flatten(0, 1).contiguous()    
        x = self.maxpool3(x)                            
        x = self.proj3_conv(x)                          
        x = self.proj3_bn(x).reshape(T, B, -1, H//8, W//8).contiguous() 
    
        x = self.proj4_lif(x).flatten(0, 1).contiguous() 
        x = self.maxpool4(x)                            
        x = self.proj4_conv(x)                          
        x = self.proj4_bn(x).reshape(T, B, -1, H//16, W//16).contiguous() 
        return x, (None, None)
    

    class vit_snn(nn.Module): def init(self, img_size_h=128, img_size_w=128, patch_size=16, in_channels=2, num_classes=11, embed_dims=[64, 128, 256], num_heads=[1, 2, 4], mlp_ratios=[4, 4, 4], qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=nn.LayerNorm, depths=[6, 8, 6], sr_ratios=[8, 4, 2], T=4, pretrained_cfg=None, ): super().init() self.num_classes = num_classes self.depths = depths self.T = T dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depths)]

        patch_embed = SpikingTokenizer(img_size_h=img_size_h,
                          img_size_w=img_size_w,
                          patch_size=patch_size,
                          in_channels=in_channels,
                          embed_dims=embed_dims)
        num_patches = patch_embed.num_patches
        block = nn.ModuleList([SpikingTransformer(
            dim=embed_dims, num_heads=num_heads, mlp_ratio=mlp_ratios, qkv_bias=qkv_bias,
            qk_scale=qk_scale, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[j],
            norm_layer=norm_layer, sr_ratio=sr_ratios)
            for j in range(depths)])
    
        self.Bottleneck_block = nn.ModuleList([BottleneckTransformer(
            dim=embed_dims, num_heads=4, mlp_ratio=mlp_ratios,
            qkv_bias=qkv_bias, drop=0., attn_drop=0., init_values=None,
            drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm)
            for j in range(2)])
    
        setattr(self, f"patch_embed", patch_embed)
        setattr(self, f"block", block)
    
        # classification head
        self.head = nn.Linear(8192, num_classes) if num_classes > 0 else nn.Identity()
        self.apply(self._init_weights)
        
        self.bottleneck_initfeature = nn.Parameter(torch.randn(1, 64, 256))
        kaiming_init(self.bottleneck_initfeature, mode='fan_out')  
        self.pos_embed = nn.Parameter(torch.zeros(1, 400, 256))
        self.pos_drop = nn.Dropout(p=0.)
    
    def _init_weights(self, m):
        if isinstance(m, nn.Linear):
            trunc_normal_(m.weight, std=.02)
            if isinstance(m, nn.Linear) and m.bias is not None:
                nn.init.constant_(m.bias, 0)
        elif isinstance(m, nn.LayerNorm):
            nn.init.constant_(m.bias, 0)
            nn.init.constant_(m.weight, 1.0)
    
    def forward_features(self, x):
        block = getattr(self, f"block")
        patch_embed = getattr(self, f"patch_embed")
    
        x, (H, W) = patch_embed(x)   #x--torch.Size([4, 2, 256, 21, 16])
        for blk in block:
            x = blk(x)     #torch.Size([16, 8, 256, 21, 16])     #torch.Size([4, 2, 256, 21, 16])  
        T,B,C,H,W =x.shape
    
        botlk_x = x.flatten(3)   #torch.Size([4, 2, 256, 336])   T, B, C, N
        botlk_x = botlk_x.flatten(0, 1).permute(0,2,1)  #torch.Size([8, 336, 256])
    
        bottleneck_feature = self.bottleneck_initfeature.expand(T*B,64,C)
        fusion_feature_agg = torch.cat((bottleneck_feature,botlk_x),1)  #BT, 672,256
        fusion_feature_agg = self.pos_drop(fusion_feature_agg + self.pos_embed)  # 
    
        for bottleneck_blk in self.Bottleneck_block:
            fusion_feature_agg = bottleneck_blk(fusion_feature_agg)
        bottleneck_feature = fusion_feature_agg[:,0:336,:].mean(1).reshape(B,1,-1) #torch.Size([16, 2, 336, 256])
        return x.flatten(3).mean(3), bottleneck_feature   #torch.Size([4, 2, 256])
    
    def forward(self, event_tensor):
        B, N, C, H, W = event_tensor.shape
        event_tensor = event_tensor.permute(1, 0, 2, 3, 4)  # [T, N, 2, *, *]    torch.Size([16, 2, 12, 346, 260])
        event_tensor, bottleneck_feature = self.forward_features(event_tensor) #torch.Size([16, 2, 256, 21, 16])   torch.Size([16, 2, 336, 256])
        event_tensor = event_tensor.permute(1,0,2).reshape(B,-1)
        x1 = self.head(event_tensor)
    
        return x1
    

    @register_model def Spikingformer(pretrained=False, **kwargs): model = vit_snn( patch_size=16, embed_dims=256, num_heads=8, mlp_ratios=4, in_channels=12, num_classes=300, qkv_bias=False, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=2, sr_ratios=1, **kwargs ) model.default_cfg = _cfg() return model

    。

  2. Assistant

    好的,让我们将 VisionTransformerCE 类中的 event 部分的特征提取网络替换为 vit_snn。在这个过程中,我们需要调整相关代码,以确保新特征提取器的输入和输出满足 VisionTransformerCE 的需求。以下是更新后的代码:

    import math
    import logging
    from functools import partial
    from collections import OrderedDict
    from copy import deepcopy
    
    import torch
    import torch.nn as nn
    import torch.nn.functional as F
    
    from timm.models.layers import to_2tuple
    
    from lib.models.layers.patch_embed import PatchEmbed, xcorr_depthwise
    from .utils import combine_tokens, recover_tokens
    from .vit import VisionTransformer
    from ..layers.attn_blocks import CEBlock
    
    # Spikingformer related imports
    import torch
    import torch.nn as nn
    from spikingjelly.clock_driven.neuron import MultiStepParametricLIFNode, MultiStepLIFNode
    from spikingjelly.clock_driven import layer
    from timm.models.layers import to_2tuple, trunc_normal_, DropPath
    from timm.models.registry import register_model
    from timm.models.vision_transformer import _cfg
    from einops.layers.torch import Rearrange
    import torch.nn.functional as F
    from functools import partial
    from timm.models import create_model
    from Bottleneck import BottleneckTransformer
    from mmcv.cnn import kaiming_init
    
    _logger = logging.getLogger(__name__)
    
    
    class VisionTransformerCE(VisionTransformer):
        """ Vision Transformer with candidate elimination (CE) module
    
        A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
            - https://arxiv.org/abs/2010.11929
    
        Includes distillation token & head support for `DeiT: Data-efficient Image Transformers`
            - https://arxiv.org/abs/2012.12877
        """
        def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
                     num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False,
                     drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None,
                     act_layer=None, weight_init='',
                     ce_loc=None, ce_keep_ratio=None):
            super().__init__()
            if isinstance(img_size, tuple):
                self.img_size = img_size
            else:
                self.img_size = to_2tuple(img_size)
            self.patch_size = patch_size
            self.in_chans = in_chans
    
            self.num_classes = num_classes
            self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
            self.num_tokens = 2 if distilled else 1
            norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
            act_layer = act_layer or nn.GELU
    
            self.patch_embed = embed_layer(
                img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
            num_patches = self.patch_embed.num_patches
    
            self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
            self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None
            self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
            self.pos_drop = nn.Dropout(p=drop_rate)
            self.pos_embed_event = vit_snn(img_size_h=img_size, img_size_w=img_size, patch_size=4, in_channels=32, embed_dims=768) # Replace with vit_snn
            dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
            blocks = []
            ce_index = 0
            self.ce_loc = ce_loc
            for i in range(depth):
                ce_keep_ratio_i = 1.0
                if ce_loc is not None and i in ce_loc:
                    ce_keep_ratio_i = ce_keep_ratio[ce_index]
                    ce_index += 1
    
                blocks.append(
                    CEBlock(
                        dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate,
                        attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer,
                        keep_ratio_search=ce_keep_ratio_i)
                )
    
            self.blocks = nn.Sequential(*blocks)
            self.norm = norm_layer(embed_dim)
    
            self.init_weights(weight_init)
    
        def forward_features(self, z, x, event_z, event_x,
                             mask_z=None, mask_x=None,
                             ce_template_mask=None, ce_keep_rate=None,
                             return_last_attn=False
                             ):
            # 分支1 处理流程
            B, H, W = x.shape[0], x.shape[2], x.shape[3]
    
            x = self.patch_embed(x)
            z = self.patch_embed(z)
            z += self.pos_embed_z
            x += self.pos_embed_x        
    
            if mask_z is not None and mask_x is not None:
                mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
                mask_z = mask_z.flatten(1).unsqueeze(-1)
    
                mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
                mask_x = mask_x.flatten(1).unsqueeze(-1)
    
                mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode)
                mask_x = mask_x.squeeze(-1)
    
            if self.add_cls_token:
                cls_tokens = self.cls_token.expand(B, -1, -1)
                cls_tokens = cls_tokens + self.cls_pos_embed
            if self.add_sep_seg:
                x += self.search_segment_pos_embed
                z += self.template_segment_pos_embed
            x = combine_tokens(z, x, mode=self.cat_mode)
            if self.add_cls_token:
                x = torch.cat([cls_tokens, x], dim=1)
            x = self.pos_drop(x)
    
            lens_z = self.pos_embed_z.shape[1]
            lens_x = self.pos_embed_x.shape[1]
            global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device)
            global_index_t = global_index_t.repeat(B, 1)
            global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device)
            global_index_s = global_index_s.repeat(B, 1)
            removed_indexes_s = []
    
            for i, blk in enumerate(self.blocks):
                x, global_index_t, global_index_s, removed_index_s, attn = \
                    blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate)
    
    
                if self.ce_loc is not None and i in self.ce_loc:
                    removed_indexes_s.append(removed_index_s)
            x = self.norm(x) # # [bs, n_patch, dim] = [bs, 320, 768] 320 = 64 + 256
           
    
            # # 分支2 处理流程
            event_x = self.pos_embed_event(event_x)  # 使用自定义Spikingformer特征提取器
            event_z = self.pos_embed_event(event_z)  # 使用自定义Spikingformer特征提取器
    
            event_x += self.pos_embed_x
            event_z += self.pos_embed_z
    
            event_x = combine_tokens(event_z, event_x, mode=self.cat_mode)
    
            if self.add_cls_token:
                event_x = torch.cat([cls_tokens, event_x], dim=1)
    
            lens_z = self.pos_embed_z.shape[1]
            lens_x = self.pos_embed_x.shape[1]
    
            global_index_t1 = torch.linspace(0, lens_z - 1, lens_z).to(event_x.device)
            global_index_t1 = global_index_t1.repeat(B, 1)
            global_index_s1 = torch.linspace(0, lens_x - 1, lens_x).to(event_x.device)
            global_index_s1 = global_index_s1.repeat(B, 1)
    
            removed_indexes_s1 = []
            for i, blk in enumerate(self.blocks):
                event_x, global_index_t1, global_index_s1, removed_index_s1, attn = \
                    blk(event_x, global_index_t1, global_index_s1, mask_x, ce_template_mask, ce_keep_rate)
    
                if self.ce_loc is not None and i in self.ce_loc:
                    removed_indexes_s1.append(removed_index_s1)
            event_x = self.norm(event_x)
            x_cat = torch.cat([x, event_x], dim=1) 
            x = x_cat
    
            aux_dict = {
                "attn": attn,
                "removed_indexes_s": removed_indexes_s,  # used for visualization
            }
            
            return x, aux_dict 
    
        def forward(self, z, x, event_z, event_x,
                    ce_template_mask=None, ce_keep_rate=None,
                    tnc_keep_rate=None,
                    return_last_attn=False):
    
            x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,)
    
            return x, aux_dict
    
    
    def _create_vision_transformer(pretrained=False, **kwargs):
        model = VisionTransformerCE(**kwargs)
        
        if pretrained:
            if 'npz' in pretrained:
                model.load_pretrained(pretrained, prefix='')
            else:
                checkpoint = torch.load(pretrained, map_location="cpu")
                missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False)
                print('Load pretrained model from: ' + pretrained)
    
        return model
    
    
    def vit_base_patch16_224_ce(pretrained=False, **kwargs):
        """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
        """
        model_kwargs = dict(
            patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs)
        model = _create_vision_transformer(pretrained=pretrained, **model_kwargs)
        return model
    
    
    def vit_large_patch16_224_ce(pretrained=False, **kwargs):
        """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929).
        """
        model_kwargs = dict(
            patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs)
        model = _create_vision_transformer(pretrained=pretrained, **model_kwargs)
        return model
    
    # Spikingformer definition
    tau_thr = 1.5
    
    class MLP(nn.Module):
        def __init__(self, in_features, hidden_features=None, out_features=None, drop=0.):
            super().__init__()
            out_features = out_features or in_features
            hidden_features = hidden_features or in_features
            self.mlp1_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)  #, backend='cupy'
            self.mlp1_conv = nn.Conv2d(in_features, hidden_features, kernel_size=1, stride=1)
            self.mlp1_bn = nn.BatchNorm2d(hidden_features)
    
            self.mlp2_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
            self.mlp2_conv = nn.Conv2d(hidden_features, out_features, kernel_size=1, stride=1)
            self.mlp2_bn = nn.BatchNorm2d(out_features)
    
            self.c_hidden = hidden_features
            self.c_output = out_features
    
        def forward(self, x):
            T, B, C, H, W = x.shape
    
            x = self.mlp1_lif(x)
            x = self.mlp1_conv(x.flatten(0, 1))
            x = self.mlp1_bn(x).reshape(T, B, self.c_hidden, H, W)
    
            x = self.mlp2_lif(x)
            x = self.mlp2_conv(x.flatten(0, 1))
            x = self.mlp2_bn(x).reshape(T, B, C, H, W)
            return x
    
    
    class SpikingSelfAttention(nn.Module):
        def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., sr_ratio=1):
            super().__init__()
            assert dim % num_heads == 0, f"dim {dim} should be divided by num_heads {num_heads}."
    
            self.dim = dim
            self.num_heads = num_heads
    
            self.proj_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
    
            self.q_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1, bias=False)
            self.q_bn = nn.BatchNorm1d(dim)
            self.q_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
    
            self.k_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1, bias=False)
            self.k_bn = nn.BatchNorm1d(dim)
            self.k_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
    
            self.v_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1, bias=False)
            self.v_bn = nn.BatchNorm1d(dim)
            self.v_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
    
            self.attn_lif = MultiStepLIFNode(tau=tau_thr, v_threshold=0.5, detach_reset=True)
    
            self.proj_conv = nn.Conv1d(dim, dim, kernel_size=1, stride=1)
            self.proj_bn = nn.BatchNorm1d(dim)
    
    
        def forward(self, x):
            T, B, C, H, W = x.shape 
            x = self.proj_lif(x)
    
            x = x.flatten(3)   
            T, B, C, N = x.shape
            x_for_qkv = x.flatten(0, 1) 
    
            q_conv_out = self.q_conv(x_for_qkv)
            q_conv_out = self.q_bn(q_conv_out).reshape(T, B, C, N)
            q_conv_out = self.q_lif(q_conv_out)
            q = q_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C // self.num_heads).permute(0, 1, 3, 2, 4) #torch.Size([4, 2, 16, 336, 16])
    
            k_conv_out = self.k_conv(x_for_qkv)
            k_conv_out = self.k_bn(k_conv_out).reshape(T, B, C, N)
            k_conv_out = self.k_lif(k_conv_out)
            k = k_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C // self.num_heads).permute(0, 1, 3, 2, 4)#torch.Size([4, 2, 16, 336, 16])
    
            v_conv_out = self.v_conv(x_for_qkv)
            v_conv_out = self.v_bn(v_conv_out).reshape(T, B, C, N)
            v_conv_out = self.v_lif(v_conv_out)
            v = v_conv_out.transpose(-1, -2).reshape(T, B, N, self.num_heads, C // self.num_heads).permute(0, 1, 3, 2, 4)  #torch.Size([4, 2, 16, 336, 16])
    
            attn = (q @ k.transpose(-2, -1))  #torch.Size([4, 2, 16, 336, 336])
            x = (attn @ v) * 0.125  #torch.Size([4, 2, 16, 336, 16])
    
            x = x.transpose(3, 4).reshape(T, B, C, N)
            x = self.attn_lif(x)
            x = x.flatten(0, 1)
            x = self.proj_bn(self.proj_conv(x)).reshape(T, B, C, H, W)  #torch.Size([4, 2, 256, 21, 16])
    
            return x
    
    
    class SpikingTransformer(nn.Module):
        def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
                     drop_path=0., norm_layer=nn.LayerNorm, sr_ratio=1):
            super().__init__()
            self.norm1 = norm_layer(dim)
            self.attn = SpikingSelfAttention(dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale,
                                               attn_drop=attn_drop, proj_drop=drop, sr_ratio=sr_ratio)
            self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
            self.norm2 = norm_layer(dim)
            mlp_hidden_dim = int(dim * mlp_ratio)
            self.mlp = MLP(in_features=dim, hidden_features=mlp_hidden_dim, drop=drop)
    
        def forward(self, x):
            x = x + self.attn(x)
            x = x + self.mlp(x)
            return x
    
    
    class SpikingTokenizer(nn.Module):
        def __init__(self, img_size_h=128, img_size_w=128, patch_size=4, in_channels=2, embed_dims=256):
            super().__init__()
            self.image_size = [img_size_h, img_size_w]
            patch_size = to_2tuple(patch_size)
            self.patch_size = patch_size
            self.C = in_channels
            self.H, self.W = self.image_size[0] // patch_size[0], self.image_size[1] // patch_size[1]
            self.num_patches = self.H * self.W
    
            self.proj_conv = nn.Conv2d(in_channels, embed_dims//8, kernel_size=3, stride=1, padding=1, bias=False)
            self.proj_bn = nn.BatchNorm2d(embed_dims//8)
    
            self.proj1_lif = MultiStepLIFNode(tau=tau_thr, detach_reset=True)
            self.maxpool1 = torch.nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
            self.proj1_conv = nn.Conv2d(embed_dims//8, embed_dims//4, kernel_size=3, stride=1, padding=1, bias=False)
            self.proj1_bn =