Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-08-06· OpenAIAi2 WildChat-4.8M (AI2)

分析模型:# Copyright (c) 2015-present, Facebook, Inc. All rights reserved. import torch…

  1. User

    分析模型:# Copyright (c) 2015-present, Facebook, Inc.

    All rights reserved.

    import torch import torch.nn as nn from functools import partial from torch import Tensor from typing import Optional

    from timm.models.vision_transformer import VisionTransformer, cfg from timm.models.registry import register_model from timm.models.layers import trunc_normal, lecun_normal_

    from timm.models.layers import DropPath, to_2tuple from timm.models.vision_transformer import _load_weights

    import math

    from collections import namedtuple

    from mamba_ssm.modules.mamba_simple import Mamba from mamba_ssm.utils.generation import GenerationMixin from mamba_ssm.utils.hf import load_config_hf, load_state_dict_hf

    from lib.models.mamba_fetrack.rope import * import random from lib.models.layers.head import build_box_head import importlib import lib.train.admin.settings as ws_settings from .utils import combine_tokens, recover_tokens

    try: from mamba_ssm.ops.triton.layernorm import RMSNorm, layer_norm_fn, rms_norm_fn except ImportError: RMSNorm, layer_norm_fn, rms_norm_fn = None, None, None

    all = [ 'vim_tiny_patch16_224', 'vim_small_patch16_224', 'vim_base_patch16_224', 'vim_tiny_patch16_384', 'vim_small_patch16_384', 'vim_base_patch16_384', ]

    class PatchEmbed(nn.Module): """ 2D Image to Patch Embedding """ def init(self, img_size=224, patch_size=16, stride=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True): super().init() img_size = to_2tuple(img_size) patch_size = to_2tuple(patch_size) self.img_size = img_size self.patch_size = patch_size self.grid_size = ((img_size[0] - patch_size[0]) // stride + 1, (img_size[1] - patch_size[1]) // stride + 1) self.num_patches = self.grid_size[0] * self.grid_size[1] self.flatten = flatten

        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=stride)
        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
    
    def forward(self, x):
        B, C, H, W = x.shape
        # assert H == self.img_size[0] and W == self.img_size[1], \
        #     f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})."
        x = self.proj(x)
        if self.flatten:
            x = x.flatten(2).transpose(1, 2)  # BCHW -> BNC
        x = self.norm(x)
        return x
    

    class Block(nn.Module): def init( self, dim, mixer_cls, norm_cls=nn.LayerNorm, fused_add_norm=False, residual_in_fp32=False,drop_path=0., ): """ Simple block wrapping a mixer class with LayerNorm/RMSNorm and residual connection"

        This Block has a slightly different structure compared to a regular
        prenorm Transformer block.
        The standard block is: LN -> MHA/MLP -> Add.
        [Ref: https://arxiv.org/abs/2002.04745]
        Here we have: Add -> LN -> Mixer, returning both
        the hidden_states (output of the mixer) and the residual.
        This is purely for performance reasons, as we can fuse add and LayerNorm.
        The residual needs to be provided (except for the very first block).
        """
        super().__init__()
        self.residual_in_fp32 = residual_in_fp32
        self.fused_add_norm = fused_add_norm
        self.mixer = mixer_cls(dim)
        self.norm = norm_cls(dim)
        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
        if self.fused_add_norm:
            assert RMSNorm is not None, "RMSNorm import fails"
            assert isinstance(
                self.norm, (nn.LayerNorm, RMSNorm)
            ), "Only LayerNorm and RMSNorm are supported for fused_add_norm"
    
    def forward(
        self, hidden_states: Tensor, residual: Optional[Tensor] = None, inference_params=None
    ):
        r"""Pass the input through the encoder layer.
    
        Args:
            hidden_states: the sequence to the encoder layer (required).
            residual: hidden_states = Mixer(LN(residual))
        """
        if not self.fused_add_norm:
            if residual is None:
                residual = hidden_states
            else:
                residual = residual + self.drop_path(hidden_states)
            
            hidden_states = self.norm(residual.to(dtype=self.norm.weight.dtype))
            if self.residual_in_fp32:
                residual = residual.to(torch.float32)
        else:
            fused_add_norm_fn = rms_norm_fn if isinstance(self.norm, RMSNorm) else layer_norm_fn
            if residual is None:
                hidden_states, residual = fused_add_norm_fn(
                    hidden_states,
                    self.norm.weight,
                    self.norm.bias,
                    residual=residual,
                    prenorm=True,
                    residual_in_fp32=self.residual_in_fp32,
                    eps=self.norm.eps,
                )
            else:
                hidden_states, residual = fused_add_norm_fn(
                    self.drop_path(hidden_states),
                    self.norm.weight,
                    self.norm.bias,
                    residual=residual,
                    prenorm=True,
                    residual_in_fp32=self.residual_in_fp32,
                    eps=self.norm.eps,
                )    
        hidden_states = self.mixer(hidden_states, inference_params=inference_params)
        return hidden_states, residual
    
    def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs):
        return self.mixer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)
    

    def create_block( d_model, ssm_cfg=None, norm_epsilon=1e-5, drop_path=0., rms_norm=False, residual_in_fp32=False, fused_add_norm=False, layer_idx=None, device=None, dtype=None, if_bimamba=False, bimamba_type="none", if_devide_out=False, init_layer_scale=None, ): if if_bimamba: bimamba_type = "v1" if ssm_cfg is None: ssm_cfg = {} factory_kwargs = {"device": device, "dtype": dtype} mixer_cls = partial(Mamba, layer_idx=layer_idx, bimamba_type=bimamba_type, if_devide_out=if_devide_out, init_layer_scale=init_layer_scale, **ssm_cfg, **factory_kwargs) norm_cls = partial( nn.LayerNorm if not rms_norm else RMSNorm, eps=norm_epsilon, **factory_kwargs ) block = Block( d_model, mixer_cls, norm_cls=norm_cls, drop_path=drop_path, fused_add_norm=fused_add_norm, residual_in_fp32=residual_in_fp32, ) block.layer_idx = layer_idx return block

    https://github.com/huggingface/transformers/blob/c28d04e9e252a1a099944e325685f14d242ecdcd/src/transformers/models/gpt2/modeling_gpt2.py#L454

    def init_weights( module, n_layer, initializer_range=0.02, # Now only used for embedding layer. rescale_prenorm_residual=True, n_residuals_per_layer=1, # Change to 2 if we have MLP ): if isinstance(module, nn.Linear): if module.bias is not None: if not getattr(module.bias, "no_reinit", False): nn.init.zeros(module.bias) elif isinstance(module, nn.Embedding): nn.init.normal(module.weight, std=initializer_range)

    if rescale_prenorm_residual:
        # Reinitialize selected weights subject to the OpenAI GPT-2 Paper Scheme:
        #   > A modified initialization which accounts for the accumulation on the residual path with model depth. Scale
        #   > the weights of residual layers at initialization by a factor of 1/√N where N is the # of residual layers.
        #   >   -- GPT-2 :: https://openai.com/blog/better-language-models/
        #
        # Reference (Megatron-LM): https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/model/gpt_model.py
        for name, p in module.named_parameters():
            if name in ["out_proj.weight", "fc2.weight"]:
                # Special Scaled Initialization --> There are 2 Layer Norms per Transformer Block
                # Following Pytorch init, except scale by 1/sqrt(2 * n_layer)
                # We need to reinit p since this code could be called multiple times
                # Having just p *= scale would repeatedly scale it down
                nn.init.kaiming_uniform_(p, a=math.sqrt(5))
                with torch.no_grad():
                    p /= math.sqrt(n_residuals_per_layer * n_layer)
    

    def segm_init_weights(m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=0.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.Conv2d): # NOTE conv was left to pytorch default in my original init lecun_normal_(m.weight) if m.bias is not None: nn.init.zeros_(m.bias) elif isinstance(m, (nn.LayerNorm, nn.GroupNorm, nn.BatchNorm2d)): nn.init.zeros_(m.bias) nn.init.ones_(m.weight)

    class VisionMamba(nn.Module): def init(self, img_size=224, patch_size=16, stride=16, depth=24, embed_dim=192, channels=3, num_classes=1000, ssm_cfg=None, drop_rate=0., drop_path_rate=0.1, norm_epsilon: float = 1e-5, rms_norm: bool = False, initializer_cfg=None, fused_add_norm=False, residual_in_fp32=False, device=None, dtype=None, ft_seq_len=None, pt_hw_seq_len=14, if_bidirectional=False, final_pool_type='none', if_abs_pos_embed=False, if_rope=False, if_rope_residual=False, flip_img_sequences_ratio=-1., if_bimamba=False, bimamba_type="none", if_cls_token=False, if_devide_out=False, init_layer_scale=None, use_double_cls_token=False, use_middle_cls_token=False, **kwargs): factory_kwargs = {"device": device, "dtype": dtype} # add factory_kwargs into kwargs kwargs.update(factory_kwargs) super().init() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.if_bidirectional = if_bidirectional self.final_pool_type = final_pool_type self.if_abs_pos_embed = if_abs_pos_embed self.if_rope = if_rope self.if_rope_residual = if_rope_residual self.flip_img_sequences_ratio = flip_img_sequences_ratio # self.if_cls_token = if_cls_token self.if_cls_token = False self.use_double_cls_token = use_double_cls_token self.use_middle_cls_token = use_middle_cls_token self.num_tokens = 1 if if_cls_token else 0

        # pretrain parameters
        self.num_classes = num_classes
        self.d_model = self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
    
        self.patch_embed = PatchEmbed(
            img_size=img_size, patch_size=patch_size, stride=stride, in_chans=channels, embed_dim=embed_dim)
        num_patches = self.patch_embed.num_patches
    
        if if_cls_token:
            if use_double_cls_token:
                self.cls_token_head = nn.Parameter(torch.zeros(1, 1, self.embed_dim))
                self.cls_token_tail = nn.Parameter(torch.zeros(1, 1, self.embed_dim))
                self.num_tokens = 2
            else:
                self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embed_dim))
                # self.num_tokens = 1
            
        if if_abs_pos_embed:
            # self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, self.embed_dim))
            self.pos_embed_x = nn.Parameter(torch.zeros(1, 256, self.embed_dim))
            self.pos_embed_z = nn.Parameter(torch.zeros(1, 64, self.embed_dim))
            self.pos_drop = nn.Dropout(p=drop_rate)
    
        if if_rope:
            half_head_dim = embed_dim // 2
            hw_seq_len = img_size // patch_size
            self.rope = VisionRotaryEmbeddingFast(
                dim=half_head_dim,
                pt_seq_len=pt_hw_seq_len,
                ft_seq_len=hw_seq_len
            )
        # self.head = nn.Linear(self.num_features, num_classes) if num_classes > 0 else nn.Identity()
        # settings = ws_settings.Settings()
        # config_module = importlib.import_module("lib.config.%s.config" % settings.script_name)
        # cfg = config_module.cfg
        # self.head = build_box_head(cfg, embed_dim)
    
    
        # TODO: release this comment
        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
        # import ipdb;ipdb.set_trace()
        inter_dpr = [0.0] + dpr
        self.drop_path = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity()
                # transformer blocks
        self.layers = nn.ModuleList(
            [
                create_block(
                    embed_dim,
                    ssm_cfg=ssm_cfg,
                    norm_epsilon=norm_epsilon,
                    rms_norm=rms_norm,
                    residual_in_fp32=residual_in_fp32,
                    fused_add_norm=fused_add_norm,
                    layer_idx=i,
                    if_bimamba=if_bimamba,
                    bimamba_type=bimamba_type,
                    drop_path=inter_dpr[i],
                    if_devide_out=if_devide_out,
                    init_layer_scale=init_layer_scale,
                    **factory_kwargs,
                )
                for i in range(depth)
            ]
        )
        
        # output head
        self.norm_f = (nn.LayerNorm if not rms_norm else RMSNorm)(
            embed_dim, eps=norm_epsilon, **factory_kwargs
        )
    
        # self.pre_logits = nn.Identity()
    
        # original init
        self.patch_embed.apply(segm_init_weights)
        # self.head.apply(segm_init_weights)
        if if_abs_pos_embed:
            trunc_normal_(self.pos_embed_x, std=.02)
            trunc_normal_(self.pos_embed_z, std=.02)
            
        if if_cls_token:
            if use_double_cls_token:
                trunc_normal_(self.cls_token_head, std=.02)
                trunc_normal_(self.cls_token_tail, std=.02)
            else:
                trunc_normal_(self.cls_token, std=.02)
    
        # mamba init
        self.apply(
            partial(
                _init_weights,
                n_layer=depth,
                **(initializer_cfg if initializer_cfg is not None else {}),
            )
        )
    
    
    def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs):
        return {
            i: layer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)
            for i, layer in enumerate(self.layers)
        }
    
    @torch.jit.ignore
    def no_weight_decay(self):
        return {"pos_embed", "cls_token", "dist_token", "cls_token_head", "cls_token_tail"}
    
    @torch.jit.ignore()
    def load_pretrained(self, checkpoint_path, prefix=""):
        _load_weights(self, checkpoint_path, prefix)
    
    def forward_features(self, z, x, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False):
        # taken from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py
        # with slight modifications to add the dist_token
        x = self.patch_embed(x)                  #x.shape = torch.Size([B, 3, 256, 256])  -> torch.Size([2, 256, 384])
        z = self.patch_embed(z)                  #z.shape = torch.Size([B, 3, 128, 128])  -> torch.Size([2, 64, 384])
        B, M, _ = x.shape
       
        if self.if_cls_token:                 # False
            if self.use_double_cls_token:
                cls_token_head = self.cls_token_head.expand(B, -1, -1)
                cls_token_tail = self.cls_token_tail.expand(B, -1, -1)
                token_position = [0, M + 1]
                x = torch.cat((cls_token_head, x, cls_token_tail), dim=1)
                M = x.shape[1]
            else:
                if self.use_middle_cls_token:
                    cls_token = self.cls_token.expand(B, -1, -1)
                    token_position = M // 2
                    # add cls token in the middle
                    x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1)       
                elif if_random_cls_token_position:
                    cls_token = self.cls_token.expand(B, -1, -1)
                    token_position = random.randint(0, M)
                    x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1)
                    print("token_position: ", token_position)
                else:
                    cls_token = self.cls_token.expand(B, -1, -1)  # stole cls_tokens impl from Phil Wang, thanks
                    token_position = 0
                    x = torch.cat((cls_token, x), dim=1)
                M = x.shape[1]                 
       
        if self.if_abs_pos_embed:                  # True 
            x = x + self.pos_embed_x               # x = x + positon_embemding =torch.Size([B, 256, 384]) + torch.Size([1, 256, 384]) = torch.Size([B, 256, 384])
            z = z + self.pos_embed_z               # z = z + positon_embemding =torch.Size([B, 64, 384]) + torch.Size([1, 64, 384]) = torch.Size([B, 64, 384])
            x = torch.cat((z, x), dim=1)           # torch.Size([B, 320, 384])
            x = self.pos_drop(x)                   # x.shape = torch.Size([B, 320, 384])
            
        if if_random_token_rank:                   #False
            # 生成随机 shuffle 索引
            shuffle_indices = torch.randperm(M)
    
            if isinstance(token_position, list):
                print("original value: ", x[0, token_position[0], 0], x[0, token_position[1], 0])
            else:
                print("original value: ", x[0, token_position, 0])
            print("original token_position: ", token_position)
    
            # 执行 shuffle
            x = x[:, shuffle_indices, :]
    
            if isinstance(token_position, list):
                # 找到 cls token 在 shuffle 之后的新位置
                new_token_position = [torch.where(shuffle_indices == token_position[i])[0].item() for i in range(len(token_position))]
                token_position = new_token_position
            else:
                # 找到 cls token 在 shuffle 之后的新位置
                token_position = torch.where(shuffle_indices == token_position)[0].item()
    
            if isinstance(token_position, list):
                print("new value: ", x[0, token_position[0], 0], x[0, token_position[1], 0])
            else:
                print("new value: ", x[0, token_position, 0])
            print("new token_position: ", token_position)
    
    
        if_flip_img_sequences = False
        if self.flip_img_sequences_ratio > 0 and (self.flip_img_sequences_ratio - random.random()) > 1e-5:        # False
            x = x.flip([1])
            if_flip_img_sequences = True
    
        # mamba impl
        residual = None
        hidden_states = x
        if not self.if_bidirectional:                                 # True
            for layer in self.layers:
                if if_flip_img_sequences and self.if_rope:            # False
                    hidden_states = hidden_states.flip([1])
                    if residual is not None:
                        residual = residual.flip([1])
    
                # rope about
                if self.if_rope:                                       # False
                    hidden_states = self.rope(hidden_states)
                    if residual is not None and self.if_rope_residual:
                        residual = self.rope(residual)
    
                if if_flip_img_sequences and self.if_rope:             # False
                    hidden_states = hidden_states.flip([1])
                    if residual is not None:
                        residual = residual.flip([1])
    
                hidden_states, residual = layer(
                    hidden_states, residual, inference_params=inference_params
                )
        
        else:             # False
            # get two layers in a single for-loop
            for i in range(len(self.layers) // 2):
                if self.if_rope:
                    hidden_states = self.rope(hidden_states)
                    if residual is not None and self.if_rope_residual:
                        residual = self.rope(residual)
    
                hidden_states_f, residual_f = self.layers[i * 2](
                    hidden_states, residual, inference_params=inference_params
                )
                hidden_states_b, residual_b = self.layers[i * 2 + 1](
                    hidden_states.flip([1]), None if residual == None else residual.flip([1]), inference_params=inference_params
                )
                hidden_states = hidden_states_f + hidden_states_b.flip([1])
                residual = residual_f + residual_b.flip([1])
      
        if not self.fused_add_norm:         #False
            if residual is None:
                residual = hidden_states
            else:
                residual = residual + self.drop_path(hidden_states)
            hidden_states = self.norm_f(residual.to(dtype=self.norm_f.weight.dtype))
        else:       #True
            # Set prenorm=False here since we don't need the residual
            fused_add_norm_fn = rms_norm_fn if isinstance(self.norm_f, RMSNorm) else layer_norm_fn
            hidden_states = fused_add_norm_fn(                                         # hidden_states.shape = torch.Size([B, 320, 384])
                self.drop_path(hidden_states),
                self.norm_f.weight,
                self.norm_f.bias,
                eps=self.norm_f.eps,
                residual=residual,
                prenorm=False,
                residual_in_fp32=self.residual_in_fp32,
            )
    
        # return only cls token if it exists
        if self.if_cls_token:          #False
            if self.use_double_cls_token:
                return (hidden_states[:, token_position[0], :] + hidden_states[:, token_position[1], :]) / 2
            else:
                if self.use_middle_cls_token:
                    return hidden_states[:, token_position, :]
                elif if_random_cls_token_position:
                    return hidden_states[:, token_position, :]
                else:
                    return hidden_states[:, token_position, :]
    
        if self.final_pool_type == 'none':
            return hidden_states[:, -1, :]
        elif self.final_pool_type == 'mean':         #True
            return hidden_states.mean(dim=1)         #hidden_states.shape = torch.Size([2, 384])
        elif self.final_pool_type == 'max':
            return hidden_states
        elif self.final_pool_type == 'all':
            return hidden_states
        else:
            raise NotImplementedError
    
    def forward(self, x, return_features=False, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False):
        x = self.forward_features(x, inference_params, if_random_cls_token_position=if_random_cls_token_position, if_random_token_rank=if_random_token_rank)
        if return_features:
            return x
        x = self.head(x)
        if self.final_pool_type == 'max':
            x = x.max(dim=1)[0]
        return x
    

    @register_model def vim_tiny_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model

    @register_model def vim_tiny_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model

    @register_model def vim_small_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='all', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg()

    if pretrained:
        # checkpoint = torch.hub.load_state_dict_from_url(
        #     url="to.do",
        #     map_location="cpu", check_hash=True
        # )
        # model.load_state_dict(checkpoint["model"])
        checkpoint = torch.load(pretrained, map_location="cpu")
        missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False)
        print('Load pretrained model from: ' + pretrained)
    return model
    

    @register_model def vim_small_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model 和 """ Basic mamda_fetrack model. """ import math import os from typing import List

    import torch from torch import nn from torch.nn.modules.transformer import _get_clones

    from lib.models.layers.head import build_box_head from lib.utils.box_ops import box_xyxy_to_cxcywh from lib.models.mamba_fetrack.models_mamba import create_block from timm.models import create_model import torch.nn.functional as F from lib.models.mamba_fetrack.mamba_cross import CrossMamba from thop import profile

    class Mamba_FEtrack(nn.Module): """ This is the base class for mamda_fetrack """

    def __init__(self, visionmamba, cross_mamba, box_head, aux_loss=False, head_type="CORNER"):
        """ Initializes the model.
        Parameters:
            transformer: torch module of the transformer architecture.
            aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used.
        """
        super().__init__()
        self.backbone = visionmamba
        self.cross_mamba= cross_mamba
        self.box_head = box_head
    
        self.aux_loss = aux_loss
        self.head_type = head_type
        if head_type == "CORNER" or head_type == "CENTER":
            self.feat_sz_s = int(box_head.feat_sz)
            self.feat_len_s = int(box_head.feat_sz ** 2)
    
        if self.aux_loss:
            self.box_head = _get_clones(self.box_head, 6)
    
    def forward(self, template: torch.Tensor,
                search: torch.Tensor,
                event_template: torch.Tensor,       
                event_search: torch.Tensor,          
                ce_template_mask=None,
                ce_keep_rate=None,
                return_last_attn=False,
                ):
      
        rgb_feature = self.backbone.forward_features( z=template, x=search,                                                                     #[B, 320, 384]
                                                inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False)
        event_feature = self.backbone.forward_features(z=event_template, x=event_search,                                                        #[B, 320, 384]
                                                inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False)
    
        residual_event_f = 0
        residual_rgb_f = 0
        event_f = self.cross_mamba(event_feature,residual_event_f,rgb_feature) + event_feature
        rgb_f = self.cross_mamba(rgb_feature,residual_rgb_f,event_feature) + rgb_feature
        
        event_searh = event_f[:, -self.feat_len_s:]
        rgb_search = rgb_f[:, -self.feat_len_s:]
        x = torch.cat((event_searh,rgb_search),dim=-1)
        
        
        # Forward head
        feat_last = x             
        if isinstance(x, list):
            feat_last = x[-1]
        out = self.forward_head(feat_last, None)
       
        out['backbone_feat'] = x
        return out
    
    def forward_head(self, cat_feature, gt_score_map=None):
        """
        cat_feature: output embeddings of the backbone, it can be (HW1+HW2, B, C) or (HW2, B, C)
        """
        search_feature = cat_feature
        opt = (search_feature.unsqueeze(-1)).permute((0, 3, 2, 1)).contiguous()           # opt.shape = torch.Size([B, 1, 384, 256])
        bs, Nq, C, HW = opt.size()
        opt_feat = opt.view(-1, C, self.feat_sz_s, self.feat_sz_s)                       # opt_feat.shape = torch.Size([B, 384, 16, 16])
    
        if self.head_type == "CORNER":
            # run the corner head
            pred_box, score_map = self.box_head(opt_feat, True)
            outputs_coord = box_xyxy_to_cxcywh(pred_box)
            outputs_coord_new = outputs_coord.view(bs, Nq, 4)
            out = {'pred_boxes': outputs_coord_new,
                   'score_map': score_map,
                   }
            return out
    
        elif self.head_type == "CENTER":
            # run the center head
           
            score_map_ctr, bbox, size_map, offset_map = self.box_head(opt_feat, gt_score_map)
            outputs_coord = bbox
            outputs_coord_new = outputs_coord.view(bs, Nq, 4)
            out = {'pred_boxes': outputs_coord_new,
                   'score_map': score_map_ctr,
                   'size_map': size_map,
                   'offset_map': offset_map}
            return out
        else:
            raise NotImplementedError
    

    def build_mamba_fetrack(cfg, training=True): current_dir = os.path.dirname(os.path.abspath(file)) # This is your Project Root pretrained_path = os.path.join(current_dir, '../../../pretrained_models') if cfg.MODEL.PRETRAIN_FILE and ('Mamba_FETrack' not in cfg.MODEL.PRETRAIN_FILE) and training: pretrained = os.path.join(pretrained_path, cfg.MODEL.PRETRAIN_FILE) else: pretrained = ''

    backbone = create_model( model_name= cfg.MODEL.BACKBONE.TYPE, pretrained= pretrained, num_classes=1000,
            drop_rate=0.0, drop_path_rate=cfg.TRAIN.DROP_PATH_RATE, drop_block_rate=None, img_size=256
            )
    hidden_dim = 384
    cross_mamba = CrossMamba(hidden_dim)
    box_head = build_box_head(cfg, hidden_dim*2)
    model = Mamba_FEtrack(
        backbone,
        cross_mamba,
        box_head,
        aux_loss=False,
        head_type=cfg.MODEL.HEAD.TYPE,
    )
    
    if 'Mamba_FETrack' in cfg.MODEL.PRETRAIN_FILE and training:
        checkpoint = torch.load(cfg.MODEL.PRETRAIN_FILE, map_location="cpu")
        missing_keys, unexpected_keys = model.load_state_dict(checkpoint["net"], strict=False)
        print('Load pretrained model from: ' + cfg.MODEL.PRETRAIN_FILE)
    
    return model
    

    以及import math import torch import torch.nn as nn import torch.nn.functional as F from einops import rearrange import numbers from lib.models.mamba_fetrack.mamba_cross_simple import Mamba

    def to_3d(x): return rearrange(x, 'b c h w -> b (h w) c') def to_4d(x, h, w): return rearrange(x, 'b (h w) c -> b c h w', h=h, w=w)

    class BiasFree_LayerNorm(nn.Module): def init(self, normalized_shape): super(BiasFree_LayerNorm, self).init() if isinstance(normalized_shape, numbers.Integral): normalized_shape = (normalized_shape,) normalized_shape = torch.Size(normalized_shape)

        assert len(normalized_shape) == 1
    
        self.weight = nn.Parameter(torch.ones(normalized_shape))
        self.normalized_shape = normalized_shape
    
    def forward(self, x):
        sigma = x.var(-1, keepdim=True, unbiased=False)
        return x / torch.sqrt(sigma+1e-5) * self.weight
    

    class WithBias_LayerNorm(nn.Module): def init(self, normalized_shape): super(WithBias_LayerNorm, self).init() if isinstance(normalized_shape, numbers.Integral): normalized_shape = (normalized_shape,) normalized_shape = torch.Size(normalized_shape)

        assert len(normalized_shape) == 1
    
        self.weight = nn.Parameter(torch.ones(normalized_shape))
        self.bias = nn.Parameter(torch.zeros(normalized_shape))
        self.normalized_shape = normalized_shape
    
    def forward(self, x):
        mu = x.mean(-1, keepdim=True)
        sigma = x.var(-1, keepdim=True, unbiased=False)
        return (x - mu) / torch.sqrt(sigma+1e-5) * self.weight + self.bias
    

    class LayerNorm(nn.Module): def init(self, dim, LayerNorm_type): super(LayerNorm, self).init() if LayerNorm_type =='BiasFree': self.body = BiasFree_LayerNorm(dim) else: self.body = WithBias_LayerNorm(dim)

    def forward(self, x):
        h, w = x.shape[-2:]
        return to_4d(self.body(to_3d(x)), h, w)
    

    class CrossMamba(nn.Module): def init(self, dim): super(CrossMamba, self).init() self.cross_mamba = Mamba(dim,bimamba_type="v3") self.norm1 = LayerNorm(dim,'with_bias') self.norm2 = LayerNorm(dim,'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) def forward(self,m1,m1_resi,m2):
    m1_resi = m1+m1_resi
    global_f = self.cross_mamba(m1, extra_emb=m2) return global_f 。

  2. Assistant

    The provided code consists of various components likely from a vision transformer model implementation with a focus on a specific architecture related to "Mamba" and "FEtrack". These components include custom layers and blocks, patch embedding, block creation, forward functions for feature extraction, and specific network configurations with named registration models. Here is an overview and analysis of the critical sections:

    1. Patch Embedding (PatchEmbed): Converts a 2D image into patches that are then processed through a convolutional layer to create patch embeddings for input into a transformer model. This is a common step in Vision Transformers (ViT) to handle the input data.

    2. Block Architecture (Block, create_block): Represents individual blocks in the transformer model. Each block contains operations such as normalization, dropout, and a mixer (likely inspired by mixers in architecture variants such as MLP-Mixers).

    3. Vision Transformer (VisionMamba): An extended architecture for Vision Transformers. It includes additional functionalities such as using rotary embeddings (rope), absolute positional embeddings, and potential Support for bidirectionality and specialized pooling mechanisms.

    4. Weight Initialization: Functions for initializing weights specific to the architecture. Proper initialization is crucial for stabilizing the learning process in deep networks, especially those involving complex transformer layers.

    5. Model Definition and Registration: Several functions define complete architecture configurations with specific settings for patch size, stride, embedding dimension, etc. Named registration functions like vim_tiny_patch16_224_bimambav2... suggest different variants of models based on these configurations.

    6. Multi-modal Processing (Mamba_FEtrack): Combines features from both conventional images and event-driven data (possibly used in event-based cameras). It's designed to utilize cross-attention and blending of features from different modalities to improve the feature representations used for tasks like object detection or tracking.

    7. Cross Attention Module (CrossMamba): Implements interaction between dual modalities through a combination of transformer-based cross-attention and convolutional operations to refine feature maps from different inputs.

    8. Auxiliary Components and Custom Layers: Includes the alternative Layer Normalization techniques and specific operations like Depthwise Convolution.

    Overall, this setup is indicative of advanced architectures targeted towards multi-modal or multi-view data processing, typically used in scenarios requiring robust and diverse data assimilation, like event-based vision systems. The model heavily relies on transformers and cross-attention concepts to weave together different input streams to produce coherent and enriched output features essential for high-level vision tasks.

  3. User

    中文回答

  4. Assistant

    这段代码主要是关于一种称为“Mamba”及其变体的视觉Transformer模型的实现,可能用于复杂的视觉任务,比如物体检测或跟踪。以下是对代码关键部分的中文分析:

    1. Patch Embedding(补丁嵌入):PatchEmbed类将2D图像转换为小块(patch),并通过卷积层处理以创建可输入Transformer模型的嵌入。这是Vision Transformers(ViT)中的常见步骤,以便处理输入数据。

    2. 块结构(Block, create_block):每个块代表Transformer模型中的一个组成单位。每个块包含归一化、dropout以及一个混合器操作(可能受MLP-Mixers中混合器的影响)。

    3. 视觉Transformer(VisionMamba):扩展的Vision Transformer架构。该模型包括使用旋转嵌入(rotary embeddings)、绝对位置嵌入和双向性支持等功能。此外,还包括特定的池化机制。

    4. 权重初始化:提供专门用于此架构的权重初始化函数。正确的初始化对稳定深层网络的学习过程特别重要,尤其是涉及复杂的Transformer层时。

    5. 模型定义和注册:多个函数定义了完整的架构配置,具体设置如补丁大小、步幅、嵌入维度等。通过诸如vim_tiny_patch16_224_bimambav2...的命名注册函数标识不同模型变体,便于管理和调用。

    6. 多模态处理(Mamba_FEtrack):结合常规图像和事件驱动数据(可能用于事件摄像头),旨在通过跨注意力机制和不同模态特征的融合来提升特征表示能力,以便执行如物体检测或跟踪的任务。

    7. 跨注意力模块(CrossMamba):实现双方模态之间的交互,这通过Transformer的跨注意力和卷积操作的结合来优化不同输入的特征图。

    8. 辅助组件和自定义层:包括替代的层归一化技术和特定操作,如深度卷积。

    总体来看,此设置表明模型针对多模态或多视角数据处理进行了优化,通常用于需要可靠和多样的数据汇聚的场景,例如事件驱动的视觉系统。该模型主要依赖于Transformer和跨注意力概念,将不同的输入流编织在一起,以生成一致且丰富的输出特征,这对于高阶的视觉任务至关重要。

  5. User

    当前将crossMamba设计改成了:class BiasFree_LayerNorm(nn.Module): def init(self, normalized_shape): super(BiasFree_LayerNorm, self).init() if isinstance(normalized_shape, numbers.Integral): normalized_shape = (normalized_shape,) normalized_shape = torch.Size(normalized_shape)

        assert len(normalized_shape) == 1
    
        self.weight = nn.Parameter(torch.ones(normalized_shape))
        self.normalized_shape = normalized_shape
    
    def forward(self, x):
        sigma = x.var(-1, keepdim=True, unbiased=False)
        return x / torch.sqrt(sigma + 1e-5) * self.weight
    

    class WithBias_LayerNorm(nn.Module): def init(self, normalized_shape): super(WithBias_LayerNorm, self).init() if isinstance(normalized_shape, numbers.Integral): normalized_shape = (normalized_shape,) normalized_shape = torch.Size(normalized_shape)

        assert len(normalized_shape) == 1
    
        self.weight = nn.Parameter(torch.ones(normalized_shape))
        self.bias = nn.Parameter(torch.zeros(normalized_shape))
        self.normalized_shape = normalized_shape
    
    def forward(self, x):
        mu = x.mean(-1, keepdim=True)
        sigma = x.var(-1, keepdim=True, unbiased=False)
        return (x - mu) / torch.sqrt(sigma + 1e-5) * self.weight + self.bias
    

    class LayerNorm(nn.Module): def init(self, dim, LayerNorm_type): super(LayerNorm, self).init() if LayerNorm_type == 'BiasFree': self.body = BiasFree_LayerNorm(dim) else: self.body = WithBias_LayerNorm(dim)

    def forward(self, x):
        # h, w = x.shape[-2:]
        # return to_4d(self.body(to_3d(x)), h, w)
        return self.body(x)
    

    class DifferenceEnhancementNetwork(nn.Module): def init(self, dim): super(DifferenceEnhancementNetwork, self).init() self.fc1 = nn.Linear(dim, dim) self.fc2 = nn.Linear(dim, dim) self.activation = nn.ReLU()

    def forward(self, difference):
        enhanced_diff = self.activation(self.fc1(difference))
        enhanced_diff = self.activation(self.fc2(enhanced_diff))
        return enhanced_diff
    

    class FeatureAlignmentNetwork(nn.Module): def init(self, dim): super(FeatureAlignmentNetwork, self).init() self.project_m1 = nn.Linear(dim, dim) self.project_m2 = nn.Linear(dim, dim) self.norm = nn.LayerNorm(dim)

    def forward(self, m1, m2):
        m1_aligned = self.norm(self.project_m1(m1))
        m2_aligned = self.norm(self.project_m2(m2))
        return m1_aligned, m2_aligned
    

    class CrossMamba(nn.Module): def init(self, dim, dropout_rate=0.2): super(CrossMamba, self).init() self.cross_mamba = Mamba(dim, bimamba_type="v3") self.norm1 = LayerNorm(dim, 'with_bias') self.norm2 = LayerNorm(dim, 'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) self.diff_enhancement_net = DifferenceEnhancementNetwork(dim) self.alignment = FeatureAlignmentNetwork(dim) # Adding feature alignment self.dropout = nn.Dropout(dropout_rate) def forward(self, m1, m1_resi, m2): m1_aligned, m2_aligned = self.alignment(m1, m2) m1 = self.norm1(m1_aligned) m2 = self.norm2(m2_aligned) Fd = m1 - m2 Fd_enhanced = self.diff_enhancement_net(Fd) Fd_enhanced = self.dropout(Fd_enhanced) m1_resi = m1 + m1_resi global_f = self.cross_mamba(m1, extra_emb=m2 + Fd_enhanced) return global_f但是最终的跟踪指标并未提升,SR指标反而下降了,分析原因。

  6. Assistant

    设计新的 CrossMamba 组件时,你尝试通过引入特征差异增强和特征对齐网络来改善模型性能。尽管这些改动有潜力提升模型的特征表达能力,但在实现上仍可能存在一些问题,导致指标未提升甚至下降。以下是一些可能的原因以及建议:

    1. 特征对齐问题:

      • 引入特征对齐网络意味着对 m1 和 m2 进行线性变换和归一化,这可能会损失一些重要的特征信息,尤其是如果线性变换参数初始化不当或学习率设置不佳。
      • 尝试检查线性部分的初始化和优化过程,确保其参数能有效学习到输入特征的差异。
    2. 特征差异增强:

      • 差异增强网络尝试增强m1与m2之间的差异,但在网络的设计中,这么做可能会放大一些潜在的噪声或不相关的信息,从而影响模型的鲁棒性。
      • 需要确保增强部分专注于有效特征信息。可以评估这个部分的输出以检查增强效果,或者考虑增加正则化来抑制过拟合。
    3. 额外的参数和复杂度:

      • 引入更多的网络层和处理步骤增加了整个模型的复杂性和参数数量。这有可能导致过拟合特别是在样本量有限的情况下。
      • 此外,新的网络层还需要合适的学习率和正则化策略,以确保在训练过程中不走歪。
    4. Dropout的影响:

      • Dropout是一种有效的正则化方法,但使用不当可能会导致模型无法准确收敛。
      • 请确保Dropout率适当,并在不同上下文中(训练、评估)正确地启用或禁用。
    5. 超参数选择和训练过程:

      • 确保其他训练超参数,比如学习率和批量大小,是优化过的。
      • 新增层之后可能需要更长的训练时间来达到最佳表现。
    6. 网络集成和兼容性:

      • 新组件与现有网络其余部分的集成需要明确测试和整合,确保改动部分与原始设计或其他组件没有冲突。

    测试和验证可能主要从上述几个方面入手,通过部分地调整或回滚设计,或者逐个模块地检查变化的影响,以更深刻地了解到问题的根源并逐步改进设计。

  7. User

    那么分析其中最可能导致指标下降的操作是什么?

  8. Assistant

    在对模型设计进行修改并观察到性能下降的情况下,有几个关键因素可能导致指标下降。结合你提供的设计变更,以下是最可能导致性能下降的操作分析:

    1. 特征对齐网络 (FeatureAlignmentNetwork):

      • 该网络在m1和m2上应用线性变换,并随后进行LayerNorm操作。这可能会改变输入特征的原始值域或空间关系,从而丢失某些关键特征信息。这种对齐操作假设 m1 和 m2 可以通过简单的线性变换来对齐,但如果特征空间复杂或不易线性分离,可能导致对齐的效果不佳。
      • 为了解决这个问题,可以通过实验调整 project_m1 和 project_m2 的参数初值,或者使用非线性激活函数增加模型的复杂度和表达能力。
    2. 特征差异增强网络 (DifferenceEnhancementNetwork):

      • 差异增强的设计会着重于放大m1和m2之间的差异,这可能引入一些不希望放大的噪声或误差,特别是在特征之间本身就有噪声的情况下。
      • 过度增强可能会导致网络关注不重要的特征差异,而忽略整体重要的背景信息。
    3. Dropout使用:

      • 适当的Dropout可以防止过拟合,但过高的Dropout率可能导致特征信息过多丢失。特别是在对差异增强的部分应用Dropout时,可能使得增强后的特征变得不稳定。
      • 确保Dropout仅在适当的时候使用,并调整到适合你的模型和数据集的合理比例。

    从这些分析中,特征对齐网络的线性变换以及特征差异的过度增强可能是最可能导致性能下降的操作。建议你可以从简化这些模块或尝试不同参数初始化来入手,看是否能改善最终指标。在任何变更后,进行详细的实验和分析以观察具体影响也是很有必要的。

  9. User

    给出修改后的代码,先改一个可能导致的原因

  10. Assistant

    为了先测试和优化一个可能导致性能下降的原因,我们可以简化或调整特征对齐网络 (FeatureAlignmentNetwork)的实现。假设此模块中的线性变换对特征原始信息的损失起了主要作用,我们可以尝试如下修改以减少影响:

    1. 调整对齐网络的复杂度:
      • 在保持基本功能不变的情况下,简化线性投影的操作或者引入一种方式来保持原始特征信息。

    下面是修改后的代码示例:

    class FeatureAlignmentNetwork(nn.Module):
        def __init__(self, dim):
            super(FeatureAlignmentNetwork, self).__init__()
            # 将线性投影简化为仅一层,或者考虑不使用线性投影,只使用 normalization
            # self.project_m1 = nn.Linear(dim, dim)
            # self.project_m2 = nn.Linear(dim, dim)
            self.norm = nn.LayerNorm(dim)
    
        def forward(self, m1, m2):
            # m1_aligned = self.norm(self.project_m1(m1))
            # m2_aligned = self.norm(self.project_m2(m2))
            # 直接使用LayerNorm进行对齐,而非线性变换(可以作为调试的一步)
            m1_aligned = self.norm(m1)
            m2_aligned = self.norm(m2)
            return m1_aligned, m2_aligned
    
    class CrossMamba(nn.Module):
        def __init__(self, dim, dropout_rate=0.2):
            super(CrossMamba, self).__init__()
            self.cross_mamba = Mamba(dim, bimamba_type="v3")
            self.norm1 = LayerNorm(dim, 'with_bias')
            self.norm2 = LayerNorm(dim, 'with_bias')
            self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim)
            self.diff_enhancement_net = DifferenceEnhancementNetwork(dim)
            self.alignment = FeatureAlignmentNetwork(dim)  # 简化的特征对齐网络
            self.dropout = nn.Dropout(dropout_rate)
    
        def forward(self, m1, m1_resi, m2):
            m1_aligned, m2_aligned = self.alignment(m1, m2)  # 使用简化的对齐
            m1 = self.norm1(m1_aligned)
            m2 = self.norm2(m2_aligned)
            Fd = m1 - m2
            Fd_enhanced = self.diff_enhancement_net(Fd)
            Fd_enhanced = self.dropout(Fd_enhanced)
            m1_resi = m1 + m1_resi
            global_f = self.cross_mamba(m1, extra_emb=m2 + Fd_enhanced)
            return global_f
    

    在这种修改下,我们注释掉了对输入特征的线性变换,仅使用 LayerNorm 进行初步的对齐处理,从而试探对减少或调整特征信息的损失对最终模型表现的影响。请在训练和测试阶段观测结果,结合其他模块的表现改善对模型性能的影响。

  11. User

    那么讨论一下,m1_aligned, m2_aligned = self.alignment(m1, m2) # 使用简化的对齐 m1 = self.norm1(m1_aligned) m2 = self.norm2(m2_aligned),现在FeatureAlignmentNetwork设计的是nn.LayerNorm操作。在crossMamba中self.alignment后又做了self.norm操作,是否是重复操作?

  12. Assistant

    是的,观察到在 CrossMamba 的 forward 方法中,self.alignment 会调用 FeatureAlignmentNetwork 对 m1 和 m2 进行归一化操作(LayerNorm),然后再次在 self.norm1 和 self.norm2 中进行 LayerNorm 操作,这确实是一个重复操作。

    重复执行 LayerNorm 的影响包括:

    1. 重复归一化的影响:经过 LayerNorm 两次归一化,可能会抹去输入张量的某些重要数值特征,减弱特征的区分能力。
    2. 计算资源的浪费:在训练过程中,重复计算会增加不必要的计算开销和时间消耗。
    3. 潜在的数值不稳定性:每次 LayerNorm 都会重新计算均值和方差,这可能会导致数值不稳定(尤其是在小批量的情况下)。

    解决方案:

    • 如果 FeatureAlignmentNetwork 中的归一化足以满足对齐需求,可以移除 CrossMamba 中的 self.norm1 和 self.norm2 操作。
    • 如果认为 self.norm1 和 self.norm2 是绝对需要的,那么可以考虑 FeatureAlignmentNetwork 不进行 LayerNorm,仅对特征进行一些其他处理。

    以下是一种修正方式:

    class FeatureAlignmentNetwork(nn.Module):
        def __init__(self, dim):
            super(FeatureAlignmentNetwork, self).__init__()
            # 如果仅需要特征变换且不希望进行重复归一化,可以去掉 LayerNorm 操作
            # self.norm = nn.LayerNorm(dim)
        
        def forward(self, m1, m2):
            # 直接返回经过可能变换后的 m1 和 m2
            return m1, m2
    
    class CrossMamba(nn.Module):
        def __init__(self, dim, dropout_rate=0.2):
            super(CrossMamba, self).__init__()
            self.cross_mamba = Mamba(dim, bimamba_type="v3")
            self.norm1 = LayerNorm(dim, 'with_bias')
            self.norm2 = LayerNorm(dim, 'with_bias')
            self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim)
            self.diff_enhancement_net = DifferenceEnhancementNetwork(dim)
            self.alignment = FeatureAlignmentNetwork(dim)  # 不再进行对齐时的归一化
            self.dropout = nn.Dropout(dropout_rate)
    
        def forward(self, m1, m1_resi, m2):
            m1_aligned, m2_aligned = self.alignment(m1, m2)  # 使用不归一化的对齐
            m1 = self.norm1(m1_aligned)
            m2 = self.norm2(m2_aligned)
            Fd = m1 - m2
            Fd_enhanced = self.diff_enhancement_net(Fd)
            Fd_enhanced = self.dropout(Fd_enhanced)
            m1_resi = m1 + m1_resi
            global_f = self.cross_mamba(m1, extra_emb=m2 + Fd_enhanced)
            return global_f
    

    通过这种方式,你可以减少不必要的重复归一化,保持特征的稳定性和高效性。