分析模型:# Copyright (c) 2015-present, Facebook, Inc. All rights reserved. import torch…
- User
分析模型:# Copyright (c) 2015-present, Facebook, Inc.
All rights reserved.
import torch import torch.nn as nn from functools import partial from torch import Tensor from typing import Optional
from timm.models.vision_transformer import VisionTransformer, cfg from timm.models.registry import register_model from timm.models.layers import trunc_normal, lecun_normal_
from timm.models.layers import DropPath, to_2tuple from timm.models.vision_transformer import _load_weights
import math
from collections import namedtuple
from mamba_ssm.modules.mamba_simple import Mamba from mamba_ssm.utils.generation import GenerationMixin from mamba_ssm.utils.hf import load_config_hf, load_state_dict_hf
from lib.models.mamba_fetrack.rope import * import random from lib.models.layers.head import build_box_head import importlib import lib.train.admin.settings as ws_settings from .utils import combine_tokens, recover_tokens
try: from mamba_ssm.ops.triton.layernorm import RMSNorm, layer_norm_fn, rms_norm_fn except ImportError: RMSNorm, layer_norm_fn, rms_norm_fn = None, None, None
all = [ 'vim_tiny_patch16_224', 'vim_small_patch16_224', 'vim_base_patch16_224', 'vim_tiny_patch16_384', 'vim_small_patch16_384', 'vim_base_patch16_384', ]
class PatchEmbed(nn.Module): """ 2D Image to Patch Embedding """ def init(self, img_size=224, patch_size=16, stride=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True): super().init() img_size = to_2tuple(img_size) patch_size = to_2tuple(patch_size) self.img_size = img_size self.patch_size = patch_size self.grid_size = ((img_size[0] - patch_size[0]) // stride + 1, (img_size[1] - patch_size[1]) // stride + 1) self.num_patches = self.grid_size[0] * self.grid_size[1] self.flatten = flatten
self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=stride) self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() def forward(self, x): B, C, H, W = x.shape # assert H == self.img_size[0] and W == self.img_size[1], \ # f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." x = self.proj(x) if self.flatten: x = x.flatten(2).transpose(1, 2) # BCHW -> BNC x = self.norm(x) return xclass Block(nn.Module): def init( self, dim, mixer_cls, norm_cls=nn.LayerNorm, fused_add_norm=False, residual_in_fp32=False,drop_path=0., ): """ Simple block wrapping a mixer class with LayerNorm/RMSNorm and residual connection"
This Block has a slightly different structure compared to a regular prenorm Transformer block. The standard block is: LN -> MHA/MLP -> Add. [Ref: https://arxiv.org/abs/2002.04745] Here we have: Add -> LN -> Mixer, returning both the hidden_states (output of the mixer) and the residual. This is purely for performance reasons, as we can fuse add and LayerNorm. The residual needs to be provided (except for the very first block). """ super().__init__() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.mixer = mixer_cls(dim) self.norm = norm_cls(dim) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() if self.fused_add_norm: assert RMSNorm is not None, "RMSNorm import fails" assert isinstance( self.norm, (nn.LayerNorm, RMSNorm) ), "Only LayerNorm and RMSNorm are supported for fused_add_norm" def forward( self, hidden_states: Tensor, residual: Optional[Tensor] = None, inference_params=None ): r"""Pass the input through the encoder layer. Args: hidden_states: the sequence to the encoder layer (required). residual: hidden_states = Mixer(LN(residual)) """ if not self.fused_add_norm: if residual is None: residual = hidden_states else: residual = residual + self.drop_path(hidden_states) hidden_states = self.norm(residual.to(dtype=self.norm.weight.dtype)) if self.residual_in_fp32: residual = residual.to(torch.float32) else: fused_add_norm_fn = rms_norm_fn if isinstance(self.norm, RMSNorm) else layer_norm_fn if residual is None: hidden_states, residual = fused_add_norm_fn( hidden_states, self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) else: hidden_states, residual = fused_add_norm_fn( self.drop_path(hidden_states), self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) hidden_states = self.mixer(hidden_states, inference_params=inference_params) return hidden_states, residual def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return self.mixer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)def create_block( d_model, ssm_cfg=None, norm_epsilon=1e-5, drop_path=0., rms_norm=False, residual_in_fp32=False, fused_add_norm=False, layer_idx=None, device=None, dtype=None, if_bimamba=False, bimamba_type="none", if_devide_out=False, init_layer_scale=None, ): if if_bimamba: bimamba_type = "v1" if ssm_cfg is None: ssm_cfg = {} factory_kwargs = {"device": device, "dtype": dtype} mixer_cls = partial(Mamba, layer_idx=layer_idx, bimamba_type=bimamba_type, if_devide_out=if_devide_out, init_layer_scale=init_layer_scale, **ssm_cfg, **factory_kwargs) norm_cls = partial( nn.LayerNorm if not rms_norm else RMSNorm, eps=norm_epsilon, **factory_kwargs ) block = Block( d_model, mixer_cls, norm_cls=norm_cls, drop_path=drop_path, fused_add_norm=fused_add_norm, residual_in_fp32=residual_in_fp32, ) block.layer_idx = layer_idx return block
https://github.com/huggingface/transformers/blob/c28d04e9e252a1a099944e325685f14d242ecdcd/src/transformers/models/gpt2/modeling_gpt2.py#L454
def init_weights( module, n_layer, initializer_range=0.02, # Now only used for embedding layer. rescale_prenorm_residual=True, n_residuals_per_layer=1, # Change to 2 if we have MLP ): if isinstance(module, nn.Linear): if module.bias is not None: if not getattr(module.bias, "no_reinit", False): nn.init.zeros(module.bias) elif isinstance(module, nn.Embedding): nn.init.normal(module.weight, std=initializer_range)
if rescale_prenorm_residual: # Reinitialize selected weights subject to the OpenAI GPT-2 Paper Scheme: # > A modified initialization which accounts for the accumulation on the residual path with model depth. Scale # > the weights of residual layers at initialization by a factor of 1/√N where N is the # of residual layers. # > -- GPT-2 :: https://openai.com/blog/better-language-models/ # # Reference (Megatron-LM): https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/model/gpt_model.py for name, p in module.named_parameters(): if name in ["out_proj.weight", "fc2.weight"]: # Special Scaled Initialization --> There are 2 Layer Norms per Transformer Block # Following Pytorch init, except scale by 1/sqrt(2 * n_layer) # We need to reinit p since this code could be called multiple times # Having just p *= scale would repeatedly scale it down nn.init.kaiming_uniform_(p, a=math.sqrt(5)) with torch.no_grad(): p /= math.sqrt(n_residuals_per_layer * n_layer)def segm_init_weights(m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=0.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.Conv2d): # NOTE conv was left to pytorch default in my original init lecun_normal_(m.weight) if m.bias is not None: nn.init.zeros_(m.bias) elif isinstance(m, (nn.LayerNorm, nn.GroupNorm, nn.BatchNorm2d)): nn.init.zeros_(m.bias) nn.init.ones_(m.weight)
class VisionMamba(nn.Module): def init(self, img_size=224, patch_size=16, stride=16, depth=24, embed_dim=192, channels=3, num_classes=1000, ssm_cfg=None, drop_rate=0., drop_path_rate=0.1, norm_epsilon: float = 1e-5, rms_norm: bool = False, initializer_cfg=None, fused_add_norm=False, residual_in_fp32=False, device=None, dtype=None, ft_seq_len=None, pt_hw_seq_len=14, if_bidirectional=False, final_pool_type='none', if_abs_pos_embed=False, if_rope=False, if_rope_residual=False, flip_img_sequences_ratio=-1., if_bimamba=False, bimamba_type="none", if_cls_token=False, if_devide_out=False, init_layer_scale=None, use_double_cls_token=False, use_middle_cls_token=False, **kwargs): factory_kwargs = {"device": device, "dtype": dtype} # add factory_kwargs into kwargs kwargs.update(factory_kwargs) super().init() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.if_bidirectional = if_bidirectional self.final_pool_type = final_pool_type self.if_abs_pos_embed = if_abs_pos_embed self.if_rope = if_rope self.if_rope_residual = if_rope_residual self.flip_img_sequences_ratio = flip_img_sequences_ratio # self.if_cls_token = if_cls_token self.if_cls_token = False self.use_double_cls_token = use_double_cls_token self.use_middle_cls_token = use_middle_cls_token self.num_tokens = 1 if if_cls_token else 0
# pretrain parameters self.num_classes = num_classes self.d_model = self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models self.patch_embed = PatchEmbed( img_size=img_size, patch_size=patch_size, stride=stride, in_chans=channels, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches if if_cls_token: if use_double_cls_token: self.cls_token_head = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) self.cls_token_tail = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) self.num_tokens = 2 else: self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) # self.num_tokens = 1 if if_abs_pos_embed: # self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, self.embed_dim)) self.pos_embed_x = nn.Parameter(torch.zeros(1, 256, self.embed_dim)) self.pos_embed_z = nn.Parameter(torch.zeros(1, 64, self.embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) if if_rope: half_head_dim = embed_dim // 2 hw_seq_len = img_size // patch_size self.rope = VisionRotaryEmbeddingFast( dim=half_head_dim, pt_seq_len=pt_hw_seq_len, ft_seq_len=hw_seq_len ) # self.head = nn.Linear(self.num_features, num_classes) if num_classes > 0 else nn.Identity() # settings = ws_settings.Settings() # config_module = importlib.import_module("lib.config.%s.config" % settings.script_name) # cfg = config_module.cfg # self.head = build_box_head(cfg, embed_dim) # TODO: release this comment dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule # import ipdb;ipdb.set_trace() inter_dpr = [0.0] + dpr self.drop_path = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity() # transformer blocks self.layers = nn.ModuleList( [ create_block( embed_dim, ssm_cfg=ssm_cfg, norm_epsilon=norm_epsilon, rms_norm=rms_norm, residual_in_fp32=residual_in_fp32, fused_add_norm=fused_add_norm, layer_idx=i, if_bimamba=if_bimamba, bimamba_type=bimamba_type, drop_path=inter_dpr[i], if_devide_out=if_devide_out, init_layer_scale=init_layer_scale, **factory_kwargs, ) for i in range(depth) ] ) # output head self.norm_f = (nn.LayerNorm if not rms_norm else RMSNorm)( embed_dim, eps=norm_epsilon, **factory_kwargs ) # self.pre_logits = nn.Identity() # original init self.patch_embed.apply(segm_init_weights) # self.head.apply(segm_init_weights) if if_abs_pos_embed: trunc_normal_(self.pos_embed_x, std=.02) trunc_normal_(self.pos_embed_z, std=.02) if if_cls_token: if use_double_cls_token: trunc_normal_(self.cls_token_head, std=.02) trunc_normal_(self.cls_token_tail, std=.02) else: trunc_normal_(self.cls_token, std=.02) # mamba init self.apply( partial( _init_weights, n_layer=depth, **(initializer_cfg if initializer_cfg is not None else {}), ) ) def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return { i: layer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs) for i, layer in enumerate(self.layers) } @torch.jit.ignore def no_weight_decay(self): return {"pos_embed", "cls_token", "dist_token", "cls_token_head", "cls_token_tail"} @torch.jit.ignore() def load_pretrained(self, checkpoint_path, prefix=""): _load_weights(self, checkpoint_path, prefix) def forward_features(self, z, x, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False): # taken from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py # with slight modifications to add the dist_token x = self.patch_embed(x) #x.shape = torch.Size([B, 3, 256, 256]) -> torch.Size([2, 256, 384]) z = self.patch_embed(z) #z.shape = torch.Size([B, 3, 128, 128]) -> torch.Size([2, 64, 384]) B, M, _ = x.shape if self.if_cls_token: # False if self.use_double_cls_token: cls_token_head = self.cls_token_head.expand(B, -1, -1) cls_token_tail = self.cls_token_tail.expand(B, -1, -1) token_position = [0, M + 1] x = torch.cat((cls_token_head, x, cls_token_tail), dim=1) M = x.shape[1] else: if self.use_middle_cls_token: cls_token = self.cls_token.expand(B, -1, -1) token_position = M // 2 # add cls token in the middle x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1) elif if_random_cls_token_position: cls_token = self.cls_token.expand(B, -1, -1) token_position = random.randint(0, M) x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1) print("token_position: ", token_position) else: cls_token = self.cls_token.expand(B, -1, -1) # stole cls_tokens impl from Phil Wang, thanks token_position = 0 x = torch.cat((cls_token, x), dim=1) M = x.shape[1] if self.if_abs_pos_embed: # True x = x + self.pos_embed_x # x = x + positon_embemding =torch.Size([B, 256, 384]) + torch.Size([1, 256, 384]) = torch.Size([B, 256, 384]) z = z + self.pos_embed_z # z = z + positon_embemding =torch.Size([B, 64, 384]) + torch.Size([1, 64, 384]) = torch.Size([B, 64, 384]) x = torch.cat((z, x), dim=1) # torch.Size([B, 320, 384]) x = self.pos_drop(x) # x.shape = torch.Size([B, 320, 384]) if if_random_token_rank: #False # 生成随机 shuffle 索引 shuffle_indices = torch.randperm(M) if isinstance(token_position, list): print("original value: ", x[0, token_position[0], 0], x[0, token_position[1], 0]) else: print("original value: ", x[0, token_position, 0]) print("original token_position: ", token_position) # 执行 shuffle x = x[:, shuffle_indices, :] if isinstance(token_position, list): # 找到 cls token 在 shuffle 之后的新位置 new_token_position = [torch.where(shuffle_indices == token_position[i])[0].item() for i in range(len(token_position))] token_position = new_token_position else: # 找到 cls token 在 shuffle 之后的新位置 token_position = torch.where(shuffle_indices == token_position)[0].item() if isinstance(token_position, list): print("new value: ", x[0, token_position[0], 0], x[0, token_position[1], 0]) else: print("new value: ", x[0, token_position, 0]) print("new token_position: ", token_position) if_flip_img_sequences = False if self.flip_img_sequences_ratio > 0 and (self.flip_img_sequences_ratio - random.random()) > 1e-5: # False x = x.flip([1]) if_flip_img_sequences = True # mamba impl residual = None hidden_states = x if not self.if_bidirectional: # True for layer in self.layers: if if_flip_img_sequences and self.if_rope: # False hidden_states = hidden_states.flip([1]) if residual is not None: residual = residual.flip([1]) # rope about if self.if_rope: # False hidden_states = self.rope(hidden_states) if residual is not None and self.if_rope_residual: residual = self.rope(residual) if if_flip_img_sequences and self.if_rope: # False hidden_states = hidden_states.flip([1]) if residual is not None: residual = residual.flip([1]) hidden_states, residual = layer( hidden_states, residual, inference_params=inference_params ) else: # False # get two layers in a single for-loop for i in range(len(self.layers) // 2): if self.if_rope: hidden_states = self.rope(hidden_states) if residual is not None and self.if_rope_residual: residual = self.rope(residual) hidden_states_f, residual_f = self.layers[i * 2]( hidden_states, residual, inference_params=inference_params ) hidden_states_b, residual_b = self.layers[i * 2 + 1]( hidden_states.flip([1]), None if residual == None else residual.flip([1]), inference_params=inference_params ) hidden_states = hidden_states_f + hidden_states_b.flip([1]) residual = residual_f + residual_b.flip([1]) if not self.fused_add_norm: #False if residual is None: residual = hidden_states else: residual = residual + self.drop_path(hidden_states) hidden_states = self.norm_f(residual.to(dtype=self.norm_f.weight.dtype)) else: #True # Set prenorm=False here since we don't need the residual fused_add_norm_fn = rms_norm_fn if isinstance(self.norm_f, RMSNorm) else layer_norm_fn hidden_states = fused_add_norm_fn( # hidden_states.shape = torch.Size([B, 320, 384]) self.drop_path(hidden_states), self.norm_f.weight, self.norm_f.bias, eps=self.norm_f.eps, residual=residual, prenorm=False, residual_in_fp32=self.residual_in_fp32, ) # return only cls token if it exists if self.if_cls_token: #False if self.use_double_cls_token: return (hidden_states[:, token_position[0], :] + hidden_states[:, token_position[1], :]) / 2 else: if self.use_middle_cls_token: return hidden_states[:, token_position, :] elif if_random_cls_token_position: return hidden_states[:, token_position, :] else: return hidden_states[:, token_position, :] if self.final_pool_type == 'none': return hidden_states[:, -1, :] elif self.final_pool_type == 'mean': #True return hidden_states.mean(dim=1) #hidden_states.shape = torch.Size([2, 384]) elif self.final_pool_type == 'max': return hidden_states elif self.final_pool_type == 'all': return hidden_states else: raise NotImplementedError def forward(self, x, return_features=False, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False): x = self.forward_features(x, inference_params, if_random_cls_token_position=if_random_cls_token_position, if_random_token_rank=if_random_token_rank) if return_features: return x x = self.head(x) if self.final_pool_type == 'max': x = x.max(dim=1)[0] return x@register_model def vim_tiny_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model
@register_model def vim_tiny_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model
@register_model def vim_small_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='all', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg()
if pretrained: # checkpoint = torch.hub.load_state_dict_from_url( # url="to.do", # map_location="cpu", check_hash=True # ) # model.load_state_dict(checkpoint["model"]) checkpoint = torch.load(pretrained, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False) print('Load pretrained model from: ' + pretrained) return model@register_model def vim_small_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model 和 """ Basic mamda_fetrack model. """ import math import os from typing import List
import torch from torch import nn from torch.nn.modules.transformer import _get_clones
from lib.models.layers.head import build_box_head from lib.utils.box_ops import box_xyxy_to_cxcywh from lib.models.mamba_fetrack.models_mamba import create_block from timm.models import create_model import torch.nn.functional as F from lib.models.mamba_fetrack.mamba_cross import CrossMamba from thop import profile
class Mamba_FEtrack(nn.Module): """ This is the base class for mamda_fetrack """
def __init__(self, visionmamba, cross_mamba, box_head, aux_loss=False, head_type="CORNER"): """ Initializes the model. Parameters: transformer: torch module of the transformer architecture. aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. """ super().__init__() self.backbone = visionmamba self.cross_mamba= cross_mamba self.box_head = box_head self.aux_loss = aux_loss self.head_type = head_type if head_type == "CORNER" or head_type == "CENTER": self.feat_sz_s = int(box_head.feat_sz) self.feat_len_s = int(box_head.feat_sz ** 2) if self.aux_loss: self.box_head = _get_clones(self.box_head, 6) def forward(self, template: torch.Tensor, search: torch.Tensor, event_template: torch.Tensor, event_search: torch.Tensor, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False, ): rgb_feature = self.backbone.forward_features( z=template, x=search, #[B, 320, 384] inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False) event_feature = self.backbone.forward_features(z=event_template, x=event_search, #[B, 320, 384] inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False) residual_event_f = 0 residual_rgb_f = 0 event_f = self.cross_mamba(event_feature,residual_event_f,rgb_feature) + event_feature rgb_f = self.cross_mamba(rgb_feature,residual_rgb_f,event_feature) + rgb_feature event_searh = event_f[:, -self.feat_len_s:] rgb_search = rgb_f[:, -self.feat_len_s:] x = torch.cat((event_searh,rgb_search),dim=-1) # Forward head feat_last = x if isinstance(x, list): feat_last = x[-1] out = self.forward_head(feat_last, None) out['backbone_feat'] = x return out def forward_head(self, cat_feature, gt_score_map=None): """ cat_feature: output embeddings of the backbone, it can be (HW1+HW2, B, C) or (HW2, B, C) """ search_feature = cat_feature opt = (search_feature.unsqueeze(-1)).permute((0, 3, 2, 1)).contiguous() # opt.shape = torch.Size([B, 1, 384, 256]) bs, Nq, C, HW = opt.size() opt_feat = opt.view(-1, C, self.feat_sz_s, self.feat_sz_s) # opt_feat.shape = torch.Size([B, 384, 16, 16]) if self.head_type == "CORNER": # run the corner head pred_box, score_map = self.box_head(opt_feat, True) outputs_coord = box_xyxy_to_cxcywh(pred_box) outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map, } return out elif self.head_type == "CENTER": # run the center head score_map_ctr, bbox, size_map, offset_map = self.box_head(opt_feat, gt_score_map) outputs_coord = bbox outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map_ctr, 'size_map': size_map, 'offset_map': offset_map} return out else: raise NotImplementedErrordef build_mamba_fetrack(cfg, training=True): current_dir = os.path.dirname(os.path.abspath(file)) # This is your Project Root pretrained_path = os.path.join(current_dir, '../../../pretrained_models') if cfg.MODEL.PRETRAIN_FILE and ('Mamba_FETrack' not in cfg.MODEL.PRETRAIN_FILE) and training: pretrained = os.path.join(pretrained_path, cfg.MODEL.PRETRAIN_FILE) else: pretrained = ''
backbone = create_model( model_name= cfg.MODEL.BACKBONE.TYPE, pretrained= pretrained, num_classes=1000, drop_rate=0.0, drop_path_rate=cfg.TRAIN.DROP_PATH_RATE, drop_block_rate=None, img_size=256 ) hidden_dim = 384 cross_mamba = CrossMamba(hidden_dim) box_head = build_box_head(cfg, hidden_dim*2) model = Mamba_FEtrack( backbone, cross_mamba, box_head, aux_loss=False, head_type=cfg.MODEL.HEAD.TYPE, ) if 'Mamba_FETrack' in cfg.MODEL.PRETRAIN_FILE and training: checkpoint = torch.load(cfg.MODEL.PRETRAIN_FILE, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["net"], strict=False) print('Load pretrained model from: ' + cfg.MODEL.PRETRAIN_FILE) return model以及import math import torch import torch.nn as nn import torch.nn.functional as F from einops import rearrange import numbers from lib.models.mamba_fetrack.mamba_cross_simple import Mamba
def to_3d(x): return rearrange(x, 'b c h w -> b (h w) c') def to_4d(x, h, w): return rearrange(x, 'b (h w) c -> b c h w', h=h, w=w)
class BiasFree_LayerNorm(nn.Module): def init(self, normalized_shape): super(BiasFree_LayerNorm, self).init() if isinstance(normalized_shape, numbers.Integral): normalized_shape = (normalized_shape,) normalized_shape = torch.Size(normalized_shape)
assert len(normalized_shape) == 1 self.weight = nn.Parameter(torch.ones(normalized_shape)) self.normalized_shape = normalized_shape def forward(self, x): sigma = x.var(-1, keepdim=True, unbiased=False) return x / torch.sqrt(sigma+1e-5) * self.weightclass WithBias_LayerNorm(nn.Module): def init(self, normalized_shape): super(WithBias_LayerNorm, self).init() if isinstance(normalized_shape, numbers.Integral): normalized_shape = (normalized_shape,) normalized_shape = torch.Size(normalized_shape)
assert len(normalized_shape) == 1 self.weight = nn.Parameter(torch.ones(normalized_shape)) self.bias = nn.Parameter(torch.zeros(normalized_shape)) self.normalized_shape = normalized_shape def forward(self, x): mu = x.mean(-1, keepdim=True) sigma = x.var(-1, keepdim=True, unbiased=False) return (x - mu) / torch.sqrt(sigma+1e-5) * self.weight + self.biasclass LayerNorm(nn.Module): def init(self, dim, LayerNorm_type): super(LayerNorm, self).init() if LayerNorm_type =='BiasFree': self.body = BiasFree_LayerNorm(dim) else: self.body = WithBias_LayerNorm(dim)
def forward(self, x): h, w = x.shape[-2:] return to_4d(self.body(to_3d(x)), h, w)class CrossMamba(nn.Module): def init(self, dim): super(CrossMamba, self).init() self.cross_mamba = Mamba(dim,bimamba_type="v3") self.norm1 = LayerNorm(dim,'with_bias') self.norm2 = LayerNorm(dim,'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) def forward(self,m1,m1_resi,m2):
m1_resi = m1+m1_resi
global_f = self.cross_mamba(m1, extra_emb=m2) return global_f 。 - Assistant
The provided code consists of various components likely from a vision transformer model implementation with a focus on a specific architecture related to "Mamba" and "FEtrack". These components include custom layers and blocks, patch embedding, block creation, forward functions for feature extraction, and specific network configurations with named registration models. Here is an overview and analysis of the critical sections:
-
Patch Embedding (
PatchEmbed): Converts a 2D image into patches that are then processed through a convolutional layer to create patch embeddings for input into a transformer model. This is a common step in Vision Transformers (ViT) to handle the input data. -
Block Architecture (
Block,create_block): Represents individual blocks in the transformer model. Each block contains operations such as normalization, dropout, and a mixer (likely inspired by mixers in architecture variants such as MLP-Mixers). -
Vision Transformer (
VisionMamba): An extended architecture for Vision Transformers. It includes additional functionalities such as using rotary embeddings (rope), absolute positional embeddings, and potential Support for bidirectionality and specialized pooling mechanisms. -
Weight Initialization: Functions for initializing weights specific to the architecture. Proper initialization is crucial for stabilizing the learning process in deep networks, especially those involving complex transformer layers.
-
Model Definition and Registration: Several functions define complete architecture configurations with specific settings for patch size, stride, embedding dimension, etc. Named registration functions like
vim_tiny_patch16_224_bimambav2...suggest different variants of models based on these configurations. -
Multi-modal Processing (
Mamba_FEtrack): Combines features from both conventional images and event-driven data (possibly used in event-based cameras). It's designed to utilize cross-attention and blending of features from different modalities to improve the feature representations used for tasks like object detection or tracking. -
Cross Attention Module (
CrossMamba): Implements interaction between dual modalities through a combination of transformer-based cross-attention and convolutional operations to refine feature maps from different inputs. -
Auxiliary Components and Custom Layers: Includes the alternative Layer Normalization techniques and specific operations like Depthwise Convolution.
Overall, this setup is indicative of advanced architectures targeted towards multi-modal or multi-view data processing, typically used in scenarios requiring robust and diverse data assimilation, like event-based vision systems. The model heavily relies on transformers and cross-attention concepts to weave together different input streams to produce coherent and enriched output features essential for high-level vision tasks.
-
- User
中文回答
- Assistant
这段代码主要是关于一种称为“Mamba”及其变体的视觉Transformer模型的实现,可能用于复杂的视觉任务,比如物体检测或跟踪。以下是对代码关键部分的中文分析:
-
Patch Embedding(补丁嵌入):
PatchEmbed类将2D图像转换为小块(patch),并通过卷积层处理以创建可输入Transformer模型的嵌入。这是Vision Transformers(ViT)中的常见步骤,以便处理输入数据。 -
块结构(Block, create_block):每个块代表Transformer模型中的一个组成单位。每个块包含归一化、dropout以及一个混合器操作(可能受MLP-Mixers中混合器的影响)。
-
视觉Transformer(VisionMamba):扩展的Vision Transformer架构。该模型包括使用旋转嵌入(rotary embeddings)、绝对位置嵌入和双向性支持等功能。此外,还包括特定的池化机制。
-
权重初始化:提供专门用于此架构的权重初始化函数。正确的初始化对稳定深层网络的学习过程特别重要,尤其是涉及复杂的Transformer层时。
-
模型定义和注册:多个函数定义了完整的架构配置,具体设置如补丁大小、步幅、嵌入维度等。通过诸如
vim_tiny_patch16_224_bimambav2...的命名注册函数标识不同模型变体,便于管理和调用。 -
多模态处理(Mamba_FEtrack):结合常规图像和事件驱动数据(可能用于事件摄像头),旨在通过跨注意力机制和不同模态特征的融合来提升特征表示能力,以便执行如物体检测或跟踪的任务。
-
跨注意力模块(CrossMamba):实现双方模态之间的交互,这通过Transformer的跨注意力和卷积操作的结合来优化不同输入的特征图。
-
辅助组件和自定义层:包括替代的层归一化技术和特定操作,如深度卷积。
总体来看,此设置表明模型针对多模态或多视角数据处理进行了优化,通常用于需要可靠和多样的数据汇聚的场景,例如事件驱动的视觉系统。该模型主要依赖于Transformer和跨注意力概念,将不同的输入流编织在一起,以生成一致且丰富的输出特征,这对于高阶的视觉任务至关重要。
-
- User
当前将crossMamba设计改成了:class BiasFree_LayerNorm(nn.Module): def init(self, normalized_shape): super(BiasFree_LayerNorm, self).init() if isinstance(normalized_shape, numbers.Integral): normalized_shape = (normalized_shape,) normalized_shape = torch.Size(normalized_shape)
assert len(normalized_shape) == 1 self.weight = nn.Parameter(torch.ones(normalized_shape)) self.normalized_shape = normalized_shape def forward(self, x): sigma = x.var(-1, keepdim=True, unbiased=False) return x / torch.sqrt(sigma + 1e-5) * self.weightclass WithBias_LayerNorm(nn.Module): def init(self, normalized_shape): super(WithBias_LayerNorm, self).init() if isinstance(normalized_shape, numbers.Integral): normalized_shape = (normalized_shape,) normalized_shape = torch.Size(normalized_shape)
assert len(normalized_shape) == 1 self.weight = nn.Parameter(torch.ones(normalized_shape)) self.bias = nn.Parameter(torch.zeros(normalized_shape)) self.normalized_shape = normalized_shape def forward(self, x): mu = x.mean(-1, keepdim=True) sigma = x.var(-1, keepdim=True, unbiased=False) return (x - mu) / torch.sqrt(sigma + 1e-5) * self.weight + self.biasclass LayerNorm(nn.Module): def init(self, dim, LayerNorm_type): super(LayerNorm, self).init() if LayerNorm_type == 'BiasFree': self.body = BiasFree_LayerNorm(dim) else: self.body = WithBias_LayerNorm(dim)
def forward(self, x): # h, w = x.shape[-2:] # return to_4d(self.body(to_3d(x)), h, w) return self.body(x)class DifferenceEnhancementNetwork(nn.Module): def init(self, dim): super(DifferenceEnhancementNetwork, self).init() self.fc1 = nn.Linear(dim, dim) self.fc2 = nn.Linear(dim, dim) self.activation = nn.ReLU()
def forward(self, difference): enhanced_diff = self.activation(self.fc1(difference)) enhanced_diff = self.activation(self.fc2(enhanced_diff)) return enhanced_diffclass FeatureAlignmentNetwork(nn.Module): def init(self, dim): super(FeatureAlignmentNetwork, self).init() self.project_m1 = nn.Linear(dim, dim) self.project_m2 = nn.Linear(dim, dim) self.norm = nn.LayerNorm(dim)
def forward(self, m1, m2): m1_aligned = self.norm(self.project_m1(m1)) m2_aligned = self.norm(self.project_m2(m2)) return m1_aligned, m2_alignedclass CrossMamba(nn.Module): def init(self, dim, dropout_rate=0.2): super(CrossMamba, self).init() self.cross_mamba = Mamba(dim, bimamba_type="v3") self.norm1 = LayerNorm(dim, 'with_bias') self.norm2 = LayerNorm(dim, 'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) self.diff_enhancement_net = DifferenceEnhancementNetwork(dim) self.alignment = FeatureAlignmentNetwork(dim) # Adding feature alignment self.dropout = nn.Dropout(dropout_rate) def forward(self, m1, m1_resi, m2): m1_aligned, m2_aligned = self.alignment(m1, m2) m1 = self.norm1(m1_aligned) m2 = self.norm2(m2_aligned) Fd = m1 - m2 Fd_enhanced = self.diff_enhancement_net(Fd) Fd_enhanced = self.dropout(Fd_enhanced) m1_resi = m1 + m1_resi global_f = self.cross_mamba(m1, extra_emb=m2 + Fd_enhanced) return global_f但是最终的跟踪指标并未提升,SR指标反而下降了,分析原因。
- Assistant
设计新的
CrossMamba组件时,你尝试通过引入特征差异增强和特征对齐网络来改善模型性能。尽管这些改动有潜力提升模型的特征表达能力,但在实现上仍可能存在一些问题,导致指标未提升甚至下降。以下是一些可能的原因以及建议:-
特征对齐问题:
- 引入特征对齐网络意味着对
m1和m2进行线性变换和归一化,这可能会损失一些重要的特征信息,尤其是如果线性变换参数初始化不当或学习率设置不佳。 - 尝试检查线性部分的初始化和优化过程,确保其参数能有效学习到输入特征的差异。
- 引入特征对齐网络意味着对
-
特征差异增强:
- 差异增强网络尝试增强m1与m2之间的差异,但在网络的设计中,这么做可能会放大一些潜在的噪声或不相关的信息,从而影响模型的鲁棒性。
- 需要确保增强部分专注于有效特征信息。可以评估这个部分的输出以检查增强效果,或者考虑增加正则化来抑制过拟合。
-
额外的参数和复杂度:
- 引入更多的网络层和处理步骤增加了整个模型的复杂性和参数数量。这有可能导致过拟合特别是在样本量有限的情况下。
- 此外,新的网络层还需要合适的学习率和正则化策略,以确保在训练过程中不走歪。
-
Dropout的影响:
- Dropout是一种有效的正则化方法,但使用不当可能会导致模型无法准确收敛。
- 请确保Dropout率适当,并在不同上下文中(训练、评估)正确地启用或禁用。
-
超参数选择和训练过程:
- 确保其他训练超参数,比如学习率和批量大小,是优化过的。
- 新增层之后可能需要更长的训练时间来达到最佳表现。
-
网络集成和兼容性:
- 新组件与现有网络其余部分的集成需要明确测试和整合,确保改动部分与原始设计或其他组件没有冲突。
测试和验证可能主要从上述几个方面入手,通过部分地调整或回滚设计,或者逐个模块地检查变化的影响,以更深刻地了解到问题的根源并逐步改进设计。
-
- User
那么分析其中最可能导致指标下降的操作是什么?
- Assistant
在对模型设计进行修改并观察到性能下降的情况下,有几个关键因素可能导致指标下降。结合你提供的设计变更,以下是最可能导致性能下降的操作分析:
-
特征对齐网络 (
FeatureAlignmentNetwork):- 该网络在
m1和m2上应用线性变换,并随后进行LayerNorm操作。这可能会改变输入特征的原始值域或空间关系,从而丢失某些关键特征信息。这种对齐操作假设m1和m2可以通过简单的线性变换来对齐,但如果特征空间复杂或不易线性分离,可能导致对齐的效果不佳。 - 为了解决这个问题,可以通过实验调整
project_m1和project_m2的参数初值,或者使用非线性激活函数增加模型的复杂度和表达能力。
- 该网络在
-
特征差异增强网络 (
DifferenceEnhancementNetwork):- 差异增强的设计会着重于放大
m1和m2之间的差异,这可能引入一些不希望放大的噪声或误差,特别是在特征之间本身就有噪声的情况下。 - 过度增强可能会导致网络关注不重要的特征差异,而忽略整体重要的背景信息。
- 差异增强的设计会着重于放大
-
Dropout使用:
- 适当的Dropout可以防止过拟合,但过高的Dropout率可能导致特征信息过多丢失。特别是在对差异增强的部分应用Dropout时,可能使得增强后的特征变得不稳定。
- 确保Dropout仅在适当的时候使用,并调整到适合你的模型和数据集的合理比例。
从这些分析中,特征对齐网络的线性变换以及特征差异的过度增强可能是最可能导致性能下降的操作。建议你可以从简化这些模块或尝试不同参数初始化来入手,看是否能改善最终指标。在任何变更后,进行详细的实验和分析以观察具体影响也是很有必要的。
-
- User
给出修改后的代码,先改一个可能导致的原因
- Assistant
为了先测试和优化一个可能导致性能下降的原因,我们可以简化或调整特征对齐网络 (
FeatureAlignmentNetwork)的实现。假设此模块中的线性变换对特征原始信息的损失起了主要作用,我们可以尝试如下修改以减少影响:- 调整对齐网络的复杂度:
- 在保持基本功能不变的情况下,简化线性投影的操作或者引入一种方式来保持原始特征信息。
下面是修改后的代码示例:
class FeatureAlignmentNetwork(nn.Module): def __init__(self, dim): super(FeatureAlignmentNetwork, self).__init__() # 将线性投影简化为仅一层,或者考虑不使用线性投影,只使用 normalization # self.project_m1 = nn.Linear(dim, dim) # self.project_m2 = nn.Linear(dim, dim) self.norm = nn.LayerNorm(dim) def forward(self, m1, m2): # m1_aligned = self.norm(self.project_m1(m1)) # m2_aligned = self.norm(self.project_m2(m2)) # 直接使用LayerNorm进行对齐,而非线性变换(可以作为调试的一步) m1_aligned = self.norm(m1) m2_aligned = self.norm(m2) return m1_aligned, m2_aligned class CrossMamba(nn.Module): def __init__(self, dim, dropout_rate=0.2): super(CrossMamba, self).__init__() self.cross_mamba = Mamba(dim, bimamba_type="v3") self.norm1 = LayerNorm(dim, 'with_bias') self.norm2 = LayerNorm(dim, 'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) self.diff_enhancement_net = DifferenceEnhancementNetwork(dim) self.alignment = FeatureAlignmentNetwork(dim) # 简化的特征对齐网络 self.dropout = nn.Dropout(dropout_rate) def forward(self, m1, m1_resi, m2): m1_aligned, m2_aligned = self.alignment(m1, m2) # 使用简化的对齐 m1 = self.norm1(m1_aligned) m2 = self.norm2(m2_aligned) Fd = m1 - m2 Fd_enhanced = self.diff_enhancement_net(Fd) Fd_enhanced = self.dropout(Fd_enhanced) m1_resi = m1 + m1_resi global_f = self.cross_mamba(m1, extra_emb=m2 + Fd_enhanced) return global_f在这种修改下,我们注释掉了对输入特征的线性变换,仅使用
LayerNorm进行初步的对齐处理,从而试探对减少或调整特征信息的损失对最终模型表现的影响。请在训练和测试阶段观测结果,结合其他模块的表现改善对模型性能的影响。 - 调整对齐网络的复杂度:
- User
那么讨论一下,m1_aligned, m2_aligned = self.alignment(m1, m2) # 使用简化的对齐 m1 = self.norm1(m1_aligned) m2 = self.norm2(m2_aligned),现在FeatureAlignmentNetwork设计的是nn.LayerNorm操作。在crossMamba中self.alignment后又做了self.norm操作,是否是重复操作?
- Assistant
是的,观察到在
CrossMamba的forward方法中,self.alignment会调用FeatureAlignmentNetwork对m1和m2进行归一化操作(LayerNorm),然后再次在self.norm1和self.norm2中进行LayerNorm操作,这确实是一个重复操作。重复执行
LayerNorm的影响包括:- 重复归一化的影响:经过
LayerNorm两次归一化,可能会抹去输入张量的某些重要数值特征,减弱特征的区分能力。 - 计算资源的浪费:在训练过程中,重复计算会增加不必要的计算开销和时间消耗。
- 潜在的数值不稳定性:每次
LayerNorm都会重新计算均值和方差,这可能会导致数值不稳定(尤其是在小批量的情况下)。
解决方案:
- 如果
FeatureAlignmentNetwork中的归一化足以满足对齐需求,可以移除CrossMamba中的self.norm1和self.norm2操作。 - 如果认为
self.norm1和self.norm2是绝对需要的,那么可以考虑FeatureAlignmentNetwork不进行LayerNorm,仅对特征进行一些其他处理。
以下是一种修正方式:
class FeatureAlignmentNetwork(nn.Module): def __init__(self, dim): super(FeatureAlignmentNetwork, self).__init__() # 如果仅需要特征变换且不希望进行重复归一化,可以去掉 LayerNorm 操作 # self.norm = nn.LayerNorm(dim) def forward(self, m1, m2): # 直接返回经过可能变换后的 m1 和 m2 return m1, m2 class CrossMamba(nn.Module): def __init__(self, dim, dropout_rate=0.2): super(CrossMamba, self).__init__() self.cross_mamba = Mamba(dim, bimamba_type="v3") self.norm1 = LayerNorm(dim, 'with_bias') self.norm2 = LayerNorm(dim, 'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) self.diff_enhancement_net = DifferenceEnhancementNetwork(dim) self.alignment = FeatureAlignmentNetwork(dim) # 不再进行对齐时的归一化 self.dropout = nn.Dropout(dropout_rate) def forward(self, m1, m1_resi, m2): m1_aligned, m2_aligned = self.alignment(m1, m2) # 使用不归一化的对齐 m1 = self.norm1(m1_aligned) m2 = self.norm2(m2_aligned) Fd = m1 - m2 Fd_enhanced = self.diff_enhancement_net(Fd) Fd_enhanced = self.dropout(Fd_enhanced) m1_resi = m1 + m1_resi global_f = self.cross_mamba(m1, extra_emb=m2 + Fd_enhanced) return global_f通过这种方式,你可以减少不必要的重复归一化,保持特征的稳定性和高效性。
- 重复归一化的影响:经过