那么分析一下:import torch import torch.nn as nn from functools import partial from torch…
- User
那么分析一下:import torch import torch.nn as nn from functools import partial from torch import Tensor from typing import Optional
from timm.models.vision_transformer import cfg from timm.models.registry import register_model from timm.models.layers import trunc_normal, lecun_normal_
from timm.models.layers import DropPath, to_2tuple from timm.models.vision_transformer import _load_weights
import math
from collections import namedtuple
from mamba_ssm.modules.mamba_simple import Mamba from mamba_ssm.utils.generation import GenerationMixin from mamba_ssm.utils.hf import load_config_hf, load_state_dict_hf #################### 结合当前项目引入 ######################### from lib.models.mamba_fetrack.rope import * import random ################################################################# try: from mamba_ssm.ops.triton.layernorm import RMSNorm, layer_norm_fn, rms_norm_fn except ImportError: RMSNorm, layer_norm_fn, rms_norm_fn = None, None, None
class PatchEmbed(nn.Module): """ 2D Image to Patch Embedding """ def init(self, img_size=224, patch_size=16, stride=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True): super().init() img_size = to_2tuple(img_size) patch_size = to_2tuple(patch_size) self.img_size = img_size self.patch_size = patch_size self.grid_size = ((img_size[0] - patch_size[0]) // stride + 1, (img_size[1] - patch_size[1]) // stride + 1) self.num_patches = self.grid_size[0] * self.grid_size[1] self.flatten = flatten
self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=stride) self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() def forward(self, x): B, C, H, W = x.shape x = self.proj(x) if self.flatten: x = x.flatten(2).transpose(1, 2) # BCHW -> BNC x = self.norm(x) return xclass Block(nn.Module): def init( self, dim, mixer_cls, norm_cls=nn.LayerNorm, fused_add_norm=False, residual_in_fp32=False,drop_path=0., ): """ Simple block wrapping a mixer class with LayerNorm/RMSNorm and residual connection"
This Block has a slightly different structure compared to a regular prenorm Transformer block. The standard block is: LN -> MHA/MLP -> Add. [Ref: https://arxiv.org/abs/2002.04745] Here we have: Add -> LN -> Mixer, returning both the hidden_states (output of the mixer) and the residual. This is purely for performance reasons, as we can fuse add and LayerNorm. The residual needs to be provided (except for the very first block). """ super().__init__() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.mixer = mixer_cls(dim) self.norm = norm_cls(dim) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() if self.fused_add_norm: assert RMSNorm is not None, "RMSNorm import fails" assert isinstance( self.norm, (nn.LayerNorm, RMSNorm) ), "Only LayerNorm and RMSNorm are supported for fused_add_norm" def forward( self, hidden_states: Tensor, residual: Optional[Tensor] = None, inference_params=None ): r"""Pass the input through the encoder layer. Args: hidden_states: the sequence to the encoder layer (required). residual: hidden_states = Mixer(LN(residual)) """ if not self.fused_add_norm: if residual is None: residual = hidden_states else: residual = residual + self.drop_path(hidden_states) hidden_states = self.norm(residual.to(dtype=self.norm.weight.dtype)) if self.residual_in_fp32: residual = residual.to(torch.float32) else: fused_add_norm_fn = rms_norm_fn if isinstance(self.norm, RMSNorm) else layer_norm_fn if residual is None: hidden_states, residual = fused_add_norm_fn( hidden_states, self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) else: hidden_states, residual = fused_add_norm_fn( self.drop_path(hidden_states), self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) hidden_states = self.mixer(hidden_states, inference_params=inference_params) return hidden_states, residual def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return self.mixer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)def create_block( d_model, ssm_cfg=None, norm_epsilon=1e-5, drop_path=0., rms_norm=False, residual_in_fp32=False, fused_add_norm=False, layer_idx=None, device=None, dtype=None, if_bimamba=False, bimamba_type="none", if_devide_out=False, init_layer_scale=None, ): if if_bimamba: bimamba_type = "v1" if ssm_cfg is None: ssm_cfg = {} factory_kwargs = {"device": device, "dtype": dtype} mixer_cls = partial(Mamba, layer_idx=layer_idx, bimamba_type=bimamba_type, if_devide_out=if_devide_out, init_layer_scale=init_layer_scale, **ssm_cfg, **factory_kwargs) norm_cls = partial( nn.LayerNorm if not rms_norm else RMSNorm, eps=norm_epsilon, **factory_kwargs ) block = Block( d_model, mixer_cls, norm_cls=norm_cls, drop_path=drop_path, fused_add_norm=fused_add_norm, residual_in_fp32=residual_in_fp32, ) block.layer_idx = layer_idx return block
https://github.com/huggingface/transformers/blob/c28d04e9e252a1a099944e325685f14d242ecdcd/src/transformers/models/gpt2/modeling_gpt2.py#L454
def init_weights( module, n_layer, initializer_range=0.02, # Now only used for embedding layer. rescale_prenorm_residual=True, n_residuals_per_layer=1, # Change to 2 if we have MLP ): if isinstance(module, nn.Linear): if module.bias is not None: if not getattr(module.bias, "no_reinit", False): nn.init.zeros(module.bias) elif isinstance(module, nn.Embedding): nn.init.normal(module.weight, std=initializer_range)
if rescale_prenorm_residual: # Reinitialize selected weights subject to the OpenAI GPT-2 Paper Scheme: # > A modified initialization which accounts for the accumulation on the residual path with model depth. Scale # > the weights of residual layers at initialization by a factor of 1/√N where N is the # of residual layers. # > -- GPT-2 :: https://openai.com/blog/better-language-models/ # # Reference (Megatron-LM): https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/model/gpt_model.py for name, p in module.named_parameters(): if name in ["out_proj.weight", "fc2.weight"]: # Special Scaled Initialization --> There are 2 Layer Norms per Transformer Block # Following Pytorch init, except scale by 1/sqrt(2 * n_layer) # We need to reinit p since this code could be called multiple times # Having just p *= scale would repeatedly scale it down nn.init.kaiming_uniform_(p, a=math.sqrt(5)) with torch.no_grad(): p /= math.sqrt(n_residuals_per_layer * n_layer)def segm_init_weights(m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=0.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.Conv2d): # NOTE conv was left to pytorch default in my original init lecun_normal_(m.weight) if m.bias is not None: nn.init.zeros_(m.bias) elif isinstance(m, (nn.LayerNorm, nn.GroupNorm, nn.BatchNorm2d)): nn.init.zeros_(m.bias) nn.init.ones_(m.weight)
class VisionMamba(nn.Module): def init(self, img_size=224, patch_size=16, stride=16, depth=24, embed_dim=192, channels=3, num_classes=1000, ssm_cfg=None, drop_rate=0., drop_path_rate=0.1, norm_epsilon: float = 1e-5, rms_norm: bool = False, initializer_cfg=None, fused_add_norm=False, residual_in_fp32=False, device=None, dtype=None, ft_seq_len=None, pt_hw_seq_len=14, if_bidirectional=False, final_pool_type='none', if_abs_pos_embed=False, if_rope=False, if_rope_residual=False, flip_img_sequences_ratio=-1., if_bimamba=False, bimamba_type="none", if_cls_token=False, if_devide_out=False, init_layer_scale=None, use_double_cls_token=False, use_middle_cls_token=False, **kwargs): factory_kwargs = {"device": device, "dtype": dtype} # add factory_kwargs into kwargs kwargs.update(factory_kwargs) super().init() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.if_bidirectional = if_bidirectional self.final_pool_type = final_pool_type self.if_abs_pos_embed = if_abs_pos_embed self.if_rope = if_rope self.if_rope_residual = if_rope_residual self.flip_img_sequences_ratio = flip_img_sequences_ratio # self.if_cls_token = if_cls_token (此处与Mamba源代码不同) self.if_cls_token = False self.use_double_cls_token = use_double_cls_token self.use_middle_cls_token = use_middle_cls_token self.num_tokens = 1 if if_cls_token else 0
# pretrain parameters self.num_classes = num_classes self.d_model = self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models self.patch_embed = PatchEmbed( img_size=img_size, patch_size=patch_size, stride=stride, in_chans=channels, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches if if_cls_token: if use_double_cls_token: self.cls_token_head = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) self.cls_token_tail = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) self.num_tokens = 2 else: self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) # self.num_tokens = 1 if if_abs_pos_embed: # self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, self.embed_dim)) # 此处与mamba不同,划分了x和z的位置编码 self.pos_embed_x = nn.Parameter(torch.zeros(1, 256, self.embed_dim)) self.pos_embed_z = nn.Parameter(torch.zeros(1, 64, self.embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) if if_rope: half_head_dim = embed_dim // 2 hw_seq_len = img_size // patch_size self.rope = VisionRotaryEmbeddingFast( dim=half_head_dim, pt_seq_len=pt_hw_seq_len, ft_seq_len=hw_seq_len ) # TODO: release this comment dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule # import ipdb;ipdb.set_trace() inter_dpr = [0.0] + dpr self.drop_path = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity() # transformer blocks self.layers = nn.ModuleList( [ create_block( embed_dim, ssm_cfg=ssm_cfg, norm_epsilon=norm_epsilon, rms_norm=rms_norm, residual_in_fp32=residual_in_fp32, fused_add_norm=fused_add_norm, layer_idx=i, if_bimamba=if_bimamba, bimamba_type=bimamba_type, drop_path=inter_dpr[i], if_devide_out=if_devide_out, init_layer_scale=init_layer_scale, **factory_kwargs, ) for i in range(depth) ] ) # output head self.norm_f = (nn.LayerNorm if not rms_norm else RMSNorm)( embed_dim, eps=norm_epsilon, **factory_kwargs ) # self.pre_logits = nn.Identity() # original init self.patch_embed.apply(segm_init_weights) # 此处与mamba不同,问题在于没有执行head相关操作 # self.head.apply(segm_init_weights) # 在pos_embed中也进行了模板和搜索的划分 if if_abs_pos_embed: trunc_normal_(self.pos_embed_x, std=.02) trunc_normal_(self.pos_embed_z, std=.02) if if_cls_token: if use_double_cls_token: trunc_normal_(self.cls_token_head, std=.02) trunc_normal_(self.cls_token_tail, std=.02) else: trunc_normal_(self.cls_token, std=.02) # mamba init self.apply( partial( _init_weights, n_layer=depth, **(initializer_cfg if initializer_cfg is not None else {}), ) ) def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return { i: layer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs) for i, layer in enumerate(self.layers) } @torch.jit.ignore def no_weight_decay(self): return {"pos_embed", "cls_token", "dist_token", "cls_token_head", "cls_token_tail"} @torch.jit.ignore() def load_pretrained(self, checkpoint_path, prefix=""): _load_weights(self, checkpoint_path, prefix) def forward_features(self, z, x, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False): # taken from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py # with slight modifications to add the dist_token x = self.patch_embed(x) #x.shape = torch.Size([B, 3, 256, 256]) -> torch.Size([2, 256, 384]) z = self.patch_embed(z) #z.shape = torch.Size([B, 3, 128, 128]) -> torch.Size([2, 64, 384]) B, M, _ = x.shape if self.if_cls_token: # False if self.use_double_cls_token: cls_token_head = self.cls_token_head.expand(B, -1, -1) cls_token_tail = self.cls_token_tail.expand(B, -1, -1) token_position = [0, M + 1] x = torch.cat((cls_token_head, x, cls_token_tail), dim=1) M = x.shape[1] else: if self.use_middle_cls_token: cls_token = self.cls_token.expand(B, -1, -1) token_position = M // 2 # add cls token in the middle x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1) elif if_random_cls_token_position: cls_token = self.cls_token.expand(B, -1, -1) token_position = random.randint(0, M) x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1) print("token_position: ", token_position) else: cls_token = self.cls_token.expand(B, -1, -1) # stole cls_tokens impl from Phil Wang, thanks token_position = 0 x = torch.cat((cls_token, x), dim=1) M = x.shape[1] # 此处由于划分了模板和搜索,pos_embed也需要单独执行 if self.if_abs_pos_embed: # True x = x + self.pos_embed_x # x = x + positon_embemding =torch.Size([B, 256, 384]) + torch.Size([1, 256, 384]) = torch.Size([B, 256, 384]) z = z + self.pos_embed_z # z = z + positon_embemding =torch.Size([B, 64, 384]) + torch.Size([1, 64, 384]) = torch.Size([B, 64, 384]) x = torch.cat((z, x), dim=1) # torch.Size([B, 320, 384]) x = self.pos_drop(x) # x.shape = torch.Size([B, 320, 384]) if if_random_token_rank: #False # 生成随机 shuffle 索引 shuffle_indices = torch.randperm(M) if isinstance(token_position, list): print("original value: ", x[0, token_position[0], 0], x[0, token_position[1], 0]) else: print("original value: ", x[0, token_position, 0]) print("original token_position: ", token_position) # 执行 shuffle x = x[:, shuffle_indices, :] if isinstance(token_position, list): # 找到 cls token 在 shuffle 之后的新位置 new_token_position = [torch.where(shuffle_indices == token_position[i])[0].item() for i in range(len(token_position))] token_position = new_token_position else: # 找到 cls token 在 shuffle 之后的新位置 token_position = torch.where(shuffle_indices == token_position)[0].item() if isinstance(token_position, list): print("new value: ", x[0, token_position[0], 0], x[0, token_position[1], 0]) else: print("new value: ", x[0, token_position, 0]) print("new token_position: ", token_position) if_flip_img_sequences = False if self.flip_img_sequences_ratio > 0 and (self.flip_img_sequences_ratio - random.random()) > 1e-5: # False x = x.flip([1]) if_flip_img_sequences = True # mamba impl residual = None hidden_states = x if not self.if_bidirectional: # True for layer in self.layers: if if_flip_img_sequences and self.if_rope: # False hidden_states = hidden_states.flip([1]) if residual is not None: residual = residual.flip([1]) # rope about if self.if_rope: # False hidden_states = self.rope(hidden_states) if residual is not None and self.if_rope_residual: residual = self.rope(residual) if if_flip_img_sequences and self.if_rope: # False hidden_states = hidden_states.flip([1]) if residual is not None: residual = residual.flip([1]) hidden_states, residual = layer( hidden_states, residual, inference_params=inference_params ) else: # False # get two layers in a single for-loop for i in range(len(self.layers) // 2): if self.if_rope: hidden_states = self.rope(hidden_states) if residual is not None and self.if_rope_residual: residual = self.rope(residual) hidden_states_f, residual_f = self.layers[i * 2]( hidden_states, residual, inference_params=inference_params ) hidden_states_b, residual_b = self.layers[i * 2 + 1]( hidden_states.flip([1]), None if residual == None else residual.flip([1]), inference_params=inference_params ) hidden_states = hidden_states_f + hidden_states_b.flip([1]) residual = residual_f + residual_b.flip([1]) if not self.fused_add_norm: #False if residual is None: residual = hidden_states else: residual = residual + self.drop_path(hidden_states) hidden_states = self.norm_f(residual.to(dtype=self.norm_f.weight.dtype)) else: #True # Set prenorm=False here since we don't need the residual fused_add_norm_fn = rms_norm_fn if isinstance(self.norm_f, RMSNorm) else layer_norm_fn hidden_states = fused_add_norm_fn( # hidden_states.shape = torch.Size([B, 320, 384]) self.drop_path(hidden_states), self.norm_f.weight, self.norm_f.bias, eps=self.norm_f.eps, residual=residual, prenorm=False, residual_in_fp32=self.residual_in_fp32, ) # return only cls token if it exists if self.if_cls_token: #False if self.use_double_cls_token: return (hidden_states[:, token_position[0], :] + hidden_states[:, token_position[1], :]) / 2 else: if self.use_middle_cls_token: return hidden_states[:, token_position, :] elif if_random_cls_token_position: return hidden_states[:, token_position, :] else: return hidden_states[:, token_position, :] if self.final_pool_type == 'none': return hidden_states[:, -1, :] elif self.final_pool_type == 'mean': #True return hidden_states.mean(dim=1) #hidden_states.shape = torch.Size([2, 384]) elif self.final_pool_type == 'max': return hidden_states elif self.final_pool_type == 'all': return hidden_states else: raise NotImplementedError def forward(self, x, return_features=False, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False): x = self.forward_features(x, inference_params, if_random_cls_token_position=if_random_cls_token_position, if_random_token_rank=if_random_token_rank) if return_features: return x x = self.head(x) if self.final_pool_type == 'max': x = x.max(dim=1)[0] return x@register_model def vim_tiny_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model
@register_model def vim_tiny_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model
@register_model def vim_small_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='all', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg()
if pretrained: # checkpoint = torch.hub.load_state_dict_from_url( # url="to.do", # map_location="cpu", check_hash=True # ) # model.load_state_dict(checkpoint["model"]) checkpoint = torch.load(pretrained, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False) print('Load pretrained model from: ' + pretrained) return model@register_model def vim_small_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model
- Assistant
该代码实现了一个基于 Vision Transformer (ViT) 框架的深度学习模型,名为
VisionMamba。这个模型具有许多可自定义的特性和高级功能,可用于图像分类和视觉任务。以下是这个代码的主要功能和结构分析:
1. 模块引入
- PyTorch: 主要的深度学习框架,用于构建和训练神经网络。
Mamba模块: 一个自定义模块,可能是实现某种特别的混合或者特性。timm库: 包含了许多预处理的图像分类模型和组件。
2. 类定义
PatchEmbed: 将2D图像转化为补丁嵌入。通过卷积操作将图像信息压缩到嵌入维度。Block: 描述一个Transformer的基本块,包括混合层(mixer)、归一化层和残差连接。VisionMamba: 这是核心模型类。构建和管理图像嵌入、Transformer块以及输出特征。添加了许多可选配置,例如绝对位置编码、双向特性和特定类型的残差连接。
3. 主要实现细节
- 嵌入处理: 使用卷积操作将输入图像转换为补丁形式,并且利用嵌入维度将其映射到内部特征空间。
- 位置编码: 通过绝对位置编码和旋转位置编码(ROPE)增强模型对位置信息的感知。
- 融合归一化: 使用RMS Norm或者Layer Norm(根据可用性)与残差连接一起用于Transformer块。
- 残差缩放初始化: 对於GPT-2风格的初始化策略进行变体,考虑到残差路径的累积影响。
- 不同特性的组合: 如双向过程、温度值处理、位置编码、不同的融合/DropPath率等。
4. 预训练模型
提供了多个预训练模型注册函数用于不同的配置组合,比如不同的补丁尺寸和步长,以及是否使用中间CLS令牌等。
5. 可重用性和扩展性
这个代码在很多方面上提供了灵活性和可扩展性:如嵌入的尺寸、层的深度、归一化策略、不同的模块和特性的选择等等,从而可以在不同的视觉任务中进行定制和优化。
总之,该代码是一个复杂而灵活的图像处理Transformer模型实现,目标是在各种计算机视觉任务中提供强大的功能和更高的表现。
- User
Visionmamba中的Mamba模块是:# Copyright (c) 2023, Tri Dao, Albert Gu.
import math from typing import Optional
import torch import torch.nn as nn import torch.nn.functional as F from torch import Tensor
from einops import rearrange, repeat
try: from causal_conv1d import causal_conv1d_fn, causal_conv1d_update except ImportError: causal_conv1d_fn, causal_conv1d_update = None
try: from mamba_ssm.ops.selective_scan_interface import selective_scan_fn, mamba_inner_fn, bimamba_inner_fn, mamba_inner_fn_no_out_proj except ImportError: selective_scan_fn, mamba_inner_fn, bimamba_inner_fn, mamba_inner_fn_no_out_proj = None, None, None, None, None
try: from mamba_ssm.ops.triton.selective_state_update import selective_state_update except ImportError: selective_state_update = None
try: from mamba_ssm.ops.triton.layernorm import RMSNorm, layer_norm_fn, rms_norm_fn except ImportError: RMSNorm, layer_norm_fn, rms_norm_fn = None, None, None
d_state又称N,是隐状态的维度。d_model代表数据在经过SSM块之前和之后的数据维度,是输入x的维度
class Mamba(nn.Module): def init( self, d_model, d_state=16, d_conv=4, expand=2, dt_rank="auto", dt_min=0.001, dt_max=0.1, dt_init="random", dt_scale=1.0, dt_init_floor=1e-4, conv_bias=True, bias=False, use_fast_path=True, # Fused kernel options layer_idx=None, device=None, dtype=None, bimamba_type="none", if_devide_out=False, init_layer_scale=None, ): factory_kwargs = {"device": device, "dtype": dtype} super().init() self.d_model = d_model self.d_state = d_state self.d_conv = d_conv self.expand = expand self.d_inner = int(self.expand * self.d_model) self.dt_rank = math.ceil(self.d_model / 16) if dt_rank == "auto" else dt_rank self.use_fast_path = use_fast_path self.layer_idx = layer_idx self.bimamba_type = bimamba_type self.if_devide_out = if_devide_out
self.init_layer_scale = init_layer_scale if init_layer_scale is not None: self.gamma = nn.Parameter(init_layer_scale * torch.ones((d_model)), requires_grad=True) self.in_proj = nn.Linear(self.d_model, self.d_inner * 2, bias=bias, **factory_kwargs) self.conv1d = nn.Conv1d( in_channels=self.d_inner, out_channels=self.d_inner, bias=conv_bias, kernel_size=d_conv, groups=self.d_inner, padding=d_conv - 1, **factory_kwargs, ) self.activation = "silu" self.act = nn.SiLU() self.x_proj = nn.Linear( self.d_inner, self.dt_rank + self.d_state * 2, bias=False, **factory_kwargs ) self.dt_proj = nn.Linear(self.dt_rank, self.d_inner, bias=True, **factory_kwargs) # Initialize special dt projection to preserve variance at initialization dt_init_std = self.dt_rank**-0.5 * dt_scale if dt_init == "constant": nn.init.constant_(self.dt_proj.weight, dt_init_std) elif dt_init == "random": nn.init.uniform_(self.dt_proj.weight, -dt_init_std, dt_init_std) else: raise NotImplementedError # Initialize dt bias so that F.softplus(dt_bias) is between dt_min and dt_max dt = torch.exp( torch.rand(self.d_inner, **factory_kwargs) * (math.log(dt_max) - math.log(dt_min)) + math.log(dt_min) ).clamp(min=dt_init_floor) # Inverse of softplus: https://github.com/pytorch/pytorch/issues/72759 inv_dt = dt + torch.log(-torch.expm1(-dt)) with torch.no_grad(): self.dt_proj.bias.copy_(inv_dt) # Our initialization would set all Linear.bias to zero, need to mark this one as _no_reinit self.dt_proj.bias._no_reinit = True # S4D real initialization A = repeat( torch.arange(1, self.d_state + 1, dtype=torch.float32, device=device), "n -> d n", d=self.d_inner, ).contiguous() A_log = torch.log(A) # Keep A_log in fp32 self.A_log = nn.Parameter(A_log) self.A_log._no_weight_decay = True # D "skip" parameter self.D = nn.Parameter(torch.ones(self.d_inner, device=device)) # Keep in fp32 self.D._no_weight_decay = True # bidirectional if bimamba_type == "v1": A_b = repeat( torch.arange(1, self.d_state + 1, dtype=torch.float32, device=device), "n -> d n", d=self.d_inner, ).contiguous() A_b_log = torch.log(A_b) # Keep A_b_log in fp32 self.A_b_log = nn.Parameter(A_b_log) self.A_b_log._no_weight_decay = True elif bimamba_type == "v2": A_b = repeat( torch.arange(1, self.d_state + 1, dtype=torch.float32, device=device), "n -> d n", d=self.d_inner, ).contiguous() A_b_log = torch.log(A_b) # Keep A_b_log in fp32 self.A_b_log = nn.Parameter(A_b_log) self.A_b_log._no_weight_decay = True self.conv1d_b = nn.Conv1d( in_channels=self.d_inner, out_channels=self.d_inner, bias=conv_bias, kernel_size=d_conv, groups=self.d_inner, padding=d_conv - 1, **factory_kwargs, ) self.x_proj_b = nn.Linear( self.d_inner, self.dt_rank + self.d_state * 2, bias=False, **factory_kwargs ) self.dt_proj_b = nn.Linear(self.dt_rank, self.d_inner, bias=True, **factory_kwargs) self.D_b = nn.Parameter(torch.ones(self.d_inner, device=device)) # Keep in fp32 self.D_b._no_weight_decay = True self.out_proj = nn.Linear(self.d_inner, self.d_model, bias=bias, **factory_kwargs) def forward(self, hidden_states, inference_params=None): """ hidden_states: (B, L, D) Returns: same shape as hidden_states """ batch, seqlen, dim = hidden_states.shape conv_state, ssm_state = None, None if inference_params is not None: conv_state, ssm_state = self._get_states_from_cache(inference_params, batch) if inference_params.seqlen_offset > 0: # The states are updated inplace out, _, _ = self.step(hidden_states, conv_state, ssm_state) return out # We do matmul and transpose BLH -> HBL at the same time xz = rearrange( self.in_proj.weight @ rearrange(hidden_states, "b l d -> d (b l)"), "d (b l) -> b d l", l=seqlen, ) if self.in_proj.bias is not None: xz = xz + rearrange(self.in_proj.bias.to(dtype=xz.dtype), "d -> d 1") A = -torch.exp(self.A_log.float()) # (d_inner, d_state) # In the backward pass we write dx and dz next to each other to avoid torch.cat if self.use_fast_path and inference_params is None: # Doesn't support outputting the states if self.bimamba_type == "v1": A_b = -torch.exp(self.A_b_log.float()) out = bimamba_inner_fn( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, self.out_proj.weight, self.out_proj.bias, A, A_b, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) elif self.bimamba_type == "v2": A_b = -torch.exp(self.A_b_log.float()) out = mamba_inner_fn_no_out_proj( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, A, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) out_b = mamba_inner_fn_no_out_proj( xz.flip([-1]), self.conv1d_b.weight, self.conv1d_b.bias, self.x_proj_b.weight, self.dt_proj_b.weight, A_b, None, None, self.D_b.float(), delta_bias=self.dt_proj_b.bias.float(), delta_softplus=True, ) # F.linear(rearrange(out_z, "b d l -> b l d"), out_proj_weight, out_proj_bias) if not self.if_devide_out: out = F.linear(rearrange(out + out_b.flip([-1]), "b d l -> b l d"), self.out_proj.weight, self.out_proj.bias) else: out = F.linear(rearrange(out + out_b.flip([-1]), "b d l -> b l d") / 2, self.out_proj.weight, self.out_proj.bias) else: out = mamba_inner_fn( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, self.out_proj.weight, self.out_proj.bias, A, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) else: x, z = xz.chunk(2, dim=1) # Compute short convolution if conv_state is not None: # If we just take x[:, :, -self.d_conv :], it will error if seqlen < self.d_conv # Instead F.pad will pad with zeros if seqlen < self.d_conv, and truncate otherwise. conv_state.copy_(F.pad(x, (self.d_conv - x.shape[-1], 0))) # Update state (B D W) if causal_conv1d_fn is None: x = self.act(self.conv1d(x)[..., :seqlen]) else: assert self.activation in ["silu", "swish"] x = causal_conv1d_fn( x=x, weight=rearrange(self.conv1d.weight, "d 1 w -> d w"), bias=self.conv1d.bias, activation=self.activation, ) # We're careful here about the layout, to avoid extra transposes. # We want dt to have d as the slowest moving dimension # and L as the fastest moving dimension, since those are what the ssm_scan kernel expects. x_dbl = self.x_proj(rearrange(x, "b d l -> (b l) d")) # (bl d) dt, B, C = torch.split(x_dbl, [self.dt_rank, self.d_state, self.d_state], dim=-1) dt = self.dt_proj.weight @ dt.t() dt = rearrange(dt, "d (b l) -> b d l", l=seqlen) B = rearrange(B, "(b l) dstate -> b dstate l", l=seqlen).contiguous() C = rearrange(C, "(b l) dstate -> b dstate l", l=seqlen).contiguous() assert self.activation in ["silu", "swish"] y = selective_scan_fn( x, dt, A, B, C, self.D.float(), z=z, delta_bias=self.dt_proj.bias.float(), delta_softplus=True, return_last_state=ssm_state is not None, ) if ssm_state is not None: y, last_state = y ssm_state.copy_(last_state) y = rearrange(y, "b d l -> b l d") out = self.out_proj(y) if self.init_layer_scale is not None: out = out * self.gamma return out def step(self, hidden_states, conv_state, ssm_state): dtype = hidden_states.dtype assert hidden_states.shape[1] == 1, "Only support decoding with 1 token at a time for now" xz = self.in_proj(hidden_states.squeeze(1)) # (B 2D) x, z = xz.chunk(2, dim=-1) # (B D) # Conv step if causal_conv1d_update is None: conv_state.copy_(torch.roll(conv_state, shifts=-1, dims=-1)) # Update state (B D W) conv_state[:, :, -1] = x x = torch.sum(conv_state * rearrange(self.conv1d.weight, "d 1 w -> d w"), dim=-1) # (B D) if self.conv1d.bias is not None: x = x + self.conv1d.bias x = self.act(x).to(dtype=dtype) else: x = causal_conv1d_update( x, conv_state, rearrange(self.conv1d.weight, "d 1 w -> d w"), self.conv1d.bias, self.activation, ) x_db = self.x_proj(x) # (B dt_rank+2*d_state) dt, B, C = torch.split(x_db, [self.dt_rank, self.d_state, self.d_state], dim=-1) # Don't add dt_bias here dt = F.linear(dt, self.dt_proj.weight) # (B d_inner) A = -torch.exp(self.A_log.float()) # (d_inner, d_state) # SSM step if selective_state_update is None: # Discretize A and B dt = F.softplus(dt + self.dt_proj.bias.to(dtype=dt.dtype)) dA = torch.exp(torch.einsum("bd,dn->bdn", dt, A)) dB = torch.einsum("bd,bn->bdn", dt, B) ssm_state.copy_(ssm_state * dA + rearrange(x, "b d -> b d 1") * dB) y = torch.einsum("bdn,bn->bd", ssm_state.to(dtype), C) y = y + self.D.to(dtype) * x y = y * self.act(z) # (B D) else: y = selective_state_update( ssm_state, x, dt, A, B, C, self.D, z=z, dt_bias=self.dt_proj.bias, dt_softplus=True ) out = self.out_proj(y) return out.unsqueeze(1), conv_state, ssm_state def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): device = self.out_proj.weight.device conv_dtype = self.conv1d.weight.dtype if dtype is None else dtype conv_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_conv, device=device, dtype=conv_dtype ) ssm_dtype = self.dt_proj.weight.dtype if dtype is None else dtype # ssm_dtype = torch.float32 ssm_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_state, device=device, dtype=ssm_dtype ) return conv_state, ssm_state def _get_states_from_cache(self, inference_params, batch_size, initialize_states=False): assert self.layer_idx is not None if self.layer_idx not in inference_params.key_value_memory_dict: batch_shape = (batch_size,) conv_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_conv, device=self.conv1d.weight.device, dtype=self.conv1d.weight.dtype, ) ssm_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_state, device=self.dt_proj.weight.device, dtype=self.dt_proj.weight.dtype, # dtype=torch.float32, ) inference_params.key_value_memory_dict[self.layer_idx] = (conv_state, ssm_state) else: conv_state, ssm_state = inference_params.key_value_memory_dict[self.layer_idx] # TODO: What if batch size changes between generation, and we reuse the same states? if initialize_states: conv_state.zero_() ssm_state.zero_() return conv_state, ssm_stateclass Block(nn.Module): def init( self, dim, mixer_cls, norm_cls=nn.LayerNorm, fused_add_norm=False, residual_in_fp32=False ): """ Simple block wrapping a mixer class with LayerNorm/RMSNorm and residual connection"
This Block has a slightly different structure compared to a regular prenorm Transformer block. The standard block is: LN -> MHA/MLP -> Add. [Ref: https://arxiv.org/abs/2002.04745] Here we have: Add -> LN -> Mixer, returning both the hidden_states (output of the mixer) and the residual. This is purely for performance reasons, as we can fuse add and LayerNorm. The residual needs to be provided (except for the very first block). """ super().__init__() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.mixer = mixer_cls(dim) self.norm = norm_cls(dim) if self.fused_add_norm: assert RMSNorm is not None, "RMSNorm import fails" assert isinstance( self.norm, (nn.LayerNorm, RMSNorm) ), "Only LayerNorm and RMSNorm are supported for fused_add_norm" def forward( self, hidden_states: Tensor, residual: Optional[Tensor] = None, inference_params=None ): r"""Pass the input through the encoder layer. Args: hidden_states: the sequence to the encoder layer (required). residual: hidden_states = Mixer(LN(residual)) """ if not self.fused_add_norm: residual = (hidden_states + residual) if residual is not None else hidden_states hidden_states = self.norm(residual.to(dtype=self.norm.weight.dtype)) if self.residual_in_fp32: residual = residual.to(torch.float32) else: fused_add_norm_fn = rms_norm_fn if isinstance(self.norm, RMSNorm) else layer_norm_fn hidden_states, residual = fused_add_norm_fn( hidden_states, self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) hidden_states = self.mixer(hidden_states, inference_params=inference_params) return hidden_states, residual def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return self.mixer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs) - Assistant
这段代码定义了一个名为
Mamba的高级模块,它结合了多种技巧来实现高效的序列建模。下面是这个模块的详细分析:1. 概述
Mamba类: 这是一个用于序列建模的模块,结合了卷积和线性投影,用于处理隐状态。它支持不同的功能,如双向处理(bidirectional)和快路径(fast path)计算。Block类: 包装了一个混合器类(mixer_cls),使用归一化和残差连接进行结构调整。不同于经典自注意力机制,这里调整了模块顺序以提高性能。
2. 主要组件
-
卷积和线性投影:
in_proj: 输入投影层,将输入的特征维度映射到内部更高维度。conv1d: 1D卷积层,用于对序列数据进行局部时间模式的处理。x_proj和dt_proj: 用于特定维度投影,以支持特定A矩阵的变换和快速路径处理。
-
参数初始化:
dt_proj.weight通过特殊初始化以保证变换值的方差稳定。A_log和D等特殊参数用于保持转换矩阵和跳跃连接的稳定性。
-
激活函数: 通过使用 SiLU 或称为 Swish 激活函数,处理卷积结果。
3. 双向支持
- 该模块支持多种 "双向" 实现(如
bimamba_type="v1"和bimamba_type="v2"),通过彼此独立的权重和卷积计算实现。
4. 高效路径与推理缓存
- 快速路径: 使用优化的内核函数以加速计算。
- 推理缓存: 模块支持内存状态缓存,这在长序列或分段处理时有助于提升效率。
5. 前向传播
- 标准前向路径: 对输入进行投影、卷积、线性变换、激活以及最终输出层计算。
- 增量更新步骤(
step方法): 用于增量推理,适合在线处理序列。
6.
Block类的实现- 包含
LayerNorm/RMSNorm和Mixer的逻辑。 - 支持融合归一化和残差的计算,用于加速处理过程。
7. 总结
Mamba模块是一个强大的、面向序列建模的组件,结合了卷积、混合投影和双向的高效实现策略。它为要处理的序列提供了灵活性和性能特点。- 模块代码结合了多个高级的技巧,包括特殊初始化策略和状态缓存,用于应对长序列的高效处理。
这种复杂设计的目的是优化模型在大规模序列任务中的表现,特别是在具有特定双向依赖性或需要快速路径推理的应用场景中。
- User
那么再分析一下上文中的MambaIR中,哪些可以用在当前Visionmamba中?
- Assistant
在
VisionMamba模型中,Mamba模块的某些功能和技巧可以直接或间接地应用,以提高视觉模型的性能和效率。以下是如何在视觉模型中利用这些功能的分析:1. 卷积和投影技术
Mamba中的conv1d卷积层可以用于处理图像的局部特征,虽然在视觉模型中更常用的是2D卷积,但1D卷积也可以用于处理某些特定的任务,比如沿某个维度的特征提取或时间序列特征。in_proj和out_proj线性投影可以用于特征维度的转换。在VisionMamba中,特征映射(embedding)过程也依赖于类似的线性投影操作。
2. 激活函数
Mamba中使用的SiLU激活函数是一种有效的替代ReLU的选择,可以在视觉模型的非线性激活阶段使用,对模型的训练稳定性和收敛速度有积极影响。
3. 特殊参数初始化
- 用于
dt_proj和A_log的特殊初始化技术可以帮助在VisionMamba中对某些层的参数进行更好的初始化,以增强模型训练的稳定性,特别是在涉及复杂权重和潜在长序列的阶段。
4. 高效路径
- 快速路径或内核优化技术可以在
VisionMamba中应用以提高大型模型或高分辨率图像的处理效率。这对于需要实时处理或在低延迟要求下运行的应用特别关键。
5. 双向处理
Mamba中的双向能力(如bimamba_type配置)可以被移植到VisionMamba以提升视觉序列或帧序列的处理能力。这对于需要同时前后文信息的序列处理(如视频分析)可能非常有用。
6. 归一化与残差融合
- 在
VisionMamba中,可以使用与Mamba中类似的归一化与残差连接的融合技术以改善模型在训练过程中的梯度流动和稳定性。
7. 推理缓存
- 尽管在图像模型中不像在序列模型中那样,推理状态缓存的作用显著,但在处理视频或长序列图像时,这一技术仍可能改进推理效率。
总结
虽然
VisionMamba和Mamba的设计目标和应用领域不同,但Mamba模块中的许多技巧和方法仍然可以被视为优化视觉模型性能的宝贵资源。这些方法的整合需根据具体应用场景的需求和限制条件进行定制化。 - User
分析:def to_3d(x): return rearrange(x, 'b c h w -> b (h w) c')
def to_4d(x, h, w): return rearrange(x, 'b (h w) c -> b c h w', h=h, w=w)
class BiasFree_LayerNorm(nn.Module): def init(self, normalized_shape): super(BiasFree_LayerNorm, self).init() if isinstance(normalized_shape, numbers.Integral): normalized_shape = (normalized_shape,) normalized_shape = torch.Size(normalized_shape)
assert len(normalized_shape) == 1 self.weight = nn.Parameter(torch.ones(normalized_shape)) self.normalized_shape = normalized_shape def forward(self, x): sigma = x.var(-1, keepdim=True, unbiased=False) return x / torch.sqrt(sigma+1e-5) * self.weightclass WithBias_LayerNorm(nn.Module): def init(self, normalized_shape): super(WithBias_LayerNorm, self).init() if isinstance(normalized_shape, numbers.Integral): normalized_shape = (normalized_shape,) normalized_shape = torch.Size(normalized_shape)
assert len(normalized_shape) == 1 self.weight = nn.Parameter(torch.ones(normalized_shape)) self.bias = nn.Parameter(torch.zeros(normalized_shape)) self.normalized_shape = normalized_shape def forward(self, x): mu = x.mean(-1, keepdim=True) sigma = x.var(-1, keepdim=True, unbiased=False) return (x - mu) / torch.sqrt(sigma+1e-5) * self.weight + self.biasclass LayerNorm(nn.Module): def init(self, dim, LayerNorm_type): super(LayerNorm, self).init() if LayerNorm_type =='BiasFree': self.body = BiasFree_LayerNorm(dim) else: self.body = WithBias_LayerNorm(dim)
def forward(self, x): h, w = x.shape[-2:] return to_4d(self.body(to_3d(x)), h, w)class CrossMamba(nn.Module): def init(self, dim): super(CrossMamba, self).init() self.cross_mamba = Mamba(dim,bimamba_type="v3") # 使用v3版本,会使用额外的输入extra_emb进行特殊处理 self.norm1 = LayerNorm(dim,'with_bias') self.norm2 = LayerNorm(dim,'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) def forward(self,m1,m1_resi,m2):
m1_resi = m1+m1_resi
global_f = self.cross_mamba(m1, extra_emb=m2) return global_f 模块的作用。 - Assistant
这段代码定义了一个
CrossMamba模块,它结合了卷积、LayerNorm 以及自定义的Mamba模块。通过这种组合,CrossMamba实现了一种高级特征融合与处理机制。以下是对这段代码的详细分析:1. 帮助函数
to_3d和to_4d: 使用einops库的rearrange方法在不同维度的表示之间进行转换。to_3d将四维张量(通常为图像数据)转换为三维,以便于某些操作,而to_4d则执行相反的操作。to_3d(x): 将形状 (b, c, h, w) 转换为 (b, h*w, c)。to_4d(x, h, w): 将形状 (b, h*w, c) 转换回 (b, c, h, w)。
2. LayerNorm 类
提供两种LayerNorm的实现方式:
BiasFree_LayerNorm和WithBias_LayerNorm,以及它们的统一接口LayerNorm。BiasFree_LayerNorm: 标准LayerNorm操作,但不具有偏置参数。WithBias_LayerNorm: 标准LayerNorm操作,允许偏置调整。LayerNorm类: 根据输入参数选择使用哪种LayerNorm实现,将输入从4D转换为3D进行规范化,再转回4D,以保持与后续操作的维度一致。
3. CrossMamba 模块
-
CrossMamba(dim)构造函数初始化了一些层,包括Mamba实例、两个带有偏置的 LayerNorm 层以及深度卷积层。Mamba(dim, bimamba_type="v3"): 使用自定义的Mamba模块处理输入特征。通过"v3"版本使用额外的输入extra_emb,这意味着它可能处理额外的上下文信息。dwconv: 这是一个深度卷积层,保留空间信息的同时进行维度内卷积,有助于在特征图上进行本地邻域更新。
-
forward方法:- 将输入 m1(表示特征)加上 m1_resi(残差)以获得更新的输入特征。
- 通过
cross_mamba模块处理融合后的特征,与 m2(可能代表额外的信息或上下文特征)一起作为上下文输入。 - 返回融合后的全局特征
global_f。
总结
CrossMamba模块的设计侧重于特征融合和规范化。通过结合深度卷积和LayerNorm的多种实现,它既能处理局部信息又能调整全局特征。Mamba模块利用额外输入特征进行交叉上下文处理,从而更好地集成来自不同来源的信息。这种设计特别适用于需要不同层次特征集成和复杂上下文的深度学习任务。 - User
crossmamba中的调用的Mamba是:# Copyright (c) 2023, Tri Dao, Albert Gu.
import math from typing import Optional
import torch import torch.nn as nn import torch.nn.functional as F from torch import Tensor
from einops import rearrange, repeat
try: from causal_conv1d import causal_conv1d_fn, causal_conv1d_update except ImportError: causal_conv1d_fn, causal_conv1d_update = None
try: from mamba_ssm.ops.selective_scan_interface import selective_scan_fn, mamba_inner_fn, bimamba_inner_fn, mamba_inner_fn_no_out_proj except ImportError: # selective_scan_fn, mamba_inner_fn, bimamba_inner_fn, mamba_inner_fn_no_out_proj = None, None, None, None, None selective_scan_fn, mamba_inner_fn, bimamba_inner_fn, mamba_inner_fn_no_out_proj = None, None, None, None try: from mamba_ssm.ops.triton.selective_state_update import selective_state_update except ImportError: selective_state_update = None
try: from mamba_ssm.ops.triton.layernorm import RMSNorm, layer_norm_fn, rms_norm_fn except ImportError: RMSNorm, layer_norm_fn, rms_norm_fn = None, None, None
class Mamba(nn.Module): def init( self, d_model, d_state=16, d_conv=4, expand=2, dt_rank="auto", dt_min=0.001, dt_max=0.1, dt_init="random", dt_scale=1.0, dt_init_floor=1e-4, conv_bias=True, bias=False, use_fast_path=True, # Fused kernel options layer_idx=None, device=None, dtype=None, bimamba_type="v2", if_devide_out=False, init_layer_scale=None, ): factory_kwargs = {"device": device, "dtype": dtype} super().init() self.d_model = d_model self.d_state = d_state self.d_conv = d_conv self.expand = expand self.d_inner = int(self.expand * self.d_model) self.dt_rank = math.ceil(self.d_model / 16) if dt_rank == "auto" else dt_rank self.use_fast_path = use_fast_path self.layer_idx = layer_idx self.bimamba_type = bimamba_type self.if_devide_out = if_devide_out
self.init_layer_scale = init_layer_scale if init_layer_scale is not None: self.gamma = nn.Parameter(init_layer_scale * torch.ones((d_model)), requires_grad=True) self.in_proj = nn.Linear(self.d_model, self.d_inner * 2, bias=bias, **factory_kwargs) self.conv1d = nn.Conv1d( in_channels=self.d_inner, out_channels=self.d_inner, bias=conv_bias, kernel_size=d_conv, groups=self.d_inner, padding=d_conv - 1, **factory_kwargs, ) self.activation = "silu" self.act = nn.SiLU() self.x_proj = nn.Linear( self.d_inner, self.dt_rank + self.d_state * 2, bias=False, **factory_kwargs ) self.dt_proj = nn.Linear(self.dt_rank, self.d_inner, bias=True, **factory_kwargs) # Initialize special dt projection to preserve variance at initialization dt_init_std = self.dt_rank**-0.5 * dt_scale if dt_init == "constant": nn.init.constant_(self.dt_proj.weight, dt_init_std) elif dt_init == "random": nn.init.uniform_(self.dt_proj.weight, -dt_init_std, dt_init_std) else: raise NotImplementedError # Initialize dt bias so that F.softplus(dt_bias) is between dt_min and dt_max dt = torch.exp( torch.rand(self.d_inner, **factory_kwargs) * (math.log(dt_max) - math.log(dt_min)) + math.log(dt_min) ).clamp(min=dt_init_floor) # Inverse of softplus: https://github.com/pytorch/pytorch/issues/72759 inv_dt = dt + torch.log(-torch.expm1(-dt)) with torch.no_grad(): self.dt_proj.bias.copy_(inv_dt) # Our initialization would set all Linear.bias to zero, need to mark this one as _no_reinit self.dt_proj.bias._no_reinit = True # S4D real initialization A = repeat( torch.arange(1, self.d_state + 1, dtype=torch.float32, device=device), "n -> d n", d=self.d_inner, ).contiguous() A_log = torch.log(A) # Keep A_log in fp32 self.A_log = nn.Parameter(A_log) self.A_log._no_weight_decay = True # D "skip" parameter self.D = nn.Parameter(torch.ones(self.d_inner, device=device)) # Keep in fp32 self.D._no_weight_decay = True # bidirectional if bimamba_type == "v1": A_b = repeat( torch.arange(1, self.d_state + 1, dtype=torch.float32, device=device), "n -> d n", d=self.d_inner, ).contiguous() A_b_log = torch.log(A_b) # Keep A_b_log in fp32 self.A_b_log = nn.Parameter(A_b_log) self.A_b_log._no_weight_decay = True elif bimamba_type == "v2" or bimamba_type=='v3': A_b = repeat( torch.arange(1, self.d_state + 1, dtype=torch.float32, device=device), "n -> d n", d=self.d_inner, ).contiguous() A_b_log = torch.log(A_b) # Keep A_b_log in fp32 self.A_b_log = nn.Parameter(A_b_log) self.A_b_log._no_weight_decay = True self.conv1d_b = nn.Conv1d( in_channels=self.d_inner, out_channels=self.d_inner, bias=conv_bias, kernel_size=d_conv, groups=self.d_inner, padding=d_conv - 1, **factory_kwargs, ) self.x_proj_b = nn.Linear( self.d_inner, self.dt_rank + self.d_state * 2, bias=False, **factory_kwargs ) self.dt_proj_b = nn.Linear(self.dt_rank, self.d_inner, bias=True, **factory_kwargs) self.D_b = nn.Parameter(torch.ones(self.d_inner, device=device)) # Keep in fp32 self.D_b._no_weight_decay = True self.in_proj_extra = nn.Linear(self.d_model, self.d_inner * 2, bias=bias, **factory_kwargs) self.out_proj = nn.Linear(self.d_inner, self.d_model, bias=bias, **factory_kwargs) def forward(self, hidden_states, inference_params=None,extra_emb=None): """ hidden_states: (B, L, D) Returns: same shape as hidden_states """ batch, seqlen, dim = hidden_states.shape conv_state, ssm_state = None, None if inference_params is not None: conv_state, ssm_state = self._get_states_from_cache(inference_params, batch) if inference_params.seqlen_offset > 0: # The states are updated inplace out, _, _ = self.step(hidden_states, conv_state, ssm_state) return out if extra_emb is None: # We do matmul and transpose BLH -> HBL at the same time xz = rearrange( self.in_proj.weight @ rearrange(hidden_states, "b l d -> d (b l)"), "d (b l) -> b d l", l=seqlen, ) if self.in_proj.bias is not None: xz = xz + rearrange(self.in_proj.bias.to(dtype=xz.dtype), "d -> d 1") else: xz = rearrange( self.in_proj.weight @ rearrange(hidden_states, "b l d -> d (b l)"), "d (b l) -> b d l", l=seqlen, ) if self.in_proj.bias is not None: xz = xz + rearrange(self.in_proj.bias.to(dtype=xz.dtype), "d -> d 1") extra_emb = rearrange( self.in_proj_extra.weight @ rearrange(extra_emb, "b l d -> d (b l)"), "d (b l) -> b d l", l=seqlen, ) if self.in_proj_extra.bias is not None: extra_emb = extra_emb + rearrange(self.in_proj_extra.bias.to(dtype=xz.dtype), "d -> d 1") A = -torch.exp(self.A_log.float()) # (d_inner, d_state) # In the backward pass we write dx and dz next to each other to avoid torch.cat if self.use_fast_path and inference_params is None: # Doesn't support outputting the states if self.bimamba_type == "v1": A_b = -torch.exp(self.A_b_log.float()) out = bimamba_inner_fn( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, self.out_proj.weight, self.out_proj.bias, A, A_b, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) elif self.bimamba_type == "v2": A_b = -torch.exp(self.A_b_log.float()) out = mamba_inner_fn_no_out_proj( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, A, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) out_b = mamba_inner_fn_no_out_proj( xz.flip([-1]), self.conv1d_b.weight, self.conv1d_b.bias, self.x_proj_b.weight, self.dt_proj_b.weight, A_b, None, None, self.D_b.float(), delta_bias=self.dt_proj_b.bias.float(), delta_softplus=True, ) # F.linear(rearrange(out_z, "b d l -> b l d"), out_proj_weight, out_proj_bias) if not self.if_devide_out: out = F.linear(rearrange(out + out_b.flip([-1]), "b d l -> b l d"), self.out_proj.weight, self.out_proj.bias) else: out = F.linear(rearrange(out + out_b.flip([-1]), "b d l -> b l d") / 2, self.out_proj.weight, self.out_proj.bias) elif self.bimamba_type == "v3": A_b = -torch.exp(self.A_b_log.float()) out = mamba_inner_fn_no_out_proj( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, A, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) out_b = mamba_inner_fn_no_out_proj( # extra_emb同上文中xz.flip不同 extra_emb, self.conv1d_b.weight, self.conv1d_b.bias, self.x_proj_b.weight, self.dt_proj_b.weight, A_b, None, None, self.D_b.float(), delta_bias=self.dt_proj_b.bias.float(), delta_softplus=True, ) # F.linear(rearrange(out_z, "b d l -> b l d"), out_proj_weight, out_proj_bias) if not self.if_devide_out: #True out = F.linear(rearrange(out + out_b, "b d l -> b l d"), self.out_proj.weight, self.out_proj.bias) else: #False out = F.linear(rearrange(out + out_b, "b d l -> b l d") / 2, self.out_proj.weight, self.out_proj.bias) else: #False out = mamba_inner_fn( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, self.out_proj.weight, self.out_proj.bias, A, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) else: #False x, z = xz.chunk(2, dim=1) # Compute short convolution if conv_state is not None: # If we just take x[:, :, -self.d_conv :], it will error if seqlen < self.d_conv # Instead F.pad will pad with zeros if seqlen < self.d_conv, and truncate otherwise. conv_state.copy_(F.pad(x, (self.d_conv - x.shape[-1], 0))) # Update state (B D W) if causal_conv1d_fn is None: x = self.act(self.conv1d(x)[..., :seqlen]) else: assert self.activation in ["silu", "swish"] x = causal_conv1d_fn( x=x, weight=rearrange(self.conv1d.weight, "d 1 w -> d w"), bias=self.conv1d.bias, activation=self.activation, ) # We're careful here about the layout, to avoid extra transposes. # We want dt to have d as the slowest moving dimension # and L as the fastest moving dimension, since those are what the ssm_scan kernel expects. x_dbl = self.x_proj(rearrange(x, "b d l -> (b l) d")) # (bl d) dt, B, C = torch.split(x_dbl, [self.dt_rank, self.d_state, self.d_state], dim=-1) dt = self.dt_proj.weight @ dt.t() dt = rearrange(dt, "d (b l) -> b d l", l=seqlen) B = rearrange(B, "(b l) dstate -> b dstate l", l=seqlen).contiguous() C = rearrange(C, "(b l) dstate -> b dstate l", l=seqlen).contiguous() assert self.activation in ["silu", "swish"] y = selective_scan_fn( x, dt, A, B, C, self.D.float(), z=z, delta_bias=self.dt_proj.bias.float(), delta_softplus=True, return_last_state=ssm_state is not None, ) if ssm_state is not None: y, last_state = y ssm_state.copy_(last_state) y = rearrange(y, "b d l -> b l d") out = self.out_proj(y) if self.init_layer_scale is not None: #False out = out * self.gamma return out def step(self, hidden_states, conv_state, ssm_state): dtype = hidden_states.dtype assert hidden_states.shape[1] == 1, "Only support decoding with 1 token at a time for now" xz = self.in_proj(hidden_states.squeeze(1)) # (B 2D) x, z = xz.chunk(2, dim=-1) # (B D) # Conv step if causal_conv1d_update is None: conv_state.copy_(torch.roll(conv_state, shifts=-1, dims=-1)) # Update state (B D W) conv_state[:, :, -1] = x x = torch.sum(conv_state * rearrange(self.conv1d.weight, "d 1 w -> d w"), dim=-1) # (B D) if self.conv1d.bias is not None: x = x + self.conv1d.bias x = self.act(x).to(dtype=dtype) else: x = causal_conv1d_update( x, conv_state, rearrange(self.conv1d.weight, "d 1 w -> d w"), self.conv1d.bias, self.activation, ) x_db = self.x_proj(x) # (B dt_rank+2*d_state) dt, B, C = torch.split(x_db, [self.dt_rank, self.d_state, self.d_state], dim=-1) # Don't add dt_bias here dt = F.linear(dt, self.dt_proj.weight) # (B d_inner) A = -torch.exp(self.A_log.float()) # (d_inner, d_state) # SSM step if selective_state_update is None: # Discretize A and B dt = F.softplus(dt + self.dt_proj.bias.to(dtype=dt.dtype)) dA = torch.exp(torch.einsum("bd,dn->bdn", dt, A)) dB = torch.einsum("bd,bn->bdn", dt, B) ssm_state.copy_(ssm_state * dA + rearrange(x, "b d -> b d 1") * dB) y = torch.einsum("bdn,bn->bd", ssm_state.to(dtype), C) y = y + self.D.to(dtype) * x y = y * self.act(z) # (B D) else: y = selective_state_update( ssm_state, x, dt, A, B, C, self.D, z=z, dt_bias=self.dt_proj.bias, dt_softplus=True ) out = self.out_proj(y) return out.unsqueeze(1), conv_state, ssm_state def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): device = self.out_proj.weight.device conv_dtype = self.conv1d.weight.dtype if dtype is None else dtype conv_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_conv, device=device, dtype=conv_dtype ) ssm_dtype = self.dt_proj.weight.dtype if dtype is None else dtype # ssm_dtype = torch.float32 ssm_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_state, device=device, dtype=ssm_dtype ) return conv_state, ssm_state def _get_states_from_cache(self, inference_params, batch_size, initialize_states=False): assert self.layer_idx is not None if self.layer_idx not in inference_params.key_value_memory_dict: batch_shape = (batch_size,) conv_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_conv, device=self.conv1d.weight.device, dtype=self.conv1d.weight.dtype, ) ssm_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_state, device=self.dt_proj.weight.device, dtype=self.dt_proj.weight.dtype, # dtype=torch.float32, ) inference_params.key_value_memory_dict[self.layer_idx] = (conv_state, ssm_state) else: conv_state, ssm_state = inference_params.key_value_memory_dict[self.layer_idx] # TODO: What if batch size changes between generation, and we reuse the same states? if initialize_states: conv_state.zero_() ssm_state.zero_() return conv_state, ssm_stateclass Block(nn.Module): def init( self, dim, mixer_cls, norm_cls=nn.LayerNorm, fused_add_norm=False, residual_in_fp32=False ): """ Simple block wrapping a mixer class with LayerNorm/RMSNorm and residual connection"
This Block has a slightly different structure compared to a regular prenorm Transformer block. The standard block is: LN -> MHA/MLP -> Add. [Ref: https://arxiv.org/abs/2002.04745] Here we have: Add -> LN -> Mixer, returning both the hidden_states (output of the mixer) and the residual. This is purely for performance reasons, as we can fuse add and LayerNorm. The residual needs to be provided (except for the very first block). """ super().__init__() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.mixer = mixer_cls(dim) self.norm = norm_cls(dim) if self.fused_add_norm: assert RMSNorm is not None, "RMSNorm import fails" assert isinstance( self.norm, (nn.LayerNorm, RMSNorm) ), "Only LayerNorm and RMSNorm are supported for fused_add_norm" def forward( self, hidden_states: Tensor, residual: Optional[Tensor] = None, inference_params=None ): r"""Pass the input through the encoder layer. Args: hidden_states: the sequence to the encoder layer (required). residual: hidden_states = Mixer(LN(residual)) """ if not self.fused_add_norm: residual = (hidden_states + residual) if residual is not None else hidden_states hidden_states = self.norm(residual.to(dtype=self.norm.weight.dtype)) if self.residual_in_fp32: residual = residual.to(torch.float32) else: fused_add_norm_fn = rms_norm_fn if isinstance(self.norm, RMSNorm) else layer_norm_fn hidden_states, residual = fused_add_norm_fn( hidden_states, self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) hidden_states = self.mixer(hidden_states, inference_params=inference_params) return hidden_states, residual def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return self.mixer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)class DWMamba(nn.Module): def init( self, d_model, d_state=16, d_conv=4, expand=2, dt_rank="auto", dt_min=0.001, dt_max=0.1, dt_init="random", dt_scale=1.0, dt_init_floor=1e-4, conv_bias=True, bias=False, use_fast_path=True, # Fused kernel options layer_idx=None, device=None, dtype=None, bimamba_type="v2", if_devide_out=False, init_layer_scale=None, ): factory_kwargs = {"device": device, "dtype": dtype} super().init() self.d_model = d_model self.d_state = d_state self.d_conv = d_conv self.expand = expand self.d_inner = int(self.expand * self.d_model) self.dt_rank = math.ceil(self.d_model / 16) if dt_rank == "auto" else dt_rank self.use_fast_path = use_fast_path self.layer_idx = layer_idx self.bimamba_type = bimamba_type self.if_devide_out = if_devide_out
self.init_layer_scale = init_layer_scale if init_layer_scale is not None: self.gamma = nn.Parameter(init_layer_scale * torch.ones((d_model)), requires_grad=True) self.in_proj = nn.Linear(self.d_model, self.d_inner * 2, bias=bias, **factory_kwargs) self.conv1d = nn.Conv1d( in_channels=self.d_inner, out_channels=self.d_inner, bias=conv_bias, kernel_size=d_conv, groups=self.d_inner, padding=d_conv - 1, **factory_kwargs, ) self.activation = "silu" self.act = nn.SiLU() self.x_proj = nn.Linear( self.d_inner, self.dt_rank + self.d_state * 2, bias=False, **factory_kwargs ) self.dt_proj = nn.Linear(self.dt_rank, self.d_inner, bias=True, **factory_kwargs) # Initialize special dt projection to preserve variance at initialization dt_init_std = self.dt_rank**-0.5 * dt_scale if dt_init == "constant": nn.init.constant_(self.dt_proj.weight, dt_init_std) elif dt_init == "random": nn.init.uniform_(self.dt_proj.weight, -dt_init_std, dt_init_std) else: raise NotImplementedError # Initialize dt bias so that F.softplus(dt_bias) is between dt_min and dt_max dt = torch.exp( torch.rand(self.d_inner, **factory_kwargs) * (math.log(dt_max) - math.log(dt_min)) + math.log(dt_min) ).clamp(min=dt_init_floor) # Inverse of softplus: https://github.com/pytorch/pytorch/issues/72759 inv_dt = dt + torch.log(-torch.expm1(-dt)) with torch.no_grad(): self.dt_proj.bias.copy_(inv_dt) # Our initialization would set all Linear.bias to zero, need to mark this one as _no_reinit self.dt_proj.bias._no_reinit = True # S4D real initialization A = repeat( torch.arange(1, self.d_state + 1, dtype=torch.float32, device=device), "n -> d n", d=self.d_inner, ).contiguous() A_log = torch.log(A) # Keep A_log in fp32 self.A_log = nn.Parameter(A_log) self.A_log._no_weight_decay = True # D "skip" parameter self.D = nn.Parameter(torch.ones(self.d_inner, device=device)) # Keep in fp32 self.D._no_weight_decay = True # bidirectional if bimamba_type == "v1": A_b = repeat( torch.arange(1, self.d_state + 1, dtype=torch.float32, device=device), "n -> d n", d=self.d_inner, ).contiguous() A_b_log = torch.log(A_b) # Keep A_b_log in fp32 self.A_b_log = nn.Parameter(A_b_log) self.A_b_log._no_weight_decay = True elif bimamba_type == "v2" or bimamba_type=='v3': A_b = repeat( torch.arange(1, self.d_state + 1, dtype=torch.float32, device=device), "n -> d n", d=self.d_inner, ).contiguous() A_b_log = torch.log(A_b) # Keep A_b_log in fp32 self.A_b_log = nn.Parameter(A_b_log) self.A_b_log._no_weight_decay = True self.conv1d_b = nn.Conv1d( in_channels=self.d_inner, out_channels=self.d_inner, bias=conv_bias, kernel_size=d_conv, groups=self.d_inner, padding=d_conv - 1, **factory_kwargs, ) self.x_proj_b = nn.Linear( self.d_inner, self.dt_rank + self.d_state * 2, bias=False, **factory_kwargs ) self.dt_proj_b = nn.Linear(self.dt_rank, self.d_inner, bias=True, **factory_kwargs) self.D_b = nn.Parameter(torch.ones(self.d_inner, device=device)) # Keep in fp32 self.D_b._no_weight_decay = True self.in_proj_extra = nn.Linear(self.d_model, self.d_inner * 2, bias=bias, **factory_kwargs) self.out_proj = nn.Linear(self.d_inner, self.d_model, bias=bias, **factory_kwargs) def forward(self, hidden_states, inference_params=None,extra_emb=None): """ hidden_states: (B, L, D) Returns: same shape as hidden_states """ batch, seqlen, dim = hidden_states.shape conv_state, ssm_state = None, None if inference_params is not None: conv_state, ssm_state = self._get_states_from_cache(inference_params, batch) if inference_params.seqlen_offset > 0: # The states are updated inplace out, _, _ = self.step(hidden_states, conv_state, ssm_state) return out if extra_emb is None: # We do matmul and transpose BLH -> HBL at the same time xz = rearrange( self.in_proj.weight @ rearrange(hidden_states, "b l d -> d (b l)"), "d (b l) -> b d l", l=seqlen, ) if self.in_proj.bias is not None: xz = xz + rearrange(self.in_proj.bias.to(dtype=xz.dtype), "d -> d 1") else: xz = rearrange( self.in_proj.weight @ rearrange(hidden_states, "b l d -> d (b l)"), "d (b l) -> b d l", l=seqlen, ) if self.in_proj.bias is not None: xz = xz + rearrange(self.in_proj.bias.to(dtype=xz.dtype), "d -> d 1") extra_emb = rearrange( self.in_proj_extra.weight @ rearrange(extra_emb, "b l d -> d (b l)"), "d (b l) -> b d l", l=seqlen, ) if self.in_proj_extra.bias is not None: extra_emb = extra_emb + rearrange(self.in_proj_extra.bias.to(dtype=xz.dtype), "d -> d 1") A = -torch.exp(self.A_log.float()) # (d_inner, d_state) # In the backward pass we write dx and dz next to each other to avoid torch.cat if self.use_fast_path and inference_params is None: # Doesn't support outputting the states # mamba3个版本 # v1版本涉及双向处理 if self.bimamba_type == "v1": # 创建指数函数,用于双向处理 A_b = -torch.exp(self.A_b_log.float()) # 调用bimamba_inner_fn进行双向数据处理 out = bimamba_inner_fn( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, self.out_proj.weight, self.out_proj.bias, A, A_b, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) # 版本2提供单向和反向处理,不直接使用输出投影合并结果 # 对每个方向分别计算状态更新,反转flip输入以模拟反向处理 # 结果通过加权和合并,然后应用线性输出投影; elif self.bimamba_type == "v2": A_b = -torch.exp(self.A_b_log.float()) out = mamba_inner_fn_no_out_proj( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, A, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) out_b = mamba_inner_fn_no_out_proj( xz.flip([-1]), self.conv1d_b.weight, self.conv1d_b.bias, self.x_proj_b.weight, self.dt_proj_b.weight, A_b, None, None, self.D_b.float(), delta_bias=self.dt_proj_b.bias.float(), delta_softplus=True, ) # F.linear(rearrange(out_z, "b d l -> b l d"), out_proj_weight, out_proj_bias) if not self.if_devide_out: out = F.linear(rearrange(out + out_b.flip([-1]), "b d l -> b l d"), self.out_proj.weight, self.out_proj.bias) else: out = F.linear(rearrange(out + out_b.flip([-1]), "b d l -> b l d") / 2, self.out_proj.weight, self.out_proj.bias) # 基于v2版本,加入对额外输入的并行处理 # 将extra_emb作为额外的上下文信息被处理,并与主输入并行处理 # 不反转extra_emb表示它可能与主序列的方向一致 # 结果合并后通过一个线性层来产生最终输出,允许融合来自两个不同源信息 # conv1d和conv1d_b 用于提取局部特征 # 状态映射层 x_proj 和 x_proj_b 用于将局部特征映射到状态更新形式 # 时间步映射 dt_proj 和 dt_proj_b 用于控制状态更新的时间步长大小 # 状态递推参数 A 和 A_b 用于控制状态递推的动态变化 # 直通参数D和D_b用于控制状态更新的跳跃大小,在更新时直接与输入相加,维持某种输入特征的直通。 elif self.bimamba_type == "v3": A_b = -torch.exp(self.A_b_log.float()) out = mamba_inner_fn_no_out_proj( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, A, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) out_b = mamba_inner_fn_no_out_proj( extra_emb, self.conv1d_b.weight, self.conv1d_b.bias, self.x_proj_b.weight, self.dt_proj_b.weight, A_b, None, None, self.D_b.float(), delta_bias=self.dt_proj_b.bias.float(), delta_softplus=True, ) # F.linear(rearrange(out_z, "b d l -> b l d"), out_proj_weight, out_proj_bias) if not self.if_devide_out: out = F.linear(rearrange(out + out_b, "b d l -> b l d"), self.out_proj.weight, self.out_proj.bias) else: out = F.linear(rearrange(out + out_b, "b d l -> b l d") / 2, self.out_proj.weight, self.out_proj.bias) else: out = mamba_inner_fn( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, self.out_proj.weight, self.out_proj.bias, A, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) else: x, z = xz.chunk(2, dim=1) # Compute short convolution if conv_state is not None: # If we just take x[:, :, -self.d_conv :], it will error if seqlen < self.d_conv # Instead F.pad will pad with zeros if seqlen < self.d_conv, and truncate otherwise. conv_state.copy_(F.pad(x, (self.d_conv - x.shape[-1], 0))) # Update state (B D W) if causal_conv1d_fn is None: x = self.act(self.conv1d(x)[..., :seqlen]) else: assert self.activation in ["silu", "swish"] x = causal_conv1d_fn( x=x, weight=rearrange(self.conv1d.weight, "d 1 w -> d w"), bias=self.conv1d.bias, activation=self.activation, ) # We're careful here about the layout, to avoid extra transposes. # We want dt to have d as the slowest moving dimension # and L as the fastest moving dimension, since those are what the ssm_scan kernel expects. x_dbl = self.x_proj(rearrange(x, "b d l -> (b l) d")) # (bl d) dt, B, C = torch.split(x_dbl, [self.dt_rank, self.d_state, self.d_state], dim=-1) dt = self.dt_proj.weight @ dt.t() dt = rearrange(dt, "d (b l) -> b d l", l=seqlen) B = rearrange(B, "(b l) dstate -> b dstate l", l=seqlen).contiguous() C = rearrange(C, "(b l) dstate -> b dstate l", l=seqlen).contiguous() assert self.activation in ["silu", "swish"] y = selective_scan_fn( x, dt, A, B, C, self.D.float(), z=z, delta_bias=self.dt_proj.bias.float(), delta_softplus=True, return_last_state=ssm_state is not None, ) if ssm_state is not None: y, last_state = y ssm_state.copy_(last_state) y = rearrange(y, "b d l -> b l d") out = self.out_proj(y) if self.init_layer_scale is not None: out = out * self.gamma return out def step(self, hidden_states, conv_state, ssm_state): dtype = hidden_states.dtype assert hidden_states.shape[1] == 1, "Only support decoding with 1 token at a time for now" xz = self.in_proj(hidden_states.squeeze(1)) # (B 2D) x, z = xz.chunk(2, dim=-1) # (B D) # Conv step if causal_conv1d_update is None: conv_state.copy_(torch.roll(conv_state, shifts=-1, dims=-1)) # Update state (B D W) conv_state[:, :, -1] = x x = torch.sum(conv_state * rearrange(self.conv1d.weight, "d 1 w -> d w"), dim=-1) # (B D) if self.conv1d.bias is not None: x = x + self.conv1d.bias x = self.act(x).to(dtype=dtype) else: x = causal_conv1d_update( x, conv_state, rearrange(self.conv1d.weight, "d 1 w -> d w"), self.conv1d.bias, self.activation, ) x_db = self.x_proj(x) # (B dt_rank+2*d_state) dt, B, C = torch.split(x_db, [self.dt_rank, self.d_state, self.d_state], dim=-1) # Don't add dt_bias here dt = F.linear(dt, self.dt_proj.weight) # (B d_inner) A = -torch.exp(self.A_log.float()) # (d_inner, d_state) # SSM step if selective_state_update is None: # Discretize A and B dt = F.softplus(dt + self.dt_proj.bias.to(dtype=dt.dtype)) dA = torch.exp(torch.einsum("bd,dn->bdn", dt, A)) dB = torch.einsum("bd,bn->bdn", dt, B) ssm_state.copy_(ssm_state * dA + rearrange(x, "b d -> b d 1") * dB) y = torch.einsum("bdn,bn->bd", ssm_state.to(dtype), C) y = y + self.D.to(dtype) * x y = y * self.act(z) # (B D) else: y = selective_state_update( ssm_state, x, dt, A, B, C, self.D, z=z, dt_bias=self.dt_proj.bias, dt_softplus=True ) out = self.out_proj(y) return out.unsqueeze(1), conv_state, ssm_state def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): device = self.out_proj.weight.device conv_dtype = self.conv1d.weight.dtype if dtype is None else dtype conv_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_conv, device=device, dtype=conv_dtype ) ssm_dtype = self.dt_proj.weight.dtype if dtype is None else dtype # ssm_dtype = torch.float32 ssm_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_state, device=device, dtype=ssm_dtype ) return conv_state, ssm_state def _get_states_from_cache(self, inference_params, batch_size, initialize_states=False): assert self.layer_idx is not None if self.layer_idx not in inference_params.key_value_memory_dict: batch_shape = (batch_size,) conv_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_conv, device=self.conv1d.weight.device, dtype=self.conv1d.weight.dtype, ) ssm_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_state, device=self.dt_proj.weight.device, dtype=self.dt_proj.weight.dtype, # dtype=torch.float32, ) inference_params.key_value_memory_dict[self.layer_idx] = (conv_state, ssm_state) else: conv_state, ssm_state = inference_params.key_value_memory_dict[self.layer_idx] # TODO: What if batch size changes between generation, and we reuse the same states? if initialize_states: conv_state.zero_() ssm_state.zero_() return conv_state, ssm_state,再分析一下
- Assistant
这里给出了一个名为
Mamba的自定义模块,它结合了卷积、线性投影、激活函数和特殊的内置函数,支持不同的版本(例如v1,v2,v3)用于序列数据的处理。以下是这个模块的详细分析:1. 模块概述
Mamba是一个实现序列处理的模块,其中特别考虑了双向序列处理(通过不同版本控制)以及快速路径计算。- 它的设计目的是高效处理长序列或需要复杂上下文感知的任务。
2. 主要组件
-
卷积与线性投影:
conv1d和conv1d_b是1D卷积层,分别用于正向和反向处理。in_proj和in_proj_extra是线性投影层,用于将输入数据转换到特定维度的内部表示。x_proj和x_proj_b用于状态更新的映射。
-
激活函数:使用了
SiLU激活函数,适合用于深层网络中,能够更好处理梯度和非线性。 -
特殊初始化:
dt_proj的特殊初始化用于保持变换时的方差稳定。A_log和A_b_log用于表示状态转移的动态调整。
-
双向处理(
bimamba_type):v1: 典型双向处理,在进行正向处理的同时并行进行反向处理。v2: 单向和反向处理分别进行,输入被反转处理。v3: 允许额外输入extra_emb,并与主要输入并行处理,特别用于需要额外上下文的信息融合。
3. 前向传播
- 使用矩阵乘法和卷积对
hidden_states进行特征提取和转换。 - 当
extra_emb存在时,模块会利用额外的线性投影来处理这些额外信息,这在v3中尤其重要。
4. 版本与实现细节
v1和v2版本关注于双向特征处理,v3引入额外的输入以增强模型在有上下文输入时的表现。- 每个版本在不同的应用场景中都有其适用性,特别是当需要考虑不同方向的信息进行整合的时候。
5. 实用性与集成
- 这个模块通过多种计算路径(比如使用fused kernel)提供了多种解决方案,可以高速处理中长序列的数据。
- 通过其独特的双向和额外输入处理能力,该模块适合在复杂特征处理中提升模型的表现。
这种设计使
Mamba成为一个灵活且功能强大的深度学习模块,能够适应各种复杂的任务需求。 - User
那么现在再CrossMamba中如何处理的两个输入特征?
- Assistant
在
CrossMamba模块中,有两个输入特征:m1和m2。以下是详细分析它们在CrossMamba中的处理流程:1. 初始化
CrossMamba模块初始化时,创建了一个包含Mamba实例的模块,以及两个LayerNorm实例和一个深度卷积层。这里Mamba模块的bimamba_type设为"v3",这意味着它能处理额外输入特征。2. 前向传播 (
forward方法)-
输入特征:
m1: 主输入特征,代表主要待处理的序列。m1_resi: 残差特征,与m1相加后用于更新m1。m2: 额外输入特征,供Mamba模块在bimamba_type="v3"时使用,意味着在处理主输入特征m1的同时,还要考虑m2的信息。
-
步骤与操作:
- 特征组合:
- 先将
m1和m1_resi相加形成更新后的m1以保留残差信息:m1_resi = m1 + m1_resi
- 先将
- 全局特征处理:
- 使用
cross_mamba(即Mamba模块实例)处理更新后的m1,同时将m2通过extra_emb输入传递给Mamba模块:global_f = self.cross_mamba(m1, extra_emb=m2) Mamba模块按照bimamba_type="v3"处理这两个输入:m1作为主要输入被处理。m2作为额外的上下文信息,通过并行数据流程与m1一起,以融合来自不同来源的特征信息。
- 使用
- 全局特征输出:
- 模块返回融合后的全局特征
global_f,该特征结合了来自m1和m2的信息。
- 模块返回融合后的全局特征
- 特征组合:
总结
CrossMamba模块通过Mamba模块的v3版本,对主输入m1和附加的上下文输入m2进行交叉集成和处理。这一设计允许在复杂上下文中有效整合信息,在多模态或需要结合多种信号进行特征融合的应用中尤为有用。 -
- User
分析经过这样的crossmamba处理,有何好处?
- Assistant
通过
CrossMamba模块所设计的这种处理策略带来了以下几个好处:1. 特征融合
-
多源信息整合:将主要特征
m1和附加特征m2结合起来,可以让模型同时考虑多个来源的信息。这对于处理需要多模态信息的任务(如视觉和语言结合的任务)非常有用。 -
增强上下文感知:通过附加信息
m2的引入,模型能获取额外的上下文信息,这可以提高对复杂输入数据的理解和处理能力。
2. 增强的模型表达能力
-
双向处理能力:
Mamba的不同版本特别是v3中对双向处理机制的支持,使得模型能更好地捕捉特征序列中的时序依赖信息。 -
灵活的网络设计:通过允许额外特征处理,
CrossMamba提供了一种灵活的网络设计,使得模型可以是非常模块化和可定制的,为扩展和特定任务的适配创造了条件。
3. 提高模型性能
-
幂次方增强:对输入进行深度卷积处理后,模型可以获得类似于幂次方增强的效果,加强了模型对局部和全局特征的提取能力。
-
有效的归一化:通过多重 LayerNorm 操作,模型的梯度流动更稳定,加速训练过程并提高收敛速度。
4. 提升对长序列数据的处理能力
-
状态保持和高效运算:借助
Mamba中的特殊内置函数(如mamba_inner_fn),该模块能有效地处理长序列数据,提高序列建模效率。 -
缓存和状态管理:模型设计中引入的缓存机制使得长序列的推断过程能够节约计算资源,提高计算效率。
5. 灵活性和可扩展性
- 适配不同任务:通过允许快速路径和带外输入,
CrossMamba能够被轻松调整以适应不同的任务需求和数据规格。
综上所述,
CrossMamba的设计提供了一种强大而灵活的框架,对于需要跨模态、多来源特征融合、长序列处理以及总体上需要增强上下文理解的应用非常有帮助。这也为设计更复杂和智能的神经网络奠定了坚实基础,有助于解决众多新的研究和应用挑战。 -
- User
那么现在是否可以增加一个分支,强调特征差异
- Assistant
增加一个分支来强调特征差异是一个很好的想法,这可以帮助模型更好地识别和利用输入之间的变化和细微差别。这种方法通常用于多模态融合中,以突出输入模态之间的差异。以下是如何在
CrossMamba模块中实现这一点的一种方法:增加特征差异分支
-
差异计算:
- 添加一个分支来计算输入特征之间的差异,例如之间的元素级差异:
diff = m1 - m2。
- 添加一个分支来计算输入特征之间的差异,例如之间的元素级差异:
-
非线性变换:
- 使用一个小型网络或激活函数对差异特征进行变换,以提供更多的非线性能力:
diff_transformed = nn.ReLU()(self.diff_fc(diff)) diff_fc可以是一个简单的nn.Linear层,用于处理差异特征。
- 使用一个小型网络或激活函数对差异特征进行变换,以提供更多的非线性能力:
-
整合差异特征:
- 将差异特征与原有的融合特征合并,可能使用简单的连接或加权的方式:
global_feature_with_diff = global_f + diff_transformed # 或者使用 torch.cat([...], dim=-1)
- 将差异特征与原有的融合特征合并,可能使用简单的连接或加权的方式:
-
加入模型中:
- 将上述过程集成到
CrossMamba的forward方法中:class CrossMambaWithDiff(nn.Module): def __init__(self, dim): super(CrossMambaWithDiff, self).__init__() self.cross_mamba = Mamba(dim, bimamba_type='v3') self.norm1 = LayerNorm(dim, 'with_bias') self.norm2 = LayerNorm(dim, 'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) self.diff_fc = nn.Linear(dim, dim) # 新增的线性层 def forward(self, m1, m1_resi, m2): m1_resi = m1 + m1_resi global_f = self.cross_mamba(m1, extra_emb=m2) # 计算其差异并变换 diff = m1 - m2 diff_transformed = nn.ReLU()(self.diff_fc(diff)) # 将差异转化后的特征与全局特征整合 global_feature_with_diff = global_f + diff_transformed return global_feature_with_diff
- 将上述过程集成到
好处
- 识别特征对比:通过强调输入特征之间的差异,模型可以更好地捕捉输入特征之间的对比与变化。
- 提高判别力:差异分支的加入可以提供额外的监督信号,帮助模型提高对重要变化和细节的捕捉能力。
- 更丰富的特征表示:通过整合不同来源的信息与差异特性,模型的特征表示将更为丰富和多样化。
通过这种方式,可以显著增强
CrossMamba模型在多源、多模态数据处理中的表现,使其能在多样化输入下表现得更出色。 -
- User
不要直接相减,最好处理为差异权重因子,然后再与原始特征相乘。
- Assistant
为了实现您提到的特征差异权重因子,可以通过以下步骤将差异特征转换为权重,然后与原始特征进行逐元素乘法操作,以强调特征差异。以下是如何实现这一点的一个例子:
增加特征差异权重分支
-
差异计算:
- 计算输入特征之间的差异,如绝对差值:
diff = torch.abs(m1 - m2)。
- 计算输入特征之间的差异,如绝对差值:
-
转换为权重因子:
- 使用一个或多个线性层将差异特征转换为权重因子,这些因子将在特征上进行乘法操作:
weight_factor = torch.sigmoid(self.diff_fc(diff)) - 使用
sigmoid函数将权重限制在[0, 1]之间。
- 使用一个或多个线性层将差异特征转换为权重因子,这些因子将在特征上进行乘法操作:
-
权重修改原始特征:
- 将生成的权重因子与原始特征相乘,以增强或抑制某些特征:
emphasized_m1 = m1 * weight_factor
- 将生成的权重因子与原始特征相乘,以增强或抑制某些特征:
-
整合到
CrossMamba模块:- 将这种机制集成到
CrossMamba的forward方法中:class CrossMambaWithWeight(nn.Module): def __init__(self, dim): super(CrossMambaWithWeight, self).__init__() self.cross_mamba = Mamba(dim, bimamba_type='v3') self.norm1 = LayerNorm(dim, 'with_bias') self.norm2 = LayerNorm(dim, 'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) self.diff_fc = nn.Linear(dim, dim) # 新增的线性层 def forward(self, m1, m1_resi, m2): m1_resi = m1 + m1_resi global_f = self.cross_mamba(m1, extra_emb=m2) # 计算其差异并变换为权重因子 diff = torch.abs(m1 - m2) weight_factor = torch.sigmoid(self.diff_fc(diff)) # 用权重因子强调或抑制特征 emphasized_m1 = m1 * weight_factor # 使用经过变换的特征进行后续处理 emphasized_global_f = self.cross_mamba(emphasized_m1, extra_emb=m2) return emphasized_global_f
- 将这种机制集成到
好处
- 适应性增强:相比直接相减,权重因子的使用可以动态调整每个特征的影响力。
- 更精细的特征控制:通过控制特征的权重,可以更细粒度地增强或抑制输入特征中的重要信息。
- 更好的特征融合:在特征融合过程中,权重因子有助于平衡和优化来自多源输入的信息。
通过这种方法,可以在对特征差异进行强调的同时,确保整体模型的稳定性和特征整合的有效性。这个模块能够适应不同来源的输入特征,从而提升模型在多模态融合任务中的表现。
-
- User
有一个疑问,差异因子仅仅与m1相乘吗?
- Assistant
根据您描述的需求,差异因子可以不仅与
m1相乘,也可以与m2相乘,甚至可以用于其他更复杂的操作。使用差异因子的目的是根据输入特征间的差异来调整模型处理特征的方式,因此可以灵活地应用这个因子。以下是一些可能的应用方式:1. 与
m1和m2同时相乘- 动机: 考虑输入特征之间的对称性和相对平衡。
- 实现:
emphasized_m1 = m1 * weight_factor emphasized_m2 = m2 * weight_factor - 解释: 这种做法对两个输入特征都施加了权重因子,以均衡地掩盖或加强它们的某些区域。
2. 交叉相乘
- 动机: 增强差异影响,通过权重因子使两个特征的变化反向互补。
- 实现:
emphasized_m1 = m1 * weight_factor emphasized_m2 = m2 * (1 - weight_factor) - 解释: 这里,
m1的差异较显著部分被放大,而相应的m2部分会趋于“平滑”(或相反),确保总信息量被巧妙组合。
3. 加权融合后的特征
- 动机: 在将两个特征进行某种形式的融合后再应用权重因子。
- 实现:
combined_feature = alpha * m1 + (1 - alpha) * m2 combined_emphasized = combined_feature * weight_factor - 解释: 这在某些融合策略(例如加权平均融合)之后应用差异因子来强调综合特征中的变化部分。
应用这个策略的好处
- 灵活增强模型能力: 不同策略能根据任务需求灵活调整模型对特征处理的细节。
- 实现更复杂的特征交互: 差异因子增强了特征之间的动态交互能力,能够适应不同任务环境。
- 适用于多任务与多模态场景: 在复杂数据场景中,通过差异因子对多个输入进行差异化处理,使得模型表现更强。
总结来说,根据模型的设计目标和任务需求,可以对差异因子进行不同的应用和实现,以最大化模型的处理能力和适应范围。
- User
上文中的crossmamba是在""" Basic mamda_fetrack model. """ import math import os from typing import List
import torch from torch import nn from torch.nn.modules.transformer import _get_clones
from lib.models.layers.head import build_box_head from lib.utils.box_ops import box_xyxy_to_cxcywh from lib.models.mamba_fetrack.models_mamba import create_block from timm.models import create_model import torch.nn.functional as F from lib.models.mamba_fetrack.mamba_cross import CrossMamba from thop import profile
class Mamba_FEtrack(nn.Module): """ This is the base class for mamda_fetrack """
def __init__(self, visionmamba, cross_mamba, box_head, aux_loss=False, head_type="CORNER"): """ Initializes the model. Parameters: transformer: torch module of the transformer architecture. aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. """ super().__init__() self.backbone = visionmamba self.cross_mamba= cross_mamba self.box_head = box_head self.aux_loss = aux_loss self.head_type = head_type if head_type == "CORNER" or head_type == "CENTER": self.feat_sz_s = int(box_head.feat_sz) self.feat_len_s = int(box_head.feat_sz ** 2) if self.aux_loss: self.box_head = _get_clones(self.box_head, 6) def forward(self, template: torch.Tensor, search: torch.Tensor, event_template: torch.Tensor, event_search: torch.Tensor, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False, ): rgb_feature = self.backbone.forward_features( z=template, x=search, #[B, 320, 384] inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False) event_feature = self.backbone.forward_features(z=event_template, x=event_search, #[B, 320, 384] inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False) residual_event_f = 0 residual_rgb_f = 0 event_f = self.cross_mamba(event_feature,residual_event_f,rgb_feature) + event_feature rgb_f = self.cross_mamba(rgb_feature,residual_rgb_f,event_feature) + rgb_feature event_searh = event_f[:, -self.feat_len_s:] rgb_search = rgb_f[:, -self.feat_len_s:] x = torch.cat((event_searh,rgb_search),dim=-1) # Forward head feat_last = x if isinstance(x, list): feat_last = x[-1] out = self.forward_head(feat_last, None) out['backbone_feat'] = x return out def forward_head(self, cat_feature, gt_score_map=None): """ cat_feature: output embeddings of the backbone, it can be (HW1+HW2, B, C) or (HW2, B, C) """ search_feature = cat_feature opt = (search_feature.unsqueeze(-1)).permute((0, 3, 2, 1)).contiguous() # opt.shape = torch.Size([B, 1, 384, 256]) bs, Nq, C, HW = opt.size() opt_feat = opt.view(-1, C, self.feat_sz_s, self.feat_sz_s) # opt_feat.shape = torch.Size([B, 384, 16, 16]) if self.head_type == "CORNER": # run the corner head pred_box, score_map = self.box_head(opt_feat, True) outputs_coord = box_xyxy_to_cxcywh(pred_box) outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map, } return out elif self.head_type == "CENTER": # run the center head score_map_ctr, bbox, size_map, offset_map = self.box_head(opt_feat, gt_score_map) outputs_coord = bbox outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map_ctr, 'size_map': size_map, 'offset_map': offset_map} return out else: raise NotImplementedErrordef build_mamba_fetrack(cfg, training=True): current_dir = os.path.dirname(os.path.abspath(file)) # This is your Project Root pretrained_path = os.path.join(current_dir, 'pretrained_models') if cfg.MODEL.PRETRAIN_FILE and ('Mamba_FETrack' not in cfg.MODEL.PRETRAIN_FILE) and training: pretrained = os.path.join(pretrained_path, cfg.MODEL.PRETRAIN_FILE) else: pretrained = ''
backbone = create_model( model_name= cfg.MODEL.BACKBONE.TYPE, pretrained= pretrained, num_classes=1000, drop_rate=0.0, drop_path_rate=cfg.TRAIN.DROP_PATH_RATE, drop_block_rate=None, img_size=256 ) hidden_dim = 384 cross_mamba = CrossMamba(hidden_dim) box_head = build_box_head(cfg, hidden_dim*2) model = Mamba_FEtrack( backbone, cross_mamba, box_head, aux_loss=False, head_type=cfg.MODEL.HEAD.TYPE, ) if 'Mamba_FETrack' in cfg.MODEL.PRETRAIN_FILE and training: checkpoint = torch.load(cfg.MODEL.PRETRAIN_FILE, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["net"], strict=False) print('Load pretrained model from: ' + cfg.MODEL.PRETRAIN_FILE) return model中调用的,主要是 residual_event_f = 0 residual_rgb_f = 0 event_f = self.cross_mamba(event_feature,residual_event_f,rgb_feature) + event_feature rgb_f = self.cross_mamba(rgb_feature,residual_rgb_f,event_feature) + rgb_feature 这里。那么现在分析一下:引入差异强调是否合适,且应该如何设计class CrossMamba(nn.Module): def init(self, dim): super(CrossMamba, self).init() self.cross_mamba = Mamba(dim,bimamba_type="v3") self.norm1 = LayerNorm(dim,'with_bias') self.norm2 = LayerNorm(dim,'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) def forward(self,m1,m1_resi,m2):
m1_resi = m1+m1_resi
global_f = self.cross_mamba(m1, extra_emb=m2) return global_f? - Assistant
在
Mamba_FEtrack模型中,CrossMamba模块被用于处理两个特征输入:event_feature和rgb_feature。这些特征分别来自事件和RGB图像数据,通过CrossMamba进行融合。考虑到引入差异强调机制的目标是通过对输入之间的不同进行加权处理以提高特征表示能力,因此在该上下文中这是合理的。下面是如何在CrossMamba中设计和实现这一机制的建议:差异强调的合理性
-
多模态融合:
event_feature和rgb_feature代表两种不同的输入模态。在视觉任务中,事件和RGB图像的数据类型存在很大差异,通过差异权重强调有助于更好地整合这些多模态信息。
-
提高特征判别力:
- 差异强调可以突出特定模态的信息,潜在地增强目标检测、特征匹配等任务的判别力。例如,当事件特征很强时,可能需要在融合时更多地关注事件特征。
设计
CrossMamba中的差异强调机制为了将差异强调机制引入到
CrossMamba中,我们可以按以下方式重新设计:-
添加差异计算和转换:
- 首先计算
m1和m2的差异,考虑使用绝对差值以确保正值。 - 将差异通过非线性变换或网络层转化为权重因子,控制特征的强度。
- 首先计算
-
应用权重因子:
- 使用计算出的权重因子来调整
m1和(或)m2,按照具体需求增强或抑制某些模态特征。
- 使用计算出的权重因子来调整
-
整合权重因子:
- 将调整后的特征发送到
Mamba模块中,完成进一步的联合处理。
- 将调整后的特征发送到
下面是一个可实施的代码示例:
class CrossMambaWithDiff(nn.Module): def __init__(self, dim): super(CrossMambaWithDiff, self).__init__() self.cross_mamba = Mamba(dim, bimamba_type="v3") self.norm1 = LayerNorm(dim, 'with_bias') self.norm2 = LayerNorm(dim, 'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) self.diff_fc = nn.Linear(dim, dim) def forward(self, m1, m1_resi, m2): m1_resi = m1 + m1_resi # 计算特征之间的差异 diff = torch.abs(m1 - m2) weight_factor = torch.sigmoid(self.diff_fc(diff)) # 用权重因子调整特征 emphasized_m1 = m1 * weight_factor emphasized_m2 = m2 * weight_factor # 将调整后的特征整合到 global_f 中 global_f = self.cross_mamba(emphasized_m1, extra_emb=emphasized_m2) return global_f注意事项
- 平衡性: 确保在处理过程中不会过于强调某个输入模态,从而导致其他模态信息缺失。
- 实验验证: 在实践中不同任务对差异强调的需求可能不同,建议通过实验调参以获得最佳性能。
这种设计能够有效地在多模态场景下利用各模态的特异性以及它们之间的差异,从而改进特征融合效果。
-
- User
改成了:class CrossMamba(nn.Module): def init(self, dim): super(CrossMamba, self).init() self.cross_mamba = Mamba(dim, bimamba_type='v3') self.norm1 = LayerNorm(dim, 'with_bias') self.norm2 = LayerNorm(dim, 'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) self.diff_fc = nn.Linear(dim, dim) # 新增的线性层
def forward(self, m1, m1_resi, m2): m1_resi = m1 + m1_resi global_f = self.cross_mamba(m1, extra_emb=m2) # 计算其差异并变换为权重因子 diff = torch.abs(m1 - m2) weight_factor = torch.sigmoid(self.diff_fc(diff)) # 用权重因子强调或抑制特征 emphasized_m1 = m1 * weight_factor emphasized_m2 = m2 * weight_factor # 使用经过变换的特征进行后续处理 emphasized_global_f = self.cross_mamba(emphasized_m1, extra_emb= emphasized_m2) return emphasized_global_f,检查一下逻辑 - Assistant
在这个重构的
CrossMamba类中,你采用了差异强调的策略来处理特征。以下是对代码逻辑的检查和分析:代码逻辑和流程
-
初始化:
cross_mamba,norm1,norm2和dwconv都被初始化。你新加入了一个名为diff_fc的nn.Linear层,用于将特征差异变换为权重因子。
-
前向传播:
-
特征融合:
- 使用
Mamba模块的v3版本处理m1和m2,首次融合结果储存在global_f中。
- 使用
-
计算差异和生成权重因子:
diff是m1和m2之间绝对值的差异大小。diff_fc将这种差异变换为权重因子weight_factor,再通过sigmoid将因子限制在[0, 1]范围内。
-
特征强调:
- 使用生成的
weight_factor对m1和m2进行强调或抑制,生成emphasized_m1和emphasized_m2。
- 使用生成的
-
重新特征融合:
emphasized_global_f是通过再次调用Mamba模块对强调后的特征进行处理得到的。这一步可能是为了更深层次地进行特征融合。
-
-
返回值:
- 返回的是经过差异处理后的
emphasized_global_f。
- 返回的是经过差异处理后的
逻辑检查和建议
-
重复特征融合:
global_f被计算以后没有直接返回,而是继续使用了强调阶段。这两个阶段的逻辑是否清晰需要依据具体任务需求来决定,看是否符合设计初衷。 -
计算效率: 重复调用
cross_mamba(Mamba内核)的代价可能较高,如果此操作对次任务的性能提升较小,可以考虑优化调用次数。 -
特征变换的一致性: 确保
weight_factor对m1和m2的应用是符合任务需求的。如果m1和m2在你的任务中担任对称角色(即二者相互可交换),则这样操作是合适的。 -
使用差异强调: 如果只是对
m1和m2应用相同的权重因子,可以考虑通过上述方式进一步控制两种模态特征的贡献度。
总体来说,代码的逻辑是合理的,并很好的将差异特征的影响加入到了特征融合过程中,需要根据具体任务目标进一步调优和测试以确保适应性和效果。
-
- User
这里使用sigmoid还是Tanh?
- Assistant
sigmoid和tanh是两种常用的激活函数,各自有其适用的场景和特性。使用
sigmoid-
输出范围:
sigmoid将输入值限制在[0, 1]之间。这意味着差异的转化权重因子只能用来减小或维持原始特征的值,但不会放大。这在某些需要抑制特征的任务中是合适的。 -
应用场景: 常用于需要正值的权重因子的场景,比如注意力权重(作为缩放比例),因为它可以将特征的影响限制在不增强的前提下进行灵活缩减。
使用
tanh-
输出范围:
tanh将输入值限制在[-1, 1]之间。这样,当权重因子为负时,原始特征可以被反转。权重因子为正时,特征可以被略微放大。 -
应用场景: 适用于希望增强某些特征而同时抑制其他特征的任务,因为
tanh几乎是在正负方向上都可以起作用。在强调特征差异,同时允许少量增强时,tanh是一个合适的选择。
在你的场景中的选择建议
如果你希望权重因子仅用于按比例缩放,而不增加特征值的幅度,
sigmoid是比较合适的选择。如果你期望某些特征可以被增强,而另一些特征被抑制,tanh会给予更大的灵活性。但需要根据任务需求去实验二者的效果。通常你可以开始使用
sigmoid,因为它通常在起始时能够保持系统稳定。如若发现特征需要扩展,tanh可以在下一轮尝试中作为替代。 -
- User
在class CrossMamba(nn.Module): def init(self, dim): super(CrossMamba, self).init() self.cross_mamba = Mamba(dim, bimamba_type='v3') self.norm1 = LayerNorm(dim, 'with_bias') self.norm2 = LayerNorm(dim, 'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) self.diff_fc = nn.Linear(dim, dim) # 新增的线性层
def forward(self, m1, m1_resi, m2): m1_resi = m1 + m1_resi # 计算其差异并变换为权重因子 diff = torch.abs(m1 - m2) weight_factor = torch.sigmoid(self.diff_fc(diff)) # 用权重因子强调或抑制特征 emphasized_m1 = m1 * weight_factor emphasized_m2 = m2 * weight_factor # 使用经过变换的特征进行后续处理 emphasized_global_f = self.cross_mamba(emphasized_m1, extra_emb= emphasized_m2) return emphasized_global_f中现在是几个分支? - Assistant
在您提供的
CrossMamba类中的forward方法中,可以看到主要有以下几个逻辑分支或步骤:-
残差计算:
- 计算
m1_resi = m1 + m1_resi,这是对输入特征m1进行残差更新,不算一个分支,而是特征预处理的一部分。
- 计算
-
差异计算:
- 计算
diff = torch.abs(m1 - m2),找到两个输入特征m1和m2之间的差异。
- 计算
-
权重因子生成:
- 使用线性层和
sigmoid激活函数将差异转化为权重因子:weight_factor = torch.sigmoid(self.diff_fc(diff))
- 使用线性层和
-
特征强调或抑制:
- 使用权重因子对特征进行强调或抑制,两条平行逻辑路径:
emphasized_m1 = m1 * weight_factor:对m1特征进行调整。emphasized_m2 = m2 * weight_factor:对m2特征进行调整。
- 使用权重因子对特征进行强调或抑制,两条平行逻辑路径:
-
强调后的特征处理:
- 将调整后的特征
emphasized_m1和emphasized_m2传入cross_mamba模块进行进一步处理,得到emphasized_global_f。
- 将调整后的特征
综上所述,代码中的逻辑可以视为五个步骤,其中“特征强调或抑制”部分既可以看作是两个分支(分别用于
m1和m2),也可以看作是使用同一个weight_factor的两个平行处理路径。因此,若狭义理解为逻辑分支的话,此代码中有两个分支:一个是对
m1与m2差异计算后生成的权重因子的调整,两条平行路径。最后,调整后的两个特征进入融合操作,进一步演绎为单一处理路径。 -
- User
对比:class CrossMamba(nn.Module): def init(self, dim): super(CrossMamba, self).init() self.cross_mamba = Mamba(dim,bimamba_type="v3") self.norm1 = LayerNorm(dim,'with_bias') self.norm2 = LayerNorm(dim,'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) def forward(self,m1,m1_resi,m2):
m1_resi = m1+m1_resi
global_f = self.cross_mamba(m1, extra_emb=m2) return global_fclass CrossMamba(nn.Module): def init(self, dim): super(CrossMamba, self).init() self.cross_mamba = Mamba(dim, bimamba_type='v3') self.norm1 = LayerNorm(dim, 'with_bias') self.norm2 = LayerNorm(dim, 'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) self.diff_fc = nn.Linear(dim, dim) # 新增的线性层
def forward(self, m1, m1_resi, m2): m1_resi = m1 + m1_resi # 计算其差异并变换为权重因子 diff = torch.abs(m1 - m2) weight_factor = torch.sigmoid(self.diff_fc(diff)) # 用权重因子强调或抑制特征 emphasized_m1 = m1 * weight_factor emphasized_m2 = m2 * weight_factor # 使用经过变换的特征进行后续处理 emphasized_global_f = self.cross_mamba(emphasized_m1, extra_emb= emphasized_m2) return emphasized_global_f - Assistant
这两个版本的
CrossMamba类在设计和功能上有显著差异。以下是两者的对比分析:原始版本的
CrossMamba-
结构简单:
- 直接使用
cross_mamba模块处理两个输入特征m1和m2。没有对输入特征进行额外的权重调整或非线性变换处理。
- 直接使用
-
处理逻辑:
- 直接将更新后的
m1(通过加上m1_resi)和m2输入到Mamba模块中进行处理。
- 直接将更新后的
-
优缺点:
- 优点: 计算效率高,因为没有额外的权重计算和特征调整步骤。
- 缺点: 缺乏对输入特征差异的细粒度控制,可能丢失一些通过特征差异捕捉到的细节信息。
带有差异强调的
CrossMamba-
增加复杂性:
- 使用
diff_fc和sigmoid激活函数计算两个输入特征m1和m2之间的权重因子。
- 使用
-
处理逻辑:
- 基于权重因子对
m1和m2进行调整,然后输入到Mamba模块进行处理。
- 基于权重因子对
-
优缺点:
- 优点:
- 通过计算差异并生成权重因子,这个版本能够在处理特征时动态调整输入的影响,从而可能实现对特征的更细粒度操控。
- 适合于在需要强调或抑制某些特定特征的场景。
- 缺点:
- 计算开销增加,因为引入了额外的计算步骤(如线性层和
sigmoid)。 - 复杂性增加,可能需要更多的调参和验证来确保权重因子的合理性。
- 计算开销增加,因为引入了额外的计算步骤(如线性层和
- 优点:
适用场景
-
原始版本适合于对计算效率要求更高且特征间差异不明显或不需要进行差异化处理的应用场景。
-
差异强调版本适合于需要动态特征调整,尤其是在多模态或多输入场景中,各个输入特征对整体结果影响可能差异较大时,非常有用。
总结来说,这两种实现各有特点,选择哪个版本要依据具体的应用需求、性能限制以及对特征处理的细致程度要求来确定。
-
- User
是否可以在原始版本中,增加新的分支,单独处理差异强调?
- Assistant
在原始版本中添加一个新的分支来单独处理差异强调是一个合理且灵活的扩展。这种设计可以保持原有路径的简洁,并提供一个可选的差异处理路径。以下是如何实现这个想法的一种方法:
结构设计
-
保持原有路径不变:保持原有的特征处理流程不变,使用
cross_mamba继续处理m1 + m1_resi和m2。 -
新增差异处理分支:
- 添加一个额外的功能模块用于计算和应用权重因子。
- 将差异处理的结果与原有结果结合起来,或者进行单独的后续处理。
下面是一个可能的实现示例:
class ExtendedCrossMamba(nn.Module): def __init__(self, dim): super(ExtendedCrossMamba, self).__init__() self.cross_mamba = Mamba(dim, bimamba_type='v3') self.norm1 = LayerNorm(dim, 'with_bias') self.norm2 = LayerNorm(dim, 'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) self.diff_fc = nn.Linear(dim, dim) # 新增的线性层用于差异处理 def forward(self, m1, m1_resi, m2): # 原始路径 m1_resi = m1 + m1_resi global_f = self.cross_mamba(m1, extra_emb=m2) # 新增的差异处理分支 diff = torch.abs(m1 - m2) weight_factor = torch.sigmoid(self.diff_fc(diff)) emphasized_m1 = m1 * weight_factor emphasized_m2 = m2 * weight_factor emphasized_global_f = self.cross_mamba(emphasized_m1, extra_emb=emphasized_m2) # 两个路径结果的整合 combined_f = global_f + emphasized_global_f # 这一步可以根据需求进行定制,比如使用加权和 return combined_f设计好处
- 灵活性: 保持了原有的简单路径,并在需要时引入差异处理,提供了可选择的复杂性。
- 性能优化: 在不需要差异强调时,可以简化为原有路径。
- 可扩展性: 可以在不大幅更改原有代码的情况下,调整和增加新的处理步骤。
注意事项
- 如何整合两个路径:
combined_f = global_f + emphasized_global_f中的整合逻辑可以是元素级相加、拼接或者其他融合方式。 - 参数调优: 当差异分支被引入时,可能需要调节
diff_fc的参数以及融合策略,以确保在不同场景下的表现稳定。 - 实验验证: 在复杂模型中引入多个路径后,需要通过实验确保模型训练的稳定性和精度。
-