分析模型中有几个模块,执行什么功能?from typing import Any, Dict, Optional, Tuple, Union import torch…
- User
分析模型中有几个模块,执行什么功能?from typing import Any, Dict, Optional, Tuple, Union import torch import torch.utils.checkpoint from torch import nn
FIXME
from mamba_ssm.modules.mamba_simple import Mamba from transformers.models.mamba.modeling_mamba import logger, is_fast_path_available, mamba_inner_fn, causal_conv1d_fn, causal_conv1d_update, selective_state_update, selective_scan_fn from transformers.models.mamba.modeling_mamba import MambaRMSNorm, MambaPreTrainedModel, MambaCache, MambaOutput, MambaMixer from transformers import MambaConfig from transformers.activations import ACT2FN
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
Define the gating model
class MambaMixer(nn.Module): """ Compute ∆, A, B, C, and D the state space parameters and compute the
contextualized_states. A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective) ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4, and is why Mamba is called selective state spaces) """def __init__(self, config, layer_idx, head_num=1): super().__init__() self.hidden_size = config.hidden_size self.ssm_state_size = config.state_size self.conv_kernel_size = config.conv_kernel self.intermediate_size = config.intermediate_size self.time_step_rank = config.time_step_rank self.layer_idx = layer_idx self.head_num = head_num self.use_conv_bias = config.use_conv_bias self.conv1d = nn.Conv1d( in_channels=self.intermediate_size, out_channels=self.intermediate_size, bias=config.use_conv_bias, kernel_size=config.conv_kernel, groups=self.intermediate_size, padding=config.conv_kernel - 1, ) self.conv1d_back = nn.Conv1d( in_channels=self.intermediate_size, out_channels=self.intermediate_size, bias=config.use_conv_bias, kernel_size=config.conv_kernel, groups=self.intermediate_size, padding=config.conv_kernel - 1, ) self.activation = config.hidden_act self.act = ACT2FN[config.hidden_act] # projection of the input hidden states self.in_proj = nn.Linear(self.hidden_size, self.intermediate_size * 2, bias=config.use_bias) self.in_proj_dif = nn.Linear(self.hidden_size, self.intermediate_size * 2, bias=config.use_bias) # selective projection used to make dt, B and C input dependant self.x_proj = nn.Linear(self.intermediate_size, self.time_step_rank + self.ssm_state_size * 2, bias=False) self.x_proj_back = nn.Linear(self.intermediate_size, self.time_step_rank + self.ssm_state_size * 2, bias=False) self.x_proj_dif = nn.Linear(self.intermediate_size, self.time_step_rank + self.ssm_state_size * 2, bias=False) self.x_proj_dif_back = nn.Linear(self.intermediate_size, self.time_step_rank + self.ssm_state_size * 2, bias=False) # time step projection (discretization) self.dt_proj = nn.Linear(self.time_step_rank, self.intermediate_size, bias=True) self.dt_proj_back = nn.Linear(self.time_step_rank, self.intermediate_size, bias=True) self.dt_proj_dif = nn.Linear(self.time_step_rank, self.intermediate_size, bias=True) self.dt_proj_dif_back = nn.Linear(self.time_step_rank, self.intermediate_size, bias=True) self.linear_hid2 = nn.Linear(self.intermediate_size,2*self.intermediate_size, bias=True) self.linear_hid2_back = nn.Linear(self.intermediate_size, 2*self.intermediate_size, bias=True) # S4D real initialization. These are not discretized! # The core is to load them, compute the discrete states, then write the updated state. Keeps the memory bounded A = torch.arange(1, self.ssm_state_size + 1, dtype=torch.float32)[None, :] A = A.expand(self.intermediate_size, -1).contiguous() self.A_log = nn.Parameter(torch.log(A)) self.A_log_back = nn.Parameter(torch.log(A)) self.D = nn.Parameter(torch.ones(self.intermediate_size)) self.D_back = nn.Parameter(torch.ones(self.intermediate_size)) self.out_proj = nn.Linear(1*self.intermediate_size, self.hidden_size, bias=config.use_bias) self.out_LN = nn.LayerNorm(self.intermediate_size) self.use_bias = config.use_bias if not is_fast_path_available: logger.warning_once( "The fast path is not available because on of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)`" " is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and" " https://github.com/Dao-AILab/causal-conv1d" ) def cuda_kernels_forward(self, hidden_states: torch.Tensor, hidden_states_dif: torch.Tensor, cache_params: Optional[MambaCache] = None, cache_params_2: Optional[MambaCache] = None): # 1. Gated MLP's linear projection batch_size, seq_len, _ = hidden_states.shape flag_one = False if hidden_states_dif is None: flag_one = True hidden_states_dif = hidden_states hidden_states = torch.cat([hidden_states, hidden_states_dif], dim=0) # input_hidden_states_dif = hidden_states_dif projected_states = self.in_proj(hidden_states).transpose(1, 2) projected_states_dif = self.in_proj_dif(hidden_states).transpose(1, 2) # process hidden_states, gate = projected_states.chunk(2, dim=1) hidden_states_dif, gate_dif = projected_states_dif.chunk(2, dim=1) # gate = gate[:batch_size] gate = gate_dif[batch_size:] if not flag_one else gate[:batch_size] gate_back = gate.flip(-1) # 2. Convolution sequence transformation conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0), self.conv1d.weight.size(2)) # conv_weights_back = self.conv1d_back.weight.view(self.conv1d_back.weight.size(0), self.conv1d_back.weight.size(2)) hidden_states_cat = causal_conv1d_fn( hidden_states, conv_weights, self.conv1d.bias, activation=self.activation ) # hidden_states_back = hidden_states.flip(-1) # hidden_states_back_cat = causal_conv1d_fn( # hidden_states_back, conv_weights_back, self.conv1d_back.bias, activation=self.activation # ) hidden_states_back_cat = hidden_states_cat.flip(-1) # 3. State Space Model sequence transformation # 3.a. input varying initialization of time_step, B and C hidden_states = hidden_states_cat[:batch_size] # [batch, seq_len, intermediate_size] ## 反向排序: hidden_states_back = hidden_states_back_cat[:batch_size] # [batch, seq_len, intermediate_size] ssm_parameters = self.x_proj(hidden_states.transpose(1, 2)) time_step, B, C = torch.split( ssm_parameters, [self.time_step_rank, self.ssm_state_size, self.ssm_state_size], dim=-1 ) discrete_time_step = self.dt_proj.weight @ time_step.transpose(1, 2) # 反向: ssm_parameters_back = self.x_proj_back(hidden_states_back.transpose(1, 2)) time_step_back, B_back, C_back = torch.split( ssm_parameters_back, [self.time_step_rank, self.ssm_state_size, self.ssm_state_size], dim=-1 ) discrete_time_step_back = self.dt_proj_back.weight @ time_step_back.transpose(1, 2) A = -torch.exp(self.A_log.float()) A_back = -torch.exp(self.A_log_back.float()) # 3.c perform the recurrence y ← SSM(A, B, C)(x) time_proj_bias = self.dt_proj.bias.float() if hasattr(self.dt_proj, "bias") else None time_proj_bias_back = self.dt_proj_back.bias.float() if hasattr(self.dt_proj_back, "bias") else None scan_outputs, ssm_state = selective_scan_fn( hidden_states, discrete_time_step, A, B.transpose(1, 2), C.transpose(1, 2), self.D.float(), gate,#None, time_proj_bias, delta_softplus=True, return_last_state=True, ) scan_outputs_back, ssm_state_back = selective_scan_fn( hidden_states.flip(-1), discrete_time_step_back, A_back, B_back.transpose(1, 2), C_back.transpose(1, 2), self.D_back.float(), gate_back, #None, # time_proj_bias_back, delta_softplus=True, return_last_state=True, ) # 4. Final linear projection contextualized_states = self.out_proj((scan_outputs+scan_outputs_back.flip(-1)).transpose(1, 2)) return contextualized_states def forward(self, hidden_states, hidden_states_2, cache_params: Optional[MambaCache] = None, cache_params_2: Optional[MambaCache] = None): if is_fast_path_available and "cuda" in self.x_proj.weight.device.type: return self.cuda_kernels_forward(hidden_states, hidden_states_2, cache_params, cache_params_2) else: raise NotImplementedError("The fast path is not available")class CaMambaBlock(nn.Module): def init(self, config, layer_idx, head_num=1, length=49, craft=False): super().init() self.config = config self.layer_idx = layer_idx self.residual_in_fp32 = config.residual_in_fp32 self.norm = MambaRMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) self.head_num = head_num self.length = length self.craft = craft self.config.intermediate_size = config.intermediate_size self.mixer = MambaMixer(config, layer_idx=layer_idx)
def forward(self, hidden_states, hidden_states_2, cache_params: Optional[MambaCache] = None, cache_params_2: Optional[MambaCache] = None,): residual = hidden_states # if hidden_states_2==None: # residual = hidden_states[:,:,:768] # hidden_states = self.linear(hidden_states) hidden_states = self.norm(hidden_states.to(dtype=self.norm.weight.dtype)) # hidden_states_2 = self.norm(hidden_states_2.to(dtype=self.norm.weight.dtype)) if self.residual_in_fp32: residual = residual.to(torch.float32) hidden_states = self.mixer(hidden_states, hidden_states_2, cache_params=cache_params, cache_params_2=cache_params_2) hidden_states = residual + hidden_states return hidden_statesclass CaMambaModel(MambaPreTrainedModel): def init(self, config, head_num=1, length=49, craft=False): super().init(config)
self.embeddings = nn.Embedding(config.vocab_size, config.hidden_size) self.layers = nn.ModuleList([CaMambaBlock(config, layer_idx=idx) for idx in range(config.num_hidden_layers)]) self.gradient_checkpointing = False self.norm_f = MambaRMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) # Initialize weights and apply final processing self.post_init() def forward( self, inputs_embeds: Optional[torch.LongTensor] = None, inputs_embeds_2: Optional[torch.LongTensor] = None, cache_params: Optional[MambaCache] = None, use_cache: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, **kwargs, # `attention_mask` is passed by the tokenizer and we don't want it ) -> Union[Tuple, MambaOutput]: output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) use_cache = use_cache if use_cache is not None else (self.config.use_cache if not self.training else False) return_dict = return_dict if return_dict is not None else self.config.use_return_dict # FIXME cache_params = None use_cache = False cache_params_2 = cache_params hidden_states = inputs_embeds hidden_states_2 = inputs_embeds_2 all_hidden_states = () if output_hidden_states else None for mixer_block in self.layers: assert len(self.layers)==1 if self.gradient_checkpointing and self.training: hidden_states = self._gradient_checkpointing_func(mixer_block.__call__, hidden_states, cache_params) else: hidden_states = mixer_block(hidden_states, hidden_states_2, cache_params=cache_params, cache_params_2=cache_params_2) if output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) if use_cache: cache_params.seqlen_offset += inputs_embeds.shape[1] cache_params_2.seqlen_offset += inputs_embeds_2.shape[1] hidden_states = self.norm_f(hidden_states) if output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) if not return_dict: return tuple(v for v in [hidden_states, cache_params, all_hidden_states] if v is not None) return MambaOutput( last_hidden_state=hidden_states, cache_params=cache_params if use_cache else None, hidden_states=all_hidden_states, )if name == "main": config = MambaConfig(num_hidden_layers=1) model = CaMambaModel(config, head_num=1, length=49, craft=True) model = model.to(device) model.eval() input_embeds = torch.randn(4, 49, 768).to(device) input_embeds_2 = torch.randn(4, 49, 768).to(device) out1 = model(inputs_embeds=input_embeds, inputs_embeds_2=input_embeds_2).last_hidden_state # print('last_hidden_state:') print(input_embeds.size()) print(input_embeds_2.size()) print(out1.size())
- Assistant
在提供的代码中,分析模型主要分为以下几个模块,每个模块执行特定的功能:
-
MambaMixer 模块:
- 计算状态空间参数 ∆, A, B, C 和 D,以及上下文状态
contextualized_states。 - 实现 selective state spaces 模型的核心,通过使用输入依赖的参数生成,这些参数主要通过线性投影和卷积操作计算得出。
- 包含 CUDA 快路径,以加速特定硬件上的推理。
- 计算状态空间参数 ∆, A, B, C 和 D,以及上下文状态
-
CaMambaBlock 模块:
- 这是一个基于
MambaMixer的层,包含残差连接和规范化(normalization)步骤。 - 执行选择性状态空间模型的计算,并将结果与输入进行残差加法结合。
- 这是一个基于
-
CaMambaModel 模块:
- 这是完整的模型结构,包含词嵌入层、多个
CaMambaBlock层和最终的规范化层。 - 管理模型的前向传播,包括可能的缓存操作和输出状态的选择(例如是否存储和返回每一层的隐藏状态)。
- 这是完整的模型结构,包含词嵌入层、多个
此外,代码中的模型还具备以下功能:
-
Embeddings:
- 通过
nn.Embedding将输入的词索引转换为嵌入向量。
- 通过
-
Gradient Checkpointing:
- 通过
gradient_checkpointing来减少训练时的显存消耗,以增加序列长度或者批量大小。
- 通过
-
Caching(缓存):
- 提供缓存机制,可以在生成序列时重用先前计算的隐藏状态。
总结来说,模型的设计主要围绕着选择性状态空间模型,结合卷积和线性投影,能在序列任务中高效地计算特征表示,并通过模块化设计,实现灵活的架构调整和训练优化。
-
- User
那么相较于原始的mamba模型有何不同?
- Assistant
相较于原始的Mamba模型,该代码实现了一种被称为选择性状态空间模型的变体,主要亮点和区别包括:
-
选择性状态空间:
- 在
MambaMixer模块中,引入了动态输入依赖的状态空间参数。这意味着部分状态空间参数(如时间步长、B和C)会根据输入数据变化,而不仅仅是静态设置。 - 这使得模型能够更灵活地处理输入序列,可能在变化的上下文中更有效地产生输出。
- 在
-
双向处理:
- 实现了对正向和反向序列的处理(通过
hidden_states_back处理),这种机制结合正反向的状态信息,有潜力提升模型理解和生成信息的能力。
- 实现了对正向和反向序列的处理(通过
-
CUDA 快路径:
- 代码集成了CUDA快路径,使得在支持的硬件上能利用特定优化以更快地完成计算。这对模型推理阶段尤其有用。
-
集成卷积层:
- 在模型中加入了卷积操作,这可以帮助捕获局部模式或特征。这是相较于其他线性变换方法的一个增强。
-
多头机制(头部数目可配置):
- 实现了简单的多头机制,使得模型能够在多个特征子空间上并行处理输入。这类似于Attention机制中的多头注意力,但在此通过多个并行状态空间处理实现。
-
MambaBlock的新结构:
CaMambaBlock模块重新组织了Mamba模型中的块,增加了批处理或序列长度上内存优化的潜力,如通过残差连接和混合块的优化可能提高训练稳定性和效率。
总结来看,相较于原始的Mamba模型,该实现可能提供了更多的灵活性、对输入的动态适应能力以及某些硬件上的性能优化。这可能会转化为在特定任务或数据集上的更好表现,尤其是在长序列或需要捕获复杂依赖关系的任务中。
-
- User
在# Copyright (c) 2015-present, Facebook, Inc.
All rights reserved.
import torch import torch.nn as nn from functools import partial from torch import Tensor from typing import Optional
from timm.models.vision_transformer import VisionTransformer, cfg from timm.models.registry import register_model from timm.models.layers import trunc_normal, lecun_normal_
from timm.models.layers import DropPath, to_2tuple from timm.models.vision_transformer import _load_weights
import math
from collections import namedtuple
from mamba_ssm.modules.mamba_simple import Mamba from mamba_ssm.utils.generation import GenerationMixin from mamba_ssm.utils.hf import load_config_hf, load_state_dict_hf
from lib.models.mamba_fetrack.rope import * import random from lib.models.layers.head import build_box_head import importlib import lib.train.admin.settings as ws_settings from .utils import combine_tokens, recover_tokens
try: from mamba_ssm.ops.triton.layernorm import RMSNorm, layer_norm_fn, rms_norm_fn except ImportError: RMSNorm, layer_norm_fn, rms_norm_fn = None, None, None
all = [ 'vim_tiny_patch16_224', 'vim_small_patch16_224', 'vim_base_patch16_224', 'vim_tiny_patch16_384', 'vim_small_patch16_384', 'vim_base_patch16_384', ]
class PatchEmbed(nn.Module): """ 2D Image to Patch Embedding """ def init(self, img_size=224, patch_size=16, stride=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True): super().init() img_size = to_2tuple(img_size) patch_size = to_2tuple(patch_size) self.img_size = img_size self.patch_size = patch_size self.grid_size = ((img_size[0] - patch_size[0]) // stride + 1, (img_size[1] - patch_size[1]) // stride + 1) self.num_patches = self.grid_size[0] * self.grid_size[1] self.flatten = flatten
self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=stride) self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() def forward(self, x): B, C, H, W = x.shape # assert H == self.img_size[0] and W == self.img_size[1], \ # f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." x = self.proj(x) if self.flatten: x = x.flatten(2).transpose(1, 2) # BCHW -> BNC x = self.norm(x) return xclass Block(nn.Module): def init( self, dim, mixer_cls, norm_cls=nn.LayerNorm, fused_add_norm=False, residual_in_fp32=False,drop_path=0., ): """ Simple block wrapping a mixer class with LayerNorm/RMSNorm and residual connection"
This Block has a slightly different structure compared to a regular prenorm Transformer block. The standard block is: LN -> MHA/MLP -> Add. [Ref: https://arxiv.org/abs/2002.04745] Here we have: Add -> LN -> Mixer, returning both the hidden_states (output of the mixer) and the residual. This is purely for performance reasons, as we can fuse add and LayerNorm. The residual needs to be provided (except for the very first block). """ super().__init__() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.mixer = mixer_cls(dim) self.norm = norm_cls(dim) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() if self.fused_add_norm: assert RMSNorm is not None, "RMSNorm import fails" assert isinstance( self.norm, (nn.LayerNorm, RMSNorm) ), "Only LayerNorm and RMSNorm are supported for fused_add_norm" def forward( self, hidden_states: Tensor, residual: Optional[Tensor] = None, inference_params=None ): r"""Pass the input through the encoder layer. Args: hidden_states: the sequence to the encoder layer (required). residual: hidden_states = Mixer(LN(residual)) """ if not self.fused_add_norm: if residual is None: residual = hidden_states else: residual = residual + self.drop_path(hidden_states) hidden_states = self.norm(residual.to(dtype=self.norm.weight.dtype)) if self.residual_in_fp32: residual = residual.to(torch.float32) else: fused_add_norm_fn = rms_norm_fn if isinstance(self.norm, RMSNorm) else layer_norm_fn if residual is None: hidden_states, residual = fused_add_norm_fn( hidden_states, self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) else: hidden_states, residual = fused_add_norm_fn( self.drop_path(hidden_states), self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) hidden_states = self.mixer(hidden_states, inference_params=inference_params) return hidden_states, residual def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return self.mixer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)def create_block( d_model, ssm_cfg=None, norm_epsilon=1e-5, drop_path=0., rms_norm=False, residual_in_fp32=False, fused_add_norm=False, layer_idx=None, device=None, dtype=None, if_bimamba=False, bimamba_type="none", if_devide_out=False, init_layer_scale=None, ): if if_bimamba: bimamba_type = "v1" if ssm_cfg is None: ssm_cfg = {} factory_kwargs = {"device": device, "dtype": dtype} mixer_cls = partial(Mamba, layer_idx=layer_idx, bimamba_type=bimamba_type, if_devide_out=if_devide_out, init_layer_scale=init_layer_scale, **ssm_cfg, **factory_kwargs) norm_cls = partial( nn.LayerNorm if not rms_norm else RMSNorm, eps=norm_epsilon, **factory_kwargs ) block = Block( d_model, mixer_cls, norm_cls=norm_cls, drop_path=drop_path, fused_add_norm=fused_add_norm, residual_in_fp32=residual_in_fp32, ) block.layer_idx = layer_idx return block
https://github.com/huggingface/transformers/blob/c28d04e9e252a1a099944e325685f14d242ecdcd/src/transformers/models/gpt2/modeling_gpt2.py#L454
def init_weights( module, n_layer, initializer_range=0.02, # Now only used for embedding layer. rescale_prenorm_residual=True, n_residuals_per_layer=1, # Change to 2 if we have MLP ): if isinstance(module, nn.Linear): if module.bias is not None: if not getattr(module.bias, "no_reinit", False): nn.init.zeros(module.bias) elif isinstance(module, nn.Embedding): nn.init.normal(module.weight, std=initializer_range)
if rescale_prenorm_residual: # Reinitialize selected weights subject to the OpenAI GPT-2 Paper Scheme: # > A modified initialization which accounts for the accumulation on the residual path with model depth. Scale # > the weights of residual layers at initialization by a factor of 1/√N where N is the # of residual layers. # > -- GPT-2 :: https://openai.com/blog/better-language-models/ # # Reference (Megatron-LM): https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/model/gpt_model.py for name, p in module.named_parameters(): if name in ["out_proj.weight", "fc2.weight"]: # Special Scaled Initialization --> There are 2 Layer Norms per Transformer Block # Following Pytorch init, except scale by 1/sqrt(2 * n_layer) # We need to reinit p since this code could be called multiple times # Having just p *= scale would repeatedly scale it down nn.init.kaiming_uniform_(p, a=math.sqrt(5)) with torch.no_grad(): p /= math.sqrt(n_residuals_per_layer * n_layer)def segm_init_weights(m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=0.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.Conv2d): # NOTE conv was left to pytorch default in my original init lecun_normal_(m.weight) if m.bias is not None: nn.init.zeros_(m.bias) elif isinstance(m, (nn.LayerNorm, nn.GroupNorm, nn.BatchNorm2d)): nn.init.zeros_(m.bias) nn.init.ones_(m.weight)
class VisionMamba(nn.Module): def init(self, img_size=224, patch_size=16, stride=16, depth=24, embed_dim=192, channels=3, num_classes=1000, ssm_cfg=None, drop_rate=0., drop_path_rate=0.1, norm_epsilon: float = 1e-5, rms_norm: bool = False, initializer_cfg=None, fused_add_norm=False, residual_in_fp32=False, device=None, dtype=None, ft_seq_len=None, pt_hw_seq_len=14, if_bidirectional=False, final_pool_type='none', if_abs_pos_embed=False, if_rope=False, if_rope_residual=False, flip_img_sequences_ratio=-1., if_bimamba=False, bimamba_type="none", if_cls_token=False, if_devide_out=False, init_layer_scale=None, use_double_cls_token=False, use_middle_cls_token=False, **kwargs): factory_kwargs = {"device": device, "dtype": dtype} # add factory_kwargs into kwargs kwargs.update(factory_kwargs) super().init() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.if_bidirectional = if_bidirectional self.final_pool_type = final_pool_type self.if_abs_pos_embed = if_abs_pos_embed self.if_rope = if_rope self.if_rope_residual = if_rope_residual self.flip_img_sequences_ratio = flip_img_sequences_ratio # self.if_cls_token = if_cls_token self.if_cls_token = False self.use_double_cls_token = use_double_cls_token self.use_middle_cls_token = use_middle_cls_token self.num_tokens = 1 if if_cls_token else 0
# pretrain parameters self.num_classes = num_classes self.d_model = self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models self.patch_embed = PatchEmbed( img_size=img_size, patch_size=patch_size, stride=stride, in_chans=channels, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches if if_cls_token: if use_double_cls_token: self.cls_token_head = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) self.cls_token_tail = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) self.num_tokens = 2 else: self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) # self.num_tokens = 1 if if_abs_pos_embed: # self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, self.embed_dim)) self.pos_embed_x = nn.Parameter(torch.zeros(1, 256, self.embed_dim)) self.pos_embed_z = nn.Parameter(torch.zeros(1, 64, self.embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) if if_rope: half_head_dim = embed_dim // 2 hw_seq_len = img_size // patch_size self.rope = VisionRotaryEmbeddingFast( dim=half_head_dim, pt_seq_len=pt_hw_seq_len, ft_seq_len=hw_seq_len ) # self.head = nn.Linear(self.num_features, num_classes) if num_classes > 0 else nn.Identity() # settings = ws_settings.Settings() # config_module = importlib.import_module("lib.config.%s.config" % settings.script_name) # cfg = config_module.cfg # self.head = build_box_head(cfg, embed_dim) # TODO: release this comment dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule # import ipdb;ipdb.set_trace() inter_dpr = [0.0] + dpr self.drop_path = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity() # transformer blocks self.layers = nn.ModuleList( [ create_block( embed_dim, ssm_cfg=ssm_cfg, norm_epsilon=norm_epsilon, rms_norm=rms_norm, residual_in_fp32=residual_in_fp32, fused_add_norm=fused_add_norm, layer_idx=i, if_bimamba=if_bimamba, bimamba_type=bimamba_type, drop_path=inter_dpr[i], if_devide_out=if_devide_out, init_layer_scale=init_layer_scale, **factory_kwargs, ) for i in range(depth) ] ) # output head self.norm_f = (nn.LayerNorm if not rms_norm else RMSNorm)( embed_dim, eps=norm_epsilon, **factory_kwargs ) # self.pre_logits = nn.Identity() # original init self.patch_embed.apply(segm_init_weights) # self.head.apply(segm_init_weights) if if_abs_pos_embed: trunc_normal_(self.pos_embed_x, std=.02) trunc_normal_(self.pos_embed_z, std=.02) if if_cls_token: if use_double_cls_token: trunc_normal_(self.cls_token_head, std=.02) trunc_normal_(self.cls_token_tail, std=.02) else: trunc_normal_(self.cls_token, std=.02) # mamba init self.apply( partial( _init_weights, n_layer=depth, **(initializer_cfg if initializer_cfg is not None else {}), ) ) def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return { i: layer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs) for i, layer in enumerate(self.layers) } @torch.jit.ignore def no_weight_decay(self): return {"pos_embed", "cls_token", "dist_token", "cls_token_head", "cls_token_tail"} @torch.jit.ignore() def load_pretrained(self, checkpoint_path, prefix=""): _load_weights(self, checkpoint_path, prefix) def forward_features(self, z, x, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False): # taken from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py # with slight modifications to add the dist_token x = self.patch_embed(x) #x.shape = torch.Size([B, 3, 256, 256]) -> torch.Size([2, 256, 384]) z = self.patch_embed(z) #z.shape = torch.Size([B, 3, 128, 128]) -> torch.Size([2, 64, 384]) B, M, _ = x.shape if self.if_cls_token: # False if self.use_double_cls_token: cls_token_head = self.cls_token_head.expand(B, -1, -1) cls_token_tail = self.cls_token_tail.expand(B, -1, -1) token_position = [0, M + 1] x = torch.cat((cls_token_head, x, cls_token_tail), dim=1) M = x.shape[1] else: if self.use_middle_cls_token: cls_token = self.cls_token.expand(B, -1, -1) token_position = M // 2 # add cls token in the middle x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1) elif if_random_cls_token_position: cls_token = self.cls_token.expand(B, -1, -1) token_position = random.randint(0, M) x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1) print("token_position: ", token_position) else: cls_token = self.cls_token.expand(B, -1, -1) # stole cls_tokens impl from Phil Wang, thanks token_position = 0 x = torch.cat((cls_token, x), dim=1) M = x.shape[1] if self.if_abs_pos_embed: # True x = x + self.pos_embed_x # x = x + positon_embemding =torch.Size([B, 256, 384]) + torch.Size([1, 256, 384]) = torch.Size([B, 256, 384]) z = z + self.pos_embed_z # z = z + positon_embemding =torch.Size([B, 64, 384]) + torch.Size([1, 64, 384]) = torch.Size([B, 64, 384]) x = torch.cat((z, x), dim=1) # torch.Size([B, 320, 384]) x = self.pos_drop(x) # x.shape = torch.Size([B, 320, 384]) if if_random_token_rank: #False # 生成随机 shuffle 索引 shuffle_indices = torch.randperm(M) if isinstance(token_position, list): print("original value: ", x[0, token_position[0], 0], x[0, token_position[1], 0]) else: print("original value: ", x[0, token_position, 0]) print("original token_position: ", token_position) # 执行 shuffle x = x[:, shuffle_indices, :] if isinstance(token_position, list): # 找到 cls token 在 shuffle 之后的新位置 new_token_position = [torch.where(shuffle_indices == token_position[i])[0].item() for i in range(len(token_position))] token_position = new_token_position else: # 找到 cls token 在 shuffle 之后的新位置 token_position = torch.where(shuffle_indices == token_position)[0].item() if isinstance(token_position, list): print("new value: ", x[0, token_position[0], 0], x[0, token_position[1], 0]) else: print("new value: ", x[0, token_position, 0]) print("new token_position: ", token_position) if_flip_img_sequences = False if self.flip_img_sequences_ratio > 0 and (self.flip_img_sequences_ratio - random.random()) > 1e-5: # False x = x.flip([1]) if_flip_img_sequences = True # mamba impl residual = None hidden_states = x if not self.if_bidirectional: # True for layer in self.layers: if if_flip_img_sequences and self.if_rope: # False hidden_states = hidden_states.flip([1]) if residual is not None: residual = residual.flip([1]) # rope about if self.if_rope: # False hidden_states = self.rope(hidden_states) if residual is not None and self.if_rope_residual: residual = self.rope(residual) if if_flip_img_sequences and self.if_rope: # False hidden_states = hidden_states.flip([1]) if residual is not None: residual = residual.flip([1]) hidden_states, residual = layer( hidden_states, residual, inference_params=inference_params ) else: # False # get two layers in a single for-loop for i in range(len(self.layers) // 2): if self.if_rope: hidden_states = self.rope(hidden_states) if residual is not None and self.if_rope_residual: residual = self.rope(residual) hidden_states_f, residual_f = self.layers[i * 2]( hidden_states, residual, inference_params=inference_params ) hidden_states_b, residual_b = self.layers[i * 2 + 1]( hidden_states.flip([1]), None if residual == None else residual.flip([1]), inference_params=inference_params ) hidden_states = hidden_states_f + hidden_states_b.flip([1]) residual = residual_f + residual_b.flip([1]) if not self.fused_add_norm: #False if residual is None: residual = hidden_states else: residual = residual + self.drop_path(hidden_states) hidden_states = self.norm_f(residual.to(dtype=self.norm_f.weight.dtype)) else: #True # Set prenorm=False here since we don't need the residual fused_add_norm_fn = rms_norm_fn if isinstance(self.norm_f, RMSNorm) else layer_norm_fn hidden_states = fused_add_norm_fn( # hidden_states.shape = torch.Size([B, 320, 384]) self.drop_path(hidden_states), self.norm_f.weight, self.norm_f.bias, eps=self.norm_f.eps, residual=residual, prenorm=False, residual_in_fp32=self.residual_in_fp32, ) # return only cls token if it exists if self.if_cls_token: #False if self.use_double_cls_token: return (hidden_states[:, token_position[0], :] + hidden_states[:, token_position[1], :]) / 2 else: if self.use_middle_cls_token: return hidden_states[:, token_position, :] elif if_random_cls_token_position: return hidden_states[:, token_position, :] else: return hidden_states[:, token_position, :] if self.final_pool_type == 'none': return hidden_states[:, -1, :] elif self.final_pool_type == 'mean': #True return hidden_states.mean(dim=1) #hidden_states.shape = torch.Size([2, 384]) elif self.final_pool_type == 'max': return hidden_states elif self.final_pool_type == 'all': return hidden_states else: raise NotImplementedError def forward(self, x, return_features=False, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False): x = self.forward_features(x, inference_params, if_random_cls_token_position=if_random_cls_token_position, if_random_token_rank=if_random_token_rank) if return_features: return x x = self.head(x) if self.final_pool_type == 'max': x = x.max(dim=1)[0] return x@register_model def vim_tiny_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model
@register_model def vim_tiny_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model
@register_model def vim_small_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='all', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg()
if pretrained: # checkpoint = torch.hub.load_state_dict_from_url( # url="to.do", # map_location="cpu", check_hash=True # ) # model.load_state_dict(checkpoint["model"]) checkpoint = torch.load(pretrained, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False) print('Load pretrained model from: ' + pretrained) return model@register_model def vim_small_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model
中,或者# Copyright (c) 2023, Tri Dao, Albert Gu.
import math from typing import Optional
import torch import torch.nn as nn import torch.nn.functional as F from torch import Tensor
from einops import rearrange, repeat
try: from causal_conv1d import causal_conv1d_fn, causal_conv1d_update except ImportError: causal_conv1d_fn, causal_conv1d_update = None
try: from mamba_ssm.ops.selective_scan_interface import selective_scan_fn, mamba_inner_fn, bimamba_inner_fn, mamba_inner_fn_no_out_proj except ImportError: selective_scan_fn, mamba_inner_fn, bimamba_inner_fn, mamba_inner_fn_no_out_proj = None, None, None, None, None
try: from mamba_ssm.ops.triton.selective_state_update import selective_state_update except ImportError: selective_state_update = None
try: from mamba_ssm.ops.triton.layernorm import RMSNorm, layer_norm_fn, rms_norm_fn except ImportError: RMSNorm, layer_norm_fn, rms_norm_fn = None, None, None
d_state又称N,是隐状态的维度。d_model代表数据在经过SSM块之前和之后的数据维度,是输入x的维度
class Mamba(nn.Module): def init( self, d_model, d_state=16, d_conv=4, expand=2, dt_rank="auto", dt_min=0.001, dt_max=0.1, dt_init="random", dt_scale=1.0, dt_init_floor=1e-4, conv_bias=True, bias=False, use_fast_path=True, # Fused kernel options layer_idx=None, device=None, dtype=None, bimamba_type="none", if_devide_out=False, init_layer_scale=None, ): factory_kwargs = {"device": device, "dtype": dtype} super().init() self.d_model = d_model self.d_state = d_state self.d_conv = d_conv self.expand = expand self.d_inner = int(self.expand * self.d_model) self.dt_rank = math.ceil(self.d_model / 16) if dt_rank == "auto" else dt_rank self.use_fast_path = use_fast_path self.layer_idx = layer_idx self.bimamba_type = bimamba_type self.if_devide_out = if_devide_out
self.init_layer_scale = init_layer_scale if init_layer_scale is not None: self.gamma = nn.Parameter(init_layer_scale * torch.ones((d_model)), requires_grad=True) self.in_proj = nn.Linear(self.d_model, self.d_inner * 2, bias=bias, **factory_kwargs) self.conv1d = nn.Conv1d( in_channels=self.d_inner, out_channels=self.d_inner, bias=conv_bias, kernel_size=d_conv, groups=self.d_inner, padding=d_conv - 1, **factory_kwargs, ) self.activation = "silu" self.act = nn.SiLU() self.x_proj = nn.Linear( self.d_inner, self.dt_rank + self.d_state * 2, bias=False, **factory_kwargs ) self.dt_proj = nn.Linear(self.dt_rank, self.d_inner, bias=True, **factory_kwargs) # Initialize special dt projection to preserve variance at initialization dt_init_std = self.dt_rank**-0.5 * dt_scale if dt_init == "constant": nn.init.constant_(self.dt_proj.weight, dt_init_std) elif dt_init == "random": nn.init.uniform_(self.dt_proj.weight, -dt_init_std, dt_init_std) else: raise NotImplementedError # Initialize dt bias so that F.softplus(dt_bias) is between dt_min and dt_max dt = torch.exp( torch.rand(self.d_inner, **factory_kwargs) * (math.log(dt_max) - math.log(dt_min)) + math.log(dt_min) ).clamp(min=dt_init_floor) # Inverse of softplus: https://github.com/pytorch/pytorch/issues/72759 inv_dt = dt + torch.log(-torch.expm1(-dt)) with torch.no_grad(): self.dt_proj.bias.copy_(inv_dt) # Our initialization would set all Linear.bias to zero, need to mark this one as _no_reinit self.dt_proj.bias._no_reinit = True # S4D real initialization A = repeat( torch.arange(1, self.d_state + 1, dtype=torch.float32, device=device), "n -> d n", d=self.d_inner, ).contiguous() A_log = torch.log(A) # Keep A_log in fp32 self.A_log = nn.Parameter(A_log) self.A_log._no_weight_decay = True # D "skip" parameter self.D = nn.Parameter(torch.ones(self.d_inner, device=device)) # Keep in fp32 self.D._no_weight_decay = True # bidirectional if bimamba_type == "v1": A_b = repeat( torch.arange(1, self.d_state + 1, dtype=torch.float32, device=device), "n -> d n", d=self.d_inner, ).contiguous() A_b_log = torch.log(A_b) # Keep A_b_log in fp32 self.A_b_log = nn.Parameter(A_b_log) self.A_b_log._no_weight_decay = True elif bimamba_type == "v2": A_b = repeat( torch.arange(1, self.d_state + 1, dtype=torch.float32, device=device), "n -> d n", d=self.d_inner, ).contiguous() A_b_log = torch.log(A_b) # Keep A_b_log in fp32 self.A_b_log = nn.Parameter(A_b_log) self.A_b_log._no_weight_decay = True self.conv1d_b = nn.Conv1d( in_channels=self.d_inner, out_channels=self.d_inner, bias=conv_bias, kernel_size=d_conv, groups=self.d_inner, padding=d_conv - 1, **factory_kwargs, ) self.x_proj_b = nn.Linear( self.d_inner, self.dt_rank + self.d_state * 2, bias=False, **factory_kwargs ) self.dt_proj_b = nn.Linear(self.dt_rank, self.d_inner, bias=True, **factory_kwargs) self.D_b = nn.Parameter(torch.ones(self.d_inner, device=device)) # Keep in fp32 self.D_b._no_weight_decay = True self.out_proj = nn.Linear(self.d_inner, self.d_model, bias=bias, **factory_kwargs) def forward(self, hidden_states, inference_params=None): """ hidden_states: (B, L, D) Returns: same shape as hidden_states """ batch, seqlen, dim = hidden_states.shape conv_state, ssm_state = None, None if inference_params is not None: conv_state, ssm_state = self._get_states_from_cache(inference_params, batch) if inference_params.seqlen_offset > 0: # The states are updated inplace out, _, _ = self.step(hidden_states, conv_state, ssm_state) return out # We do matmul and transpose BLH -> HBL at the same time xz = rearrange( self.in_proj.weight @ rearrange(hidden_states, "b l d -> d (b l)"), "d (b l) -> b d l", l=seqlen, ) if self.in_proj.bias is not None: xz = xz + rearrange(self.in_proj.bias.to(dtype=xz.dtype), "d -> d 1") A = -torch.exp(self.A_log.float()) # (d_inner, d_state) # In the backward pass we write dx and dz next to each other to avoid torch.cat if self.use_fast_path and inference_params is None: # Doesn't support outputting the states if self.bimamba_type == "v1": A_b = -torch.exp(self.A_b_log.float()) out = bimamba_inner_fn( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, self.out_proj.weight, self.out_proj.bias, A, A_b, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) elif self.bimamba_type == "v2": A_b = -torch.exp(self.A_b_log.float()) out = mamba_inner_fn_no_out_proj( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, A, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) out_b = mamba_inner_fn_no_out_proj( xz.flip([-1]), self.conv1d_b.weight, self.conv1d_b.bias, self.x_proj_b.weight, self.dt_proj_b.weight, A_b, None, None, self.D_b.float(), delta_bias=self.dt_proj_b.bias.float(), delta_softplus=True, ) # F.linear(rearrange(out_z, "b d l -> b l d"), out_proj_weight, out_proj_bias) if not self.if_devide_out: out = F.linear(rearrange(out + out_b.flip([-1]), "b d l -> b l d"), self.out_proj.weight, self.out_proj.bias) else: out = F.linear(rearrange(out + out_b.flip([-1]), "b d l -> b l d") / 2, self.out_proj.weight, self.out_proj.bias) else: out = mamba_inner_fn( xz, self.conv1d.weight, self.conv1d.bias, self.x_proj.weight, self.dt_proj.weight, self.out_proj.weight, self.out_proj.bias, A, None, # input-dependent B None, # input-dependent C self.D.float(), delta_bias=self.dt_proj.bias.float(), delta_softplus=True, ) else: x, z = xz.chunk(2, dim=1) # Compute short convolution if conv_state is not None: # If we just take x[:, :, -self.d_conv :], it will error if seqlen < self.d_conv # Instead F.pad will pad with zeros if seqlen < self.d_conv, and truncate otherwise. conv_state.copy_(F.pad(x, (self.d_conv - x.shape[-1], 0))) # Update state (B D W) if causal_conv1d_fn is None: x = self.act(self.conv1d(x)[..., :seqlen]) else: assert self.activation in ["silu", "swish"] x = causal_conv1d_fn( x=x, weight=rearrange(self.conv1d.weight, "d 1 w -> d w"), bias=self.conv1d.bias, activation=self.activation, ) # We're careful here about the layout, to avoid extra transposes. # We want dt to have d as the slowest moving dimension # and L as the fastest moving dimension, since those are what the ssm_scan kernel expects. x_dbl = self.x_proj(rearrange(x, "b d l -> (b l) d")) # (bl d) dt, B, C = torch.split(x_dbl, [self.dt_rank, self.d_state, self.d_state], dim=-1) dt = self.dt_proj.weight @ dt.t() dt = rearrange(dt, "d (b l) -> b d l", l=seqlen) B = rearrange(B, "(b l) dstate -> b dstate l", l=seqlen).contiguous() C = rearrange(C, "(b l) dstate -> b dstate l", l=seqlen).contiguous() assert self.activation in ["silu", "swish"] y = selective_scan_fn( x, dt, A, B, C, self.D.float(), z=z, delta_bias=self.dt_proj.bias.float(), delta_softplus=True, return_last_state=ssm_state is not None, ) if ssm_state is not None: y, last_state = y ssm_state.copy_(last_state) y = rearrange(y, "b d l -> b l d") out = self.out_proj(y) if self.init_layer_scale is not None: out = out * self.gamma return out def step(self, hidden_states, conv_state, ssm_state): dtype = hidden_states.dtype assert hidden_states.shape[1] == 1, "Only support decoding with 1 token at a time for now" xz = self.in_proj(hidden_states.squeeze(1)) # (B 2D) x, z = xz.chunk(2, dim=-1) # (B D) # Conv step if causal_conv1d_update is None: conv_state.copy_(torch.roll(conv_state, shifts=-1, dims=-1)) # Update state (B D W) conv_state[:, :, -1] = x x = torch.sum(conv_state * rearrange(self.conv1d.weight, "d 1 w -> d w"), dim=-1) # (B D) if self.conv1d.bias is not None: x = x + self.conv1d.bias x = self.act(x).to(dtype=dtype) else: x = causal_conv1d_update( x, conv_state, rearrange(self.conv1d.weight, "d 1 w -> d w"), self.conv1d.bias, self.activation, ) x_db = self.x_proj(x) # (B dt_rank+2*d_state) dt, B, C = torch.split(x_db, [self.dt_rank, self.d_state, self.d_state], dim=-1) # Don't add dt_bias here dt = F.linear(dt, self.dt_proj.weight) # (B d_inner) A = -torch.exp(self.A_log.float()) # (d_inner, d_state) # SSM step if selective_state_update is None: # Discretize A and B dt = F.softplus(dt + self.dt_proj.bias.to(dtype=dt.dtype)) dA = torch.exp(torch.einsum("bd,dn->bdn", dt, A)) dB = torch.einsum("bd,bn->bdn", dt, B) ssm_state.copy_(ssm_state * dA + rearrange(x, "b d -> b d 1") * dB) y = torch.einsum("bdn,bn->bd", ssm_state.to(dtype), C) y = y + self.D.to(dtype) * x y = y * self.act(z) # (B D) else: y = selective_state_update( ssm_state, x, dt, A, B, C, self.D, z=z, dt_bias=self.dt_proj.bias, dt_softplus=True ) out = self.out_proj(y) return out.unsqueeze(1), conv_state, ssm_state def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): device = self.out_proj.weight.device conv_dtype = self.conv1d.weight.dtype if dtype is None else dtype conv_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_conv, device=device, dtype=conv_dtype ) ssm_dtype = self.dt_proj.weight.dtype if dtype is None else dtype # ssm_dtype = torch.float32 ssm_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_state, device=device, dtype=ssm_dtype ) return conv_state, ssm_state def _get_states_from_cache(self, inference_params, batch_size, initialize_states=False): assert self.layer_idx is not None if self.layer_idx not in inference_params.key_value_memory_dict: batch_shape = (batch_size,) conv_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_conv, device=self.conv1d.weight.device, dtype=self.conv1d.weight.dtype, ) ssm_state = torch.zeros( batch_size, self.d_model * self.expand, self.d_state, device=self.dt_proj.weight.device, dtype=self.dt_proj.weight.dtype, # dtype=torch.float32, ) inference_params.key_value_memory_dict[self.layer_idx] = (conv_state, ssm_state) else: conv_state, ssm_state = inference_params.key_value_memory_dict[self.layer_idx] # TODO: What if batch size changes between generation, and we reuse the same states? if initialize_states: conv_state.zero_() ssm_state.zero_() return conv_state, ssm_stateclass Block(nn.Module): def init( self, dim, mixer_cls, norm_cls=nn.LayerNorm, fused_add_norm=False, residual_in_fp32=False ): """ Simple block wrapping a mixer class with LayerNorm/RMSNorm and residual connection"
This Block has a slightly different structure compared to a regular prenorm Transformer block. The standard block is: LN -> MHA/MLP -> Add. [Ref: https://arxiv.org/abs/2002.04745] Here we have: Add -> LN -> Mixer, returning both the hidden_states (output of the mixer) and the residual. This is purely for performance reasons, as we can fuse add and LayerNorm. The residual needs to be provided (except for the very first block). """ super().__init__() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.mixer = mixer_cls(dim) self.norm = norm_cls(dim) if self.fused_add_norm: assert RMSNorm is not None, "RMSNorm import fails" assert isinstance( self.norm, (nn.LayerNorm, RMSNorm) ), "Only LayerNorm and RMSNorm are supported for fused_add_norm" def forward( self, hidden_states: Tensor, residual: Optional[Tensor] = None, inference_params=None ): r"""Pass the input through the encoder layer. Args: hidden_states: the sequence to the encoder layer (required). residual: hidden_states = Mixer(LN(residual)) """ if not self.fused_add_norm: residual = (hidden_states + residual) if residual is not None else hidden_states hidden_states = self.norm(residual.to(dtype=self.norm.weight.dtype)) if self.residual_in_fp32: residual = residual.to(torch.float32) else: fused_add_norm_fn = rms_norm_fn if isinstance(self.norm, RMSNorm) else layer_norm_fn hidden_states, residual = fused_add_norm_fn( hidden_states, self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) hidden_states = self.mixer(hidden_states, inference_params=inference_params) return hidden_states, residual def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return self.mixer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)中,是否可以集成上文中模块?
- Assistant
将上文中的模块集成到您提供的两个代码片段中需要考虑它们的核心功能和结构,并在适当的地方进行替换或集成。以下是一些可能的集成点和方法:
1. 集成
MambaMixer和CaMambaBlock替换原始的
Mamba模块- 在这两个代码片段中,
Mamba模块是核心的状态空间模型实现。如果希望利用上文中实现的MambaMixer和CaMambaBlock,可以替换原来的Mamba定义和相应的数据流。
替换或修改
Block实现- 在提供的代码片段(尤其是第二个),
Block是一个混合模块,可以将其重构为使用MambaMixer和CaMambaBlock,而不是原始的Mamba类。
2. 集成
VisionMamba结构VisionMamba是一个专为视觉任务设计的模块,它使用了图像嵌入(PatchEmbed)和深度块(Block)。- 如果目标是集成对图像数据的处理,可以利用
VisionMamba的图像处理能力,并让其使用CaMambaBlock作为其基础构建模块。
3. 性能优化方面
- 确保在CUDA或其他高性能路径上可用的操作能被正确集成到代码中。例如,
MambaMixer中提到的CUDA快路径,可以用于将执行速度最大化,用于推理过程中。 - 当整合标准化层(如
RMSNorm)时,验证它们的设备兼容性以免导致性能下降或设备不兼容。
4. 拓展功能与选项
- 在模型中可能需要支持更多功能,例如
双向处理、卷积层、初始化方法等,这些在提供的代码片段和上文模块中都有所不同。 - 可以在不同的层级集成不同的功能,来实现更复杂的任务,比如在图像任务中结合卷积和状态空间模型以获得更好的特征捕捉。
5. 注意接口和参数的调整
- 确保在替换模块后,检查接口定义和参数以保证兼容性,这涉及到如输入输出的维度匹配、设备管理(如
device、dtype的传递)以及初始化。
综合考虑:
整合新的模块会涉及到对网络结构的调整与优化。尤其是在不同任务(如自然语言处理 vs. 图像处理)中,可能需要对模型进行额外的参数设置与功能集成。建议在实现中逐步替换和测试,以确保新模型能够稳定运行并实现预期的性能目标。
- 在这两个代码片段中,
- User
上文中的模块是即插即用的,现在需要分析如何集成或替换到当前模型中,并给出原因
- Assistant
将上文中的模块(如
MambaMixer和CaMambaBlock)集成或替换到当前的模型中,核心是理解这些模块在现有模型中的位置和作用,然后决定如何替换或嵌入。以下是一些具体步骤和原因:1. 模块替代与功能定位
-
MambaMixer 替换现有的 Mixer 类: 在提供的代码中,有许多地方使用了
mixer_cls参数,如Block类中的self.mixer = mixer_cls(dim)。可以将这个mixer_cls替换为MambaMixer,这样每个块就会使用新的选择性状态空间机制。原因:
MambaMixer提供了对输入敏感的状态空间表达能力,可能在需要捕捉复杂动态特征的任务中具有优势。 -
CaMambaBlock 替换现有的 Block 实现: 然后可以将
CaMambaBlock替代或组合在现有的Block实现中,使用其特有的特征,如独特的正则化方式和残差接入点。原因:
CaMambaBlock中的规范化和残差连接可能有助于训练过程的稳定性和加速收敛。
2. 适配模型结构
-
整合 VisionMamba 架构: 如果目标模型主要用于视觉任务,可以结合
VisionMamba的结构,将其作为主框架,并在其各个块中使用CaMambaBlock和MambaMixer。原因:
VisionMamba源于视觉任务设计,它原生支持图像分块和位置编码,有助于提高视觉特征提取的效率。
3. 性能优化与控制
-
使用 CUDA 快路径: 在需要高效推理的情况下,确保
MambaMixer的 CUDA 快路径得以实现。这个过程可能涉及到条件判断以及确保依赖库的正确安装和配置。原因:在特定硬件上,CUDA 的利用能显著提高推理速度,有益于处理大规模数据。
4. 接口一致性与兼容
-
匹配输入输出接口: 确保替换模块的输入输出接口与现有的模型架构兼容。在替换
Block或者Mixer的实现时,特别要注意其接受的输入形状、类型和输出。原因:维护模型结构的完整性和可运行性需要确保接口一致性,即使在模块替换后也不会影响整个管道。
-
传递设备和数据类型配置: 确保新集成的模块可以接收和处理如
device和dtype之类的配置项,保持模型运行的灵活性。原因:这些参数对于确保模型在不同设备上的正确运行和最优性能至关重要。
5. 逐步集成与验证
-
模块化替换: 在集成上述模块时,建议逐步替换和验证,以便快速定位可能出现的问题,并及时调整。
原因:逐步替换有助于捕捉具体修改后的表现,便于调试和实现预期目标。
通过以上步骤,你可以系统地将
MambaMixer和CaMambaBlock等模块集成到现有的模型中,在确保保持现有功能和性能的同时,利用新模块的优点提升整体表现。 -
- User
那么CaMambaBlock 替换Block的代码
- Assistant
要将
CaMambaBlock替换为Block,需要确保接口和功能的一致性。以下是CaMambaBlock替换Block的示例代码:假设您有如下代码的
Block类:class Block(nn.Module): def __init__(self, dim, mixer_cls, norm_cls=nn.LayerNorm, fused_add_norm=False, residual_in_fp32=False, drop_path=0.): super().__init__() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.mixer = mixer_cls(dim) self.norm = norm_cls(dim) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() if self.fused_add_norm: assert RMSNorm is not None, "RMSNorm import fails" assert isinstance( self.norm, (nn.LayerNorm, RMSNorm) ), "Only LayerNorm and RMSNorm are supported for fused_add_norm" def forward(self, hidden_states: Tensor, residual: Optional[Tensor] = None, inference_params=None): if not self.fused_add_norm: if residual is None: residual = hidden_states else: residual = residual + self.drop_path(hidden_states) hidden_states = self.norm(residual.to(dtype=self.norm.weight.dtype)) if self.residual_in_fp32: residual = residual.to(torch.float32) else: fused_add_norm_fn = rms_norm_fn if isinstance(self.norm, RMSNorm) else layer_norm_fn hidden_states, residual = fused_add_norm_fn( self.drop_path(hidden_states), self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) hidden_states = self.mixer(hidden_states, inference_params=inference_params) return hidden_states, residual def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return self.mixer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)替换为
CaMambaBlock后:class CaMambaBlock(nn.Module): def __init__(self, config, layer_idx, head_num=1, length=49, craft=False): super().__init__() self.config = config self.layer_idx = layer_idx self.residual_in_fp32 = config.residual_in_fp32 self.norm = MambaRMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) self.head_num = head_num self.length = length self.craft = craft self.config.intermediate_size = config.intermediate_size self.mixer = MambaMixer(config, layer_idx=layer_idx) def forward(self, hidden_states, hidden_states_2=None, cache_params: Optional[MambaCache] = None, cache_params_2: Optional[MambaCache] = None): residual = hidden_states hidden_states = self.norm(hidden_states.to(dtype=self.norm.weight.dtype)) if self.residual_in_fp32: residual = residual.to(torch.float32) hidden_states = self.mixer(hidden_states, hidden_states_2, cache_params=cache_params, cache_params_2=cache_params_2) hidden_states = residual + hidden_states return hidden_states替换步骤和注意事项:
-
初始化参数对齐:
- 确保在初始化
CaMambaBlock时,config参数能正确设置。如果原来的Block用到dim,你需要相应在config中反映出来。
- 确保在初始化
-
功能替换:
forward方法参数保持一致,特别是hidden_states和residual的处理要保证逻辑一致。- 您可能需要考虑
inference_params的兼容性,虽然在CaMambaBlock中不使用这个参数(除非用于不同用途)。
-
模型配置和其他依赖:
CaMambaBlock依赖于MambaRMSNorm和MambaMixer,需确保模型中定义这些类或从外部引入。- 适配到现有工程时,应验证是否需要整合或调整其他块的配置,比如是否使用双向机制。
-
测试和验证:
- 在替换实施后,务必通过单元测试或模型整体验证,以确保新模块的集成不影响整个模型的准确性和稳定性。
通过这些步骤,您应该能够实现
CaMambaBlock在您现有模型中的成功集成。 -
- User
这两个block在功能上有何不同
- Assistant
Block类和CaMambaBlock类在功能上有几个关键的不同之处,这些差异主要体现在模块的设计目标、实现细节以及具体的操作中。以下是对这两者在功能上的一些差异的总结:1. 目的与设计背景
-
Block:
- 通常是基于标准的 Transformer 或具有标准层结构的模块,旨在提供一种通用的、可配置的块复用框架。
Block类更像一个通用混合模块,包含标准化(如 LayerNorm 或 RMSNorm)和预激活 MLP 或其他混合机制(如用于捕捉输入特征的线性变换)。
-
CaMambaBlock:
- 专为实现
Mamba模型的选择性状态空间模型的目标而设计。 - 旨在捕捉序列输入的复杂时空依赖关系,使用
MambaMixer实现这种选择特性。
- 专为实现
2. 特定功能和实现机制
-
Block:
- 采用标准层的正则化,如
LayerNorm或RMSNorm。 - 提供一个混合类
mixer_cls,可以实现各种不同的混合策略。 - 融合了一些额外的优化选项(如
fused_add_norm)来提高性能。
- 采用标准层的正则化,如
-
CaMambaBlock:
- 实现了状态特定的规范化
MambaRMSNorm,这可能在捕捉复杂输入模式时更有效。 - 紧密结合了
MambaMixer模块,旨在实现选择性、输入依赖的状态空间操作。 - 提供了更具体的配置选项和规范,以支持选择性与混合机制的结合。
- 实现了状态特定的规范化
3. 输入/输出与处理流程
-
Block:
- 通常处理传统的残差连接和标准化,与标准 Transformer 结构类似。
- 依赖于外部传递的 residual 和 hidden_states 来计算模型的输出。
-
CaMambaBlock:
- 内部集成了残差处理,其
forward方法直接应用规范化后进行选择性状态空间混合。 - 更加注重在混合后恢复输入的残差结构。
- 内部集成了残差处理,其
4. 灵活性与扩展性
-
Block:
- 配置较为灵活,可以根据需要替换或扩展为不同类型的混合模块。
- 适合与多种深度学习构建块组合。
-
CaMambaBlock:
- 更专注于选择性状态空间,设计更为专一。
- 适合于通过状态空间模型扩展序列模型的能力。
总结:
Block和CaMambaBlock在功能上的差异主要体现在是否专注于选择性状态空间机制的实现。CaMambaBlock更适合用于需要捕捉复杂序列关系的任务,而Block则提供了较通用的深度结构的实现。选择适合的模块取决于具体的任务需求及模型设计目标。 -