Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-08-06· OpenAIAi2 WildChat-4.8M (AI2)

分析import torch import torch.nn as nn from functools import partial from torch import…

  1. User

    分析import torch import torch.nn as nn from functools import partial from torch import Tensor from typing import Optional

    from timm.models.vision_transformer import VisionTransformer, cfg from timm.models.registry import register_model from timm.models.layers import trunc_normal, lecun_normal_

    from timm.models.layers import DropPath, to_2tuple from timm.models.vision_transformer import _load_weights

    import math

    from collections import namedtuple

    from mamba_ssm.modules.mamba_simple import Mamba from mamba_ssm.utils.generation import GenerationMixin from mamba_ssm.utils.hf import load_config_hf, load_state_dict_hf

    from lib.models.mamba_fetrack.rope import * import random from lib.models.layers.head import build_box_head import importlib import lib.train.admin.settings as ws_settings from .utils import combine_tokens, recover_tokens

    from .HinBlock import HinResBlock from .ResGLU import ResGLUAdapter, ModelArgs from .MSF import MSF try: from mamba_ssm.ops.triton.layernorm import RMSNorm, layer_norm_fn, rms_norm_fn except ImportError: RMSNorm, layer_norm_fn, rms_norm_fn = None, None, None

    all = [ 'vim_tiny_patch16_224', 'vim_small_patch16_224', 'vim_base_patch16_224', 'vim_tiny_patch16_384', 'vim_small_patch16_384', 'vim_base_patch16_384', ]

    class PatchEmbed(nn.Module): """ 2D Image to Patch Embedding """ def init(self, img_size=224, patch_size=16, stride=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True): super().init() img_size = to_2tuple(img_size) patch_size = to_2tuple(patch_size) self.img_size = img_size self.patch_size = patch_size self.grid_size = ((img_size[0] - patch_size[0]) // stride + 1, (img_size[1] - patch_size[1]) // stride + 1) self.num_patches = self.grid_size[0] * self.grid_size[1] self.flatten = flatten

        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=stride)
        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
    
    def forward(self, x):
        B, C, H, W = x.shape
        x = self.proj(x)
        if self.flatten:
            x = x.flatten(2).transpose(1, 2)  # BCHW -> BNC
        x = self.norm(x)
        return x
    

    class Block(nn.Module): def init( self, dim, mixer_cls, norm_cls=nn.LayerNorm, fused_add_norm=False, residual_in_fp32=False,drop_path=0., integrate_glu=False ): """ Simple block wrapping a mixer class with LayerNorm/RMSNorm and residual connection"

        This Block has a slightly different structure compared to a regular
        prenorm Transformer block.
        The standard block is: LN -> MHA/MLP -> Add.
        [Ref: https://arxiv.org/abs/2002.04745]
        Here we have: Add -> LN -> Mixer, returning both
        the hidden_states (output of the mixer) and the residual.
        This is purely for performance reasons, as we can fuse add and LayerNorm.
        The residual needs to be provided (except for the very first block).
        """
        super().__init__()
        self.residual_in_fp32 = residual_in_fp32
        self.fused_add_norm = fused_add_norm
        self.mixer = mixer_cls(dim)
        self.norm = norm_cls(dim)
        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
        if self.fused_add_norm:
            assert RMSNorm is not None, "RMSNorm import fails"
            assert isinstance(
                self.norm, (nn.LayerNorm, RMSNorm)
            ), "Only LayerNorm and RMSNorm are supported for fused_add_norm"
        # 引入门控机制
        model_args = ModelArgs()
        if integrate_glu:
            model_args = ModelArgs()
            self.glu_adapter = ResGLUAdapter(model_args)
        else:
            self.glu_adapter = None 
        # self.glu_adapter = ResGLUAdapter(model_args)
    
    def forward(
        self, hidden_states: Tensor, residual: Optional[Tensor] = None, inference_params=None
    ):
        r"""Pass the input through the encoder layer.
    
        Args:
            hidden_states: the sequence to the encoder layer (required).
            residual: hidden_states = Mixer(LN(residual))
        """
        if not self.fused_add_norm:
            if residual is None:
                residual = hidden_states
            else:
                residual = residual + self.drop_path(hidden_states)
            
            hidden_states = self.norm(residual.to(dtype=self.norm.weight.dtype))
            if self.residual_in_fp32:
                residual = residual.to(torch.float32)
        else:
            fused_add_norm_fn = rms_norm_fn if isinstance(self.norm, RMSNorm) else layer_norm_fn
            if residual is None:
                hidden_states, residual = fused_add_norm_fn(
                    hidden_states,
                    self.norm.weight,
                    self.norm.bias,
                    residual=residual,
                    prenorm=True,
                    residual_in_fp32=self.residual_in_fp32,
                    eps=self.norm.eps,
                )
            else:
                hidden_states, residual = fused_add_norm_fn(
                    self.drop_path(hidden_states),
                    self.norm.weight,
                    self.norm.bias,
                    residual=residual,
                    prenorm=True,
                    residual_in_fp32=self.residual_in_fp32,
                    eps=self.norm.eps,
                )    
        hidden_states = self.mixer(hidden_states, inference_params=inference_params)
        # 增加门控机制, 消除冗余和不相关的特征,保留最具表现力的输出
        if self.glu_adapter is not None:
            hidden_states = self.glu_adapter(hidden_states)
        # hidden_states = self.glu_adapter(hidden_states)
        return hidden_states, residual
    
    def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs):
        return self.mixer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)
    

    def create_block( d_model, ssm_cfg=None, norm_epsilon=1e-5, drop_path=0., rms_norm=False, residual_in_fp32=False, fused_add_norm=False, layer_idx=None, integrate_glu=False, # 新增参数 device=None, dtype=None, if_bimamba=False, bimamba_type="none", if_devide_out=False, init_layer_scale=None, ): if if_bimamba: bimamba_type = "v1" if ssm_cfg is None: ssm_cfg = {} factory_kwargs = {"device": device, "dtype": dtype} mixer_cls = partial(Mamba, layer_idx=layer_idx, bimamba_type=bimamba_type, if_devide_out=if_devide_out, init_layer_scale=init_layer_scale, **ssm_cfg, **factory_kwargs) norm_cls = partial( nn.LayerNorm if not rms_norm else RMSNorm, eps=norm_epsilon, **factory_kwargs ) block = Block( d_model, mixer_cls, norm_cls=norm_cls, drop_path=drop_path, fused_add_norm=fused_add_norm, residual_in_fp32=residual_in_fp32, integrate_glu=integrate_glu, # 传递参数 ) block.layer_idx = layer_idx return block

    https://github.com/huggingface/transformers/blob/c28d04e9e252a1a099944e325685f14d242ecdcd/src/transformers/models/gpt2/modeling_gpt2.py#L454

    def init_weights( module, n_layer, initializer_range=0.02, # Now only used for embedding layer. rescale_prenorm_residual=True, n_residuals_per_layer=1, # Change to 2 if we have MLP ): if isinstance(module, nn.Linear): if module.bias is not None: if not getattr(module.bias, "no_reinit", False): nn.init.zeros(module.bias) elif isinstance(module, nn.Embedding): nn.init.normal(module.weight, std=initializer_range)

    if rescale_prenorm_residual:
        # Reinitialize selected weights subject to the OpenAI GPT-2 Paper Scheme:
        #   > A modified initialization which accounts for the accumulation on the residual path with model depth. Scale
        #   > the weights of residual layers at initialization by a factor of 1/√N where N is the # of residual layers.
        #   >   -- GPT-2 :: https://openai.com/blog/better-language-models/
        #
        # Reference (Megatron-LM): https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/model/gpt_model.py
        for name, p in module.named_parameters():
            if name in ["out_proj.weight", "fc2.weight"]:
                # Special Scaled Initialization --> There are 2 Layer Norms per Transformer Block
                # Following Pytorch init, except scale by 1/sqrt(2 * n_layer)
                # We need to reinit p since this code could be called multiple times
                # Having just p *= scale would repeatedly scale it down
                nn.init.kaiming_uniform_(p, a=math.sqrt(5))
                with torch.no_grad():
                    p /= math.sqrt(n_residuals_per_layer * n_layer)
    

    def segm_init_weights(m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=0.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.Conv2d): # NOTE conv was left to pytorch default in my original init lecun_normal_(m.weight) if m.bias is not None: nn.init.zeros_(m.bias) elif isinstance(m, (nn.LayerNorm, nn.GroupNorm, nn.BatchNorm2d)): nn.init.zeros_(m.bias) nn.init.ones_(m.weight)

    class VisionMamba(nn.Module): def init(self, img_size=224, patch_size=16, stride=16, depth=24, embed_dim=192, channels=3, num_classes=1000, ssm_cfg=None, drop_rate=0., drop_path_rate=0.1, norm_epsilon: float = 1e-5, rms_norm: bool = False, initializer_cfg=None, fused_add_norm=False, residual_in_fp32=False, device=None, dtype=None, ft_seq_len=None, pt_hw_seq_len=14, if_bidirectional=False, final_pool_type='none', if_abs_pos_embed=False, if_rope=False, if_rope_residual=False, flip_img_sequences_ratio=-1., if_bimamba=False, bimamba_type="none", if_cls_token=False, if_devide_out=False, init_layer_scale=None, use_double_cls_token=False, use_middle_cls_token=False, msf_layers=None, # 传入在哪些层集成 MSF,例如:[0, 5, 10] 或者通过布尔表示 **kwargs): factory_kwargs = {"device": device, "dtype": dtype} # add factory_kwargs into kwargs kwargs.update(factory_kwargs) super().init() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.if_bidirectional = if_bidirectional self.final_pool_type = final_pool_type self.if_abs_pos_embed = if_abs_pos_embed self.if_rope = if_rope self.if_rope_residual = if_rope_residual self.flip_img_sequences_ratio = flip_img_sequences_ratio # self.if_cls_token = if_cls_token self.if_cls_token = False self.use_double_cls_token = use_double_cls_token self.use_middle_cls_token = use_middle_cls_token self.num_tokens = 1 if if_cls_token else 0 # self.msf_layers = set(msf_layers) if msf_layers is not None else set() self.MSF = MSF(embed_dim, embed_dim,embed_dim // 2) # # pretrain parameters self.num_classes = num_classes self.d_model = self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models base_filter=32 self.hin_block = nn.Sequential( nn.Conv2d(channels, base_filter, kernel_size=3, stride=1, padding=1), HinResBlock(base_filter, base_filter), # 第一个 HinResBlock HinResBlock(base_filter, base_filter), # 第二个 HinResBlock HinResBlock(base_filter, base_filter), # 第三个 HinResBlock nn.Conv2d(base_filter, channels, kernel_size=1) # 将通道数从32转换为3 ) ############################## self.patch_embed = PatchEmbed( img_size=img_size, patch_size=patch_size, stride=stride, in_chans=channels, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches

        if if_cls_token:
            if use_double_cls_token:
                self.cls_token_head = nn.Parameter(torch.zeros(1, 1, self.embed_dim))
                self.cls_token_tail = nn.Parameter(torch.zeros(1, 1, self.embed_dim))
                self.num_tokens = 2
            else:
                self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embed_dim))
                # self.num_tokens = 1
            
        if if_abs_pos_embed:
            # self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, self.embed_dim))
            self.pos_embed_x = nn.Parameter(torch.zeros(1, 256, self.embed_dim))
            self.pos_embed_z = nn.Parameter(torch.zeros(1, 64, self.embed_dim))
            self.pos_drop = nn.Dropout(p=drop_rate)
    
        if if_rope:
            half_head_dim = embed_dim // 2
            hw_seq_len = img_size // patch_size
            self.rope = VisionRotaryEmbeddingFast(
                dim=half_head_dim,
                pt_seq_len=pt_hw_seq_len,
                ft_seq_len=hw_seq_len
            )
            
        # TODO: release this comment
        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
        # import ipdb;ipdb.set_trace()
        inter_dpr = [0.0] + dpr
        self.drop_path = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity()
                # transformer blocks
        self.layers = nn.ModuleList(
            [
                create_block(
                    embed_dim,
                    ssm_cfg=ssm_cfg,
                    norm_epsilon=norm_epsilon,
                    rms_norm=rms_norm,
                    residual_in_fp32=residual_in_fp32,
                    fused_add_norm=fused_add_norm,
                    layer_idx=i,
                    integrate_glu=(i == 0),  # 仅在第一个块中集成 ResGLUAdapter
                    if_bimamba=if_bimamba,
                    bimamba_type=bimamba_type,
                    drop_path=inter_dpr[i],
                    if_devide_out=if_devide_out,
                    init_layer_scale=init_layer_scale,
                    **factory_kwargs,
                )
                for i in range(depth)
            ]
        )
        
        # output head
        self.norm_f = (nn.LayerNorm if not rms_norm else RMSNorm)(
            embed_dim, eps=norm_epsilon, **factory_kwargs
        )
    
        # self.pre_logits = nn.Identity()
    
        # original init
        self.patch_embed.apply(segm_init_weights)
        # self.head.apply(segm_init_weights)
        if if_abs_pos_embed:
            trunc_normal_(self.pos_embed_x, std=.02)
            trunc_normal_(self.pos_embed_z, std=.02)
            
        if if_cls_token:
            if use_double_cls_token:
                trunc_normal_(self.cls_token_head, std=.02)
                trunc_normal_(self.cls_token_tail, std=.02)
            else:
                trunc_normal_(self.cls_token, std=.02)
    
        # mamba init
        self.apply(
            partial(
                _init_weights,
                n_layer=depth,
                **(initializer_cfg if initializer_cfg is not None else {}),
            )
        )
    
    
    def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs):
        return {
            i: layer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)
            for i, layer in enumerate(self.layers)
        }
    
    @torch.jit.ignore
    def no_weight_decay(self):
        return {"pos_embed", "cls_token", "dist_token", "cls_token_head", "cls_token_tail"}
    
    @torch.jit.ignore()
    def load_pretrained(self, checkpoint_path, prefix=""):
        _load_weights(self, checkpoint_path, prefix)
    
    def forward_features(self, z, x, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False):
        x = self.hin_block(x)
        z = self.hin_block(z)
    
        x = self.patch_embed(x)                  #x.shape = torch.Size([B, 3, 256, 256])  -> torch.Size([2, 256, 384])
        z = self.patch_embed(z)                  #z.shape = torch.Size([B, 3, 128, 128])  -> torch.Size([2, 64, 384])
        B, M, _ = x.shape
       
        if self.if_cls_token:                 # False
            if self.use_double_cls_token:
                cls_token_head = self.cls_token_head.expand(B, -1, -1)
                cls_token_tail = self.cls_token_tail.expand(B, -1, -1)
                token_position = [0, M + 1]
                x = torch.cat((cls_token_head, x, cls_token_tail), dim=1)
                M = x.shape[1]
            else:
                if self.use_middle_cls_token:
                    cls_token = self.cls_token.expand(B, -1, -1)
                    token_position = M // 2
                    # add cls token in the middle
                    x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1)       
                elif if_random_cls_token_position:
                    cls_token = self.cls_token.expand(B, -1, -1)
                    token_position = random.randint(0, M)
                    x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1)
                    print("token_position: ", token_position)
                else:
                    cls_token = self.cls_token.expand(B, -1, -1)  # stole cls_tokens impl from Phil Wang, thanks
                    token_position = 0
                    x = torch.cat((cls_token, x), dim=1)
                M = x.shape[1]                 
       
        if self.if_abs_pos_embed:                  # True 
            x = x + self.pos_embed_x               # x = x + positon_embemding =torch.Size([B, 256, 384]) + torch.Size([1, 256, 384]) = torch.Size([B, 256, 384])
            z = z + self.pos_embed_z               # z = z + positon_embemding =torch.Size([B, 64, 384]) + torch.Size([1, 64, 384]) = torch.Size([B, 64, 384])
            x = torch.cat((z, x), dim=1)           # torch.Size([B, 320, 384])
            x = self.pos_drop(x)                   # x.shape = torch.Size([B, 320, 384])
            
        if if_random_token_rank:                   #False
            # 生成随机 shuffle 索引
            shuffle_indices = torch.randperm(M)
    
            if isinstance(token_position, list):
                print("original value: ", x[0, token_position[0], 0], x[0, token_position[1], 0])
            else:
                print("original value: ", x[0, token_position, 0])
            print("original token_position: ", token_position)
    
            # 执行 shuffle
            x = x[:, shuffle_indices, :]
    
            if isinstance(token_position, list):
                # 找到 cls token 在 shuffle 之后的新位置
                new_token_position = [torch.where(shuffle_indices == token_position[i])[0].item() for i in range(len(token_position))]
                token_position = new_token_position
            else:
                # 找到 cls token 在 shuffle 之后的新位置
                token_position = torch.where(shuffle_indices == token_position)[0].item()
    
            if isinstance(token_position, list):
                print("new value: ", x[0, token_position[0], 0], x[0, token_position[1], 0])
            else:
                print("new value: ", x[0, token_position, 0])
            print("new token_position: ", token_position)
    
    
        if_flip_img_sequences = False
        if self.flip_img_sequences_ratio > 0 and (self.flip_img_sequences_ratio - random.random()) > 1e-5:        # False
            x = x.flip([1])
            if_flip_img_sequences = True
    
        # mamba impl
        residual = None
        hidden_states = x
        # 设置具体集成的层数
        msf_layers = {23}
        if not self.if_bidirectional:                                 # True
            for i, layer in enumerate(self.layers):
                if i in self.msf_layers:
                    hidden_states = hidden_states.permute(0, 2, 1)  # 转换到 (B, D, N)
                    hidden_states = self.MSF(hidden_states)         # 应用 MSF
                    hidden_states = hidden_states.permute(0, 2, 1)  # 转换回原格式 (B, N, D)
                    
                if if_flip_img_sequences and self.if_rope:            # False
                    hidden_states = hidden_states.flip([1])
                    if residual is not None:
                        residual = residual.flip([1])
    
                # rope about
                if self.if_rope:                                       # False
                    hidden_states = self.rope(hidden_states)
                    if residual is not None and self.if_rope_residual:
                        residual = self.rope(residual)
    
                if if_flip_img_sequences and self.if_rope:             # False
                    hidden_states = hidden_states.flip([1])
                    if residual is not None:
                        residual = residual.flip([1])
                ######### 在每个 Mamba 层之前通过多尺度操作帮助增强特征质量。
                # hidden_states = hidden_states.permute(0, 2, 1)  # Change (B, N, D) to (B, D, N)
                # hidden_states = self.MSF(hidden_states)         # Apply MSF
                # hidden_states = hidden_states.permute(0, 2, 1)  # Return to origin
                ########
                hidden_states, residual = layer(
                    hidden_states, residual, inference_params=inference_params
                )
        
        else:             # False
            print('****************************************** Start Processing ****************************************8')
            # get two layers in a single for-loop
            for i in range(len(self.layers) // 2):
                if self.if_rope:
                    hidden_states = self.rope(hidden_states)
                    if residual is not None and self.if_rope_residual:
                        residual = self.rope(residual)
    
                hidden_states_f, residual_f = self.layers[i * 2](
                    hidden_states, residual, inference_params=inference_params
                )
                hidden_states_b, residual_b = self.layers[i * 2 + 1](
                    hidden_states.flip([1]), None if residual == None else residual.flip([1]), inference_params=inference_params
                )
                hidden_states = hidden_states_f + hidden_states_b.flip([1])
                residual = residual_f + residual_b.flip([1])
      
        if not self.fused_add_norm:         #False
            if residual is None:
                residual = hidden_states
            else:
                residual = residual + self.drop_path(hidden_states)
            hidden_states = self.norm_f(residual.to(dtype=self.norm_f.weight.dtype))
        else:       #True
            # Set prenorm=False here since we don't need the residual
            fused_add_norm_fn = rms_norm_fn if isinstance(self.norm_f, RMSNorm) else layer_norm_fn
            hidden_states = fused_add_norm_fn(                                         # hidden_states.shape = torch.Size([B, 320, 384])
                self.drop_path(hidden_states),
                self.norm_f.weight,
                self.norm_f.bias,
                eps=self.norm_f.eps,
                residual=residual,
                prenorm=False,
                residual_in_fp32=self.residual_in_fp32,
            )
    
        # return only cls token if it exists
        if self.if_cls_token:          #False
            if self.use_double_cls_token:
                return (hidden_states[:, token_position[0], :] + hidden_states[:, token_position[1], :]) / 2
            else:
                if self.use_middle_cls_token:
                    return hidden_states[:, token_position, :]
                elif if_random_cls_token_position:
                    return hidden_states[:, token_position, :]
                else:
                    return hidden_states[:, token_position, :]
    
        if self.final_pool_type == 'none':
            return hidden_states[:, -1, :]
        elif self.final_pool_type == 'mean':         #True
            return hidden_states.mean(dim=1)         #hidden_states.shape = torch.Size([2, 384])
        elif self.final_pool_type == 'max':
            return hidden_states
        elif self.final_pool_type == 'all':
            return hidden_states
        else:
            raise NotImplementedError
    
    def forward(self, x, return_features=False, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False):
        x = self.forward_features(x, inference_params, if_random_cls_token_position=if_random_cls_token_position, if_random_token_rank=if_random_token_rank)
        if return_features:
            return x
        x = self.head(x)
        if self.final_pool_type == 'max':
            x = x.max(dim=1)[0]
        return x
    

    @register_model def vim_tiny_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model

    @register_model def vim_tiny_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model

    @register_model def vim_small_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='all', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg()

    if pretrained:
        checkpoint = torch.load(pretrained, map_location="cpu")
        missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False)
        print('Load pretrained model from: ' + pretrained)
    return model
    

    @register_model def vim_small_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model 中MSF模块的集成位置。

  2. Assistant

    该代码实现了一个视觉Transformer模型,称为VisionMamba,其中通过灵活的模块化设计来支持多种功能和特性。其中一个重要的模块是MSF(Multi-Scale Feature),该模块旨在加强特征的多尺度表示能力,提升模型的表现。

    在提供的代码中,MSF模块被集成在VisionMamba类中的特定层之间。具体地,MSF模块的实现和集成如下:

    1. MSF模块的定义:

      • self.MSF = MSF(embed_dim, embed_dim, embed_dim // 2):初始化MSF模块,接受输入和输出的embedding维度为embed_dim,并在初始化中配置合适的参数。
    2. MSF的集成:

      • 在forward_features方法中,self.layers表示模型的多层结构。每层可以是一个典型的Transformer块。
      • 通过self.msf_layers指定在哪些层集成MSF功能。self.msf_layers被设置为一个Python集合,这样可以容易地检查某一层是否需要集成MSF。
      • 在迭代每一层时,如果当前层的索引在self.msf_layers中(通过if i in self.msf_layers进行检查),那么:
        • 特征张量hidden_states会被转换成形状为(B, D, N),然后通过MSF模块进行处理,最后转换回到原始形状(B, N, D)。

    以下是MSF模块集成位置代码的相关部分:

    # 在 VisionMamba 类的初始化中,定义 MSF 模块
    self.MSF = MSF(embed_dim, embed_dim, embed_dim // 2)
    
    # 在 forward_features 方法中,集成 MSF 的代码:
    for i, layer in enumerate(self.layers):
        # 检查当前层是否在需要集成 MSF 的层次中
        if i in self.msf_layers:
            hidden_states = hidden_states.permute(0, 2, 1)  # 转换为 (B, D, N)
            hidden_states = self.MSF(hidden_states)         # 应用 MSF 层
            hidden_states = hidden_states.permute(0, 2, 1)  # 转换回原格式 (B, N, D)
    
        # 继续执行其他的层处理逻辑
    

    通过这种方式,MSF模块被灵活地集成到模型的指定层中,从而增强了不同层特征的多尺度表征能力。有哪些特定的层需要集成MSF,可以通过配置msf_layers参数进行灵活调整。

  3. User

    MSF层被集成在哪里?

  4. Assistant

    在提供的代码中,MSF层的集成位置是通过self.msf_layers指定的。这是一个Python集合,用来存储需要应用MSF操作的层的索引。在forward_features方法中,每当处理一个新的层时,会检查该层是否在self.msf_layers中,如果是,那么就对该层的输出应用MSF操作。

    在代码的初始化部分,它是这样设定的:

    self.msf_layers = set(msf_layers) if msf_layers is not None else set()
    

    在forward_features方法中,MSF操作是这样集成的:

    for i, layer in enumerate(self.layers):
        if i in self.msf_layers:
            hidden_states = hidden_states.permute(0, 2, 1)  # 转换为 (B, D, N)
            hidden_states = self.MSF(hidden_states)         # 应用 MSF 层
            hidden_states = hidden_states.permute(0, 2, 1)  # 转换回原格式 (B, N, D)
        # 继续处理层的其他逻辑...
    

    如果想要指定具体在哪些层应用MSF操作,可以在初始化VisionMamba实例时通过msf_layers参数传入。例如:

    model = VisionMamba(..., msf_layers=[0, 5, 10, 15, 20], ...)
    

    这样,MSF模块将被应用在索引为0, 5, 10, 15, 20的这些层。

    在提供的代码片段中,实际上缺省情况下msf_layers并没有被赋予具体的值,因为它可能在模型实例化时由用户定义。在代码中,注释# 设置具体集成的层数之后,msf_layers被设定为固定的集合{23}可能是一个代码演示或调试的遗留部分,以固定在特定索引23的位置应用MSF。实际应用中,这个配置通常是由开发者自行决定的。

  5. User

    msf_layers = {23}这里的设置是什么

  6. Assistant

    在提供的代码中,msf_layers = {23}的设置意味着MSF模块(Multi-Scale Feature模块)被应用在模型的第23层。在这种情况下,msf_layers被设置为一个包含元素23的Python集合。

    这个设置控制着在哪个层级上应用MSF。具体来说,当层级的索引在msf_layers集合中时,MSF模块将会对该层的特征输出进行处理。

    以下是相关的代码片段:

    # `msf_layers`被指定为{23}
    msf_layers = {23}
    
    for i, layer in enumerate(self.layers):
        if i in self.msf_layers:
            # 如果当前层的索引在msf_layers中,应用MSF
            hidden_states = hidden_states.permute(0, 2, 1)  # 转为(B, D, N),以适合MSF的输入
            hidden_states = self.MSF(hidden_states)         # 应用MSF操作
            hidden_states = hidden_states.permute(0, 2, 1)  # 恢复到(B, N, D)的原始格式
    

    在这个设置下,MSF模块是在特定的第23层之后应用的,这可以是一个示例设置,用来展示如何在特定层应用MSF模块的效果。

    需要注意的是,这种设置通常用于实验阶段,开发者可以通过改变msf_layers集合中的值来选择不同的层进行MSF处理,以观察对模型性能的影响,或者在特定应用中根据需要调整。

  7. User

    那么综上,在 def forward_features(self, z, x, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False): x = self.hin_block(x) z = self.hin_block(z)

        x = self.patch_embed(x)                  #x.shape = torch.Size([B, 3, 256, 256])  -> torch.Size([2, 256, 384])
        z = self.patch_embed(z)                  #z.shape = torch.Size([B, 3, 128, 128])  -> torch.Size([2, 64, 384])
        B, M, _ = x.shape
       
        if self.if_cls_token:                 # False
            if self.use_double_cls_token:
                cls_token_head = self.cls_token_head.expand(B, -1, -1)
                cls_token_tail = self.cls_token_tail.expand(B, -1, -1)
                token_position = [0, M + 1]
                x = torch.cat((cls_token_head, x, cls_token_tail), dim=1)
                M = x.shape[1]
            else:
                if self.use_middle_cls_token:
                    cls_token = self.cls_token.expand(B, -1, -1)
                    token_position = M // 2
                    # add cls token in the middle
                    x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1)       
                elif if_random_cls_token_position:
                    cls_token = self.cls_token.expand(B, -1, -1)
                    token_position = random.randint(0, M)
                    x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1)
                    print("token_position: ", token_position)
                else:
                    cls_token = self.cls_token.expand(B, -1, -1)  # stole cls_tokens impl from Phil Wang, thanks
                    token_position = 0
                    x = torch.cat((cls_token, x), dim=1)
                M = x.shape[1]                 
       
        if self.if_abs_pos_embed:                  # True 
            x = x + self.pos_embed_x               # x = x + positon_embemding =torch.Size([B, 256, 384]) + torch.Size([1, 256, 384]) = torch.Size([B, 256, 384])
            z = z + self.pos_embed_z               # z = z + positon_embemding =torch.Size([B, 64, 384]) + torch.Size([1, 64, 384]) = torch.Size([B, 64, 384])
            x = torch.cat((z, x), dim=1)           # torch.Size([B, 320, 384])
            x = self.pos_drop(x)                   # x.shape = torch.Size([B, 320, 384])
            
        if if_random_token_rank:                   #False
            # 生成随机 shuffle 索引
            shuffle_indices = torch.randperm(M)
    
            if isinstance(token_position, list):
                print("original value: ", x[0, token_position[0], 0], x[0, token_position[1], 0])
            else:
                print("original value: ", x[0, token_position, 0])
            print("original token_position: ", token_position)
    
            # 执行 shuffle
            x = x[:, shuffle_indices, :]
    
            if isinstance(token_position, list):
                # 找到 cls token 在 shuffle 之后的新位置
                new_token_position = [torch.where(shuffle_indices == token_position[i])[0].item() for i in range(len(token_position))]
                token_position = new_token_position
            else:
                # 找到 cls token 在 shuffle 之后的新位置
                token_position = torch.where(shuffle_indices == token_position)[0].item()
    
            if isinstance(token_position, list):
                print("new value: ", x[0, token_position[0], 0], x[0, token_position[1], 0])
            else:
                print("new value: ", x[0, token_position, 0])
            print("new token_position: ", token_position)
    
    
        if_flip_img_sequences = False
        if self.flip_img_sequences_ratio > 0 and (self.flip_img_sequences_ratio - random.random()) > 1e-5:        # False
            x = x.flip([1])
            if_flip_img_sequences = True
    
        # mamba impl
        residual = None
        hidden_states = x
        # 设置具体集成的层数
        msf_layers = {23}
        if not self.if_bidirectional:                                 # True
            for i, layer in enumerate(self.layers):
                if i in self.msf_layers:
                    hidden_states = hidden_states.permute(0, 2, 1)  # 转换到 (B, D, N)
                    hidden_states = self.MSF(hidden_states)         # 应用 MSF
                    hidden_states = hidden_states.permute(0, 2, 1)  # 转换回原格式 (B, N, D)
                    
                if if_flip_img_sequences and self.if_rope:            # False
                    hidden_states = hidden_states.flip([1])
                    if residual is not None:
                        residual = residual.flip([1])
    
                # rope about
                if self.if_rope:                                       # False
                    hidden_states = self.rope(hidden_states)
                    if residual is not None and self.if_rope_residual:
                        residual = self.rope(residual)
    
                if if_flip_img_sequences and self.if_rope:             # False
                    hidden_states = hidden_states.flip([1])
                    if residual is not None:
                        residual = residual.flip([1])
                ######### 在每个 Mamba 层之前通过多尺度操作帮助增强特征质量。
                # hidden_states = hidden_states.permute(0, 2, 1)  # Change (B, N, D) to (B, D, N)
                # hidden_states = self.MSF(hidden_states)         # Apply MSF
                # hidden_states = hidden_states.permute(0, 2, 1)  # Return to origin
                ########
                hidden_states, residual = layer(
                    hidden_states, residual, inference_params=inference_params
                )
        
        else:             # False
            print('****************************************** Start Processing ****************************************8')
            # get two layers in a single for-loop
            for i in range(len(self.layers) // 2):
                if self.if_rope:
                    hidden_states = self.rope(hidden_states)
                    if residual is not None and self.if_rope_residual:
                        residual = self.rope(residual)
    
                hidden_states_f, residual_f = self.layers[i * 2](
                    hidden_states, residual, inference_params=inference_params
                )
                hidden_states_b, residual_b = self.layers[i * 2 + 1](
                    hidden_states.flip([1]), None if residual == None else residual.flip([1]), inference_params=inference_params
                )
                hidden_states = hidden_states_f + hidden_states_b.flip([1])
                residual = residual_f + residual_b.flip([1])
      
        if not self.fused_add_norm:         #False
            if residual is None:
                residual = hidden_states
            else:
                residual = residual + self.drop_path(hidden_states)
            hidden_states = self.norm_f(residual.to(dtype=self.norm_f.weight.dtype))
        else:       #True
            # Set prenorm=False here since we don't need the residual
            fused_add_norm_fn = rms_norm_fn if isinstance(self.norm_f, RMSNorm) else layer_norm_fn
            hidden_states = fused_add_norm_fn(                                         # hidden_states.shape = torch.Size([B, 320, 384])
                self.drop_path(hidden_states),
                self.norm_f.weight,
                self.norm_f.bias,
                eps=self.norm_f.eps,
                residual=residual,
                prenorm=False,
                residual_in_fp32=self.residual_in_fp32,
            )
    
        # return only cls token if it exists
        if self.if_cls_token:          #False
            if self.use_double_cls_token:
                return (hidden_states[:, token_position[0], :] + hidden_states[:, token_position[1], :]) / 2
            else:
                if self.use_middle_cls_token:
                    return hidden_states[:, token_position, :]
                elif if_random_cls_token_position:
                    return hidden_states[:, token_position, :]
                else:
                    return hidden_states[:, token_position, :]
    
        if self.final_pool_type == 'none':
            return hidden_states[:, -1, :]
        elif self.final_pool_type == 'mean':         #True
            return hidden_states.mean(dim=1)         #hidden_states.shape = torch.Size([2, 384])
        elif self.final_pool_type == 'max':
            return hidden_states
        elif self.final_pool_type == 'all':
            return hidden_states
        else:
            raise NotImplementedError
    

    中,他被集成在什么位置?输入经过怎样的处理?

  8. Assistant

    在forward_features方法中,MSF模块被集成到特定的Transformer层之间。具体来说:

    1. MSF的集成位置:

      • MSF模块被应用在某些层的输出特征上,这些层的索引通过self.msf_layers指定。在初始化时,self.msf_layers被设定为{23},意思是在第23层之后应用MSF模块。因此,代码中会检查每层的索引,如果该索引在self.msf_layers中,MSF模块将被应用。
    2. 输入的处理流程:

      • 输入预处理:

        • 输入x和z首先通过self.hin_block进行处理,这是一个可能用于增强特征或预处理输入的模块。
        • 然后,PatchEmbed模块将输入的图像块嵌入到固定维度的特征空间。
      • 位置编码:

        • 如果使用绝对位置编码self.if_abs_pos_embed为True,则将预定义的绝对位置编码添加到特征中。
      • 特征序列的整合:

        • 将两部分特征z和x拼接在一起形成一个更大的特征序列。
      • 处理包含MSF层:

        • 特征序列进入一个多层的转换机制self.layers。
        • 当处理到第23(或者其他指定)层时,特征通过MSF模块进行多尺度特征处理。在MSF应用前后,特征会进行维度调整((B, N, D)到(B, D, N)再返回)。
      • 特征的最终处理:

        • 特征经过所有的层转换后,根据final_pool_type的配置进行不同类型的池化操作(如mean、max或其他)。

    通过这些步骤,输入特征经过一系列嵌入、位置编码、过层转换(包括MSF处理)和池化等操作,最终形成用于下游任务(如分类)的输出特征。

  9. User

    self.layers一共24层(0,23),那么MSF被集成在哪里?

  10. Assistant

    在提供的代码中,self.layers包含24个层(索引从0到23)。根据代码的配置和逻辑,MSF模块被集成在第23层之后。这意味着,在进行到第23层之后的输出(特别是第24层处理之前),将会应用MSF模块来处理特征。

    代码片段展示了如何检查并应用MSF模块:

    # 遍历每一层
    for i, layer in enumerate(self.layers):
        if i in self.msf_layers:
            # 整合MSF模块
            hidden_states = hidden_states.permute(0, 2, 1)  # 转换为 (B, D, N)
            hidden_states = self.MSF(hidden_states)         # 应用MSF功能
            hidden_states = hidden_states.permute(0, 2, 1)  # 转回 (B, N, D)
        
        # 处理当前层
        hidden_states, residual = layer(
            hidden_states, residual, inference_params=inference_params
        )
    

    解释:

    • 在每层被处理之后,会检查该层的索引是否在self.msf_layers中。由于msf_layers被设置为{23},因此这将在第23层索引时触发MSF模块。
    • hidden_states特征在这种情况下会转换为适合MSF处理的格式 (B, D, N),应用MSF后会恢复至(B, N, D)。
    • 然后,特征继续被当前或后续的层处理。

    因此,MSF模块在第23层的输出之后集成,在进行第24层处理之前被应用。

  11. User

    i是从0开始计数的

  12. Assistant

    对于Python中列表的索引,确实是从0开始计数的。如果self.layers包含24个层,这意味着索引范围是从0到23。

    在表示MSF模块集成位置的代码段中:

    msf_layers = {23}
    

    以及随后的遍历和检查:

    for i, layer in enumerate(self.layers):
        if i in self.msf_layers:
            # 应用 MSF 模块
            hidden_states = hidden_states.permute(0, 2, 1)  # 转换为 (B, D, N)
            hidden_states = self.MSF(hidden_states)         # 应用 MSF 功能
            hidden_states = hidden_states.permute(0, 2, 1)  # 转回 (B, N, D)
    
        hidden_states, residual = layer(
            hidden_states, residual, inference_params=inference_params
        )
    

    MSF模块被集成在第23层。这意味着:

    • 在第23层之后:在处理完第23个层(索引为22)之后,并在处理24个层(索引为23)之前,应用MSF模块。
    • 也就是说,MSF模块被集成在即将进入第24层(即self.layers[23])之前的特征序列上。

    综上所述,MSF模块实际上是在23层之后应用,并在第24个层之前被处理。

  13. User

    msf不是在block的最后一层被集成吗

  14. Assistant

    在这段代码中,MSF模块的位置可以像您提到的那样在Block结构中的最后一层(或接近最后的位置)集成。这取决于特定集成的实现细节。在vision及其相关模型中集成MSF,通常发生在Block结构内,而不是VisionMamba类的最顶层。

    从代码结构和逻辑来看,MSF的集成通常是为增强模型中一组特定特征进行的,这意味着它会在一个完整的Block或网络序列末尾被适当地插入,以确保特征在进入最终结论之前得到充分的多尺度处理。

    以下是一个示例说明,在特定模型结构内的每一个Transformer Block内或在几个Block之后可以加上一个MSF模块来集成:

    # 示例位置:在每个Block执行完毕之后
    for i, layer in enumerate(self.layers):
        
        hidden_states, residual = layer(
            hidden_states, residual, inference_params=inference_params
        )
        
        # 检查i是否在msf_layers集合中
        if i in self.msf_layers:
            # 转换特征进行多尺度处理
            hidden_states = hidden_states.permute(0, 2, 1)  # 转换到(B, D, N)
            hidden_states = self.MSF(hidden_states)         # 应用MSF层
            hidden_states = hidden_states.permute(0, 2, 1)  # 转换回(B, N, D)
    

    在这种情况下,MSF模块在一个Block的执行后应用,这也可以看作是Block后的集成。这种安排提供了一种增强特定层输出特征的策略,确保在更高层级处理之前,已经整合了多尺度特征。这也解释了为什么在一些情况下,您可能会描述MSF模块在“最后一层”应用的情况。