Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-08-06· OpenAIAi2 WildChat-4.8M (AI2)

import torch import torch.nn as nn from functools import partial from torch import…

  1. User

    import torch import torch.nn as nn from functools import partial from torch import Tensor from typing import Optional

    from timm.models.vision_transformer import VisionTransformer, cfg from timm.models.registry import register_model from timm.models.layers import trunc_normal, lecun_normal_

    from timm.models.layers import DropPath, to_2tuple from timm.models.vision_transformer import _load_weights

    import math

    from collections import namedtuple

    from mamba_ssm.modules.mamba_simple import Mamba from mamba_ssm.utils.generation import GenerationMixin from mamba_ssm.utils.hf import load_config_hf, load_state_dict_hf #################### 结合当前项目引入 ######################### from lib.models.mamba_fetrack.rope import * import random from lib.models.layers.head import build_box_head import importlib import lib.train.admin.settings as ws_settings from .utils import combine_tokens, recover_tokens ################################################################# import torch.nn.functional as F ############## try: from mamba_ssm.ops.triton.layernorm import RMSNorm, layer_norm_fn, rms_norm_fn except ImportError: RMSNorm, layer_norm_fn, rms_norm_fn = None, None, None

    all = [ 'vim_tiny_patch16_224', 'vim_small_patch16_224', 'vim_base_patch16_224', 'vim_tiny_patch16_384', 'vim_small_patch16_384', 'vim_base_patch16_384', ]

    class MultiScaleEmbed(nn.Module): def init(self,img_size = 224, patch_size=16, stride=16, in_chans=3, embed_dim = 768, scales=[1,2,4],norm_layer=None, flatten=True): super().init() img_size = to_2tuple(img_size) patch_size = to_2tuple(patch_size) self.img_size = img_size self.patch_size = patch_size self.grid_size = ((img_size[0] - patch_size[0]) // stride + 1, (img_size[1] - patch_size[1]) // stride + 1) self.num_patches = self.grid_size[0] * self.grid_size[1] self.flatten = flatten self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=stride) self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()

        # 引入额外层 
        self.additional_convs = nn.ModuleList([
            nn.Sequential(
                nn.Conv2d(in_chans, embed_dim, kernel_size =3, stride=scale,padding=1),
                nn.BatchNorm2d(embed_dim),
                nn.ReLU(inplace=True)
            )
            for scale in scales
        ])
              
    def forward(self, x):
        B, C, H, W = x.shape
        x_proj = self.proj(x)
        if self.flatten:
            x_proj = x_proj.flatten(2).transpose(1, 2)  # BCHW -> BNC
        x_proj = self.norm(x_proj)
    
        # Ensure all multi-scale outputs are resized to match x_proj
        multi_scale_features = [F.interpolate(layer(x), size=(self.grid_size[0], self.grid_size[1]), mode='bilinear', align_corners=False) for layer in self.additional_convs]
        multi_scale_features = torch.cat(multi_scale_features, dim=1)
    
        if self.flatten:
            multi_scale_features = multi_scale_features.flatten(2).transpose(1, 2)  # Ensure BNC shape
        multi_scale_features = self.norm(multi_scale_features)
    
        # Print shapes for debugging purposes
        print('multi-feature.shape: ', multi_scale_features.shape)
        print('x_proj.shape: ', x_proj.shape)
        
        # Ensure shapes are consistent for concatenation
        if multi_scale_features.shape[1] != x_proj.shape[1]:
            raise ValueError(f"Inconsistent shapes for concatenation: {multi_scale_features.shape[1]} != {x_proj.shape[1]}")
    
        # Concatenate features
        combined_features = torch.cat([x_proj, multi_scale_features], dim=-1)
        return combined_features
    

    class PatchEmbed(nn.Module): """ 2D Image to Patch Embedding """ def init(self, img_size=224, patch_size=16, stride=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True): super().init() img_size = to_2tuple(img_size) patch_size = to_2tuple(patch_size) self.img_size = img_size self.patch_size = patch_size self.grid_size = ((img_size[0] - patch_size[0]) // stride + 1, (img_size[1] - patch_size[1]) // stride + 1) self.num_patches = self.grid_size[0] * self.grid_size[1] self.flatten = flatten

        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=stride)
        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
    
    def forward(self, x):
        B, C, H, W = x.shape
        x = self.proj(x)
        if self.flatten:
            x = x.flatten(2).transpose(1, 2)  # BCHW -> BNC
        x = self.norm(x) 
        print('xxxxxxxxxxxxxxxxxx: ', x.shape)
        return x
    

    class Block(nn.Module): def init( self, dim, mixer_cls, norm_cls=nn.LayerNorm, fused_add_norm=False, residual_in_fp32=False,drop_path=0., ): super().init() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.mixer = mixer_cls(dim) self.norm = norm_cls(dim) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() if self.fused_add_norm: assert RMSNorm is not None, "RMSNorm import fails" assert isinstance( self.norm, (nn.LayerNorm, RMSNorm) ), "Only LayerNorm and RMSNorm are supported for fused_add_norm"

    def forward(
        self, hidden_states: Tensor, residual: Optional[Tensor] = None, inference_params=None
    ):
        r"""Pass the input through the encoder layer.
    
        Args:
            hidden_states: the sequence to the encoder layer (required).
            residual: hidden_states = Mixer(LN(residual))
        """
        if not self.fused_add_norm:
            if residual is None:
                residual = hidden_states
            else:
                residual = residual + self.drop_path(hidden_states)
            
            hidden_states = self.norm(residual.to(dtype=self.norm.weight.dtype))
            if self.residual_in_fp32:
                residual = residual.to(torch.float32)
        else:
            fused_add_norm_fn = rms_norm_fn if isinstance(self.norm, RMSNorm) else layer_norm_fn
            if residual is None:
                hidden_states, residual = fused_add_norm_fn(
                    hidden_states,
                    self.norm.weight,
                    self.norm.bias,
                    residual=residual,
                    prenorm=True,
                    residual_in_fp32=self.residual_in_fp32,
                    eps=self.norm.eps,
                )
            else:
                hidden_states, residual = fused_add_norm_fn(
                    self.drop_path(hidden_states),
                    self.norm.weight,
                    self.norm.bias,
                    residual=residual,
                    prenorm=True,
                    residual_in_fp32=self.residual_in_fp32,
                    eps=self.norm.eps,
                )    
        hidden_states = self.mixer(hidden_states, inference_params=inference_params)
        return hidden_states, residual
    
    def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs):
        return self.mixer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)
    

    def create_block( d_model, ssm_cfg=None, norm_epsilon=1e-5, drop_path=0., rms_norm=False, residual_in_fp32=False, fused_add_norm=False, layer_idx=None, device=None, dtype=None, if_bimamba=False, bimamba_type="none", if_devide_out=False, init_layer_scale=None, ): if if_bimamba: bimamba_type = "v1" if ssm_cfg is None: ssm_cfg = {} factory_kwargs = {"device": device, "dtype": dtype} mixer_cls = partial(Mamba, layer_idx=layer_idx, bimamba_type=bimamba_type, if_devide_out=if_devide_out, init_layer_scale=init_layer_scale, **ssm_cfg, **factory_kwargs) norm_cls = partial( nn.LayerNorm if not rms_norm else RMSNorm, eps=norm_epsilon, **factory_kwargs ) block = Block( d_model, mixer_cls, norm_cls=norm_cls, drop_path=drop_path, fused_add_norm=fused_add_norm, residual_in_fp32=residual_in_fp32, ) block.layer_idx = layer_idx return block

    https://github.com/huggingface/transformers/blob/c28d04e9e252a1a099944e325685f14d242ecdcd/src/transformers/models/gpt2/modeling_gpt2.py#L454

    def init_weights( module, n_layer, initializer_range=0.02, # Now only used for embedding layer. rescale_prenorm_residual=True, n_residuals_per_layer=1, # Change to 2 if we have MLP ): if isinstance(module, nn.Linear): if module.bias is not None: if not getattr(module.bias, "no_reinit", False): nn.init.zeros(module.bias) elif isinstance(module, nn.Embedding): nn.init.normal(module.weight, std=initializer_range)

    if rescale_prenorm_residual:
        for name, p in module.named_parameters():
            if name in ["out_proj.weight", "fc2.weight"]:
                nn.init.kaiming_uniform_(p, a=math.sqrt(5))
                with torch.no_grad():
                    p /= math.sqrt(n_residuals_per_layer * n_layer)
    

    def segm_init_weights(m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=0.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.Conv2d): # NOTE conv was left to pytorch default in my original init lecun_normal_(m.weight) if m.bias is not None: nn.init.zeros_(m.bias) elif isinstance(m, (nn.LayerNorm, nn.GroupNorm, nn.BatchNorm2d)): nn.init.zeros_(m.bias) nn.init.ones_(m.weight)

    class VisionMamba(nn.Module): def init(self, img_size=224, patch_size=16, stride=16, depth=24, embed_dim=192, channels=3, num_classes=1000, ssm_cfg=None, drop_rate=0., drop_path_rate=0.1, norm_epsilon: float = 1e-5, rms_norm: bool = False, initializer_cfg=None, fused_add_norm=False, residual_in_fp32=False, device=None, dtype=None, ft_seq_len=None, pt_hw_seq_len=14, if_bidirectional=False, final_pool_type='none', if_abs_pos_embed=False, if_rope=False, if_rope_residual=False, flip_img_sequences_ratio=-1., if_bimamba=False, bimamba_type="none", if_cls_token=False, if_devide_out=False, init_layer_scale=None, use_double_cls_token=False, use_middle_cls_token=False, **kwargs): factory_kwargs = {"device": device, "dtype": dtype} # add factory_kwargs into kwargs kwargs.update(factory_kwargs) super().init() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.if_bidirectional = if_bidirectional self.final_pool_type = final_pool_type self.if_abs_pos_embed = if_abs_pos_embed self.if_rope = if_rope self.if_rope_residual = if_rope_residual self.flip_img_sequences_ratio = flip_img_sequences_ratio # self.if_cls_token = if_cls_token (此处与Mamba源代码不同) self.if_cls_token = False self.use_double_cls_token = use_double_cls_token self.use_middle_cls_token = use_middle_cls_token self.num_tokens = 1 if if_cls_token else 0

        # pretrain parameters
        self.num_classes = num_classes
        self.d_model = self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
        ############## 原始尺度 ###################
        # self.patch_embed = PatchEmbed(
        #     img_size=img_size, patch_size=patch_size, stride=stride, in_chans=channels, embed_dim=embed_dim)
        ############### 多尺度 ##################
        self.patch_embed = MultiScaleEmbed(img_size=img_size, patch_size=patch_size, stride=stride, in_chans=channels, embed_dim=embed_dim)
        num_patches = self.patch_embed.num_patches
    
        if if_cls_token:
            if use_double_cls_token:
                self.cls_token_head = nn.Parameter(torch.zeros(1, 1, self.embed_dim))
                self.cls_token_tail = nn.Parameter(torch.zeros(1, 1, self.embed_dim))
                self.num_tokens = 2
            else:
                self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embed_dim))
                # self.num_tokens = 1
            
        if if_abs_pos_embed:
            # self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, self.embed_dim))
            # 此处与mamba不同,划分了x和z的位置编码
            self.pos_embed_x = nn.Parameter(torch.zeros(1, 256, self.embed_dim))
            self.pos_embed_z = nn.Parameter(torch.zeros(1, 64, self.embed_dim))
            self.pos_drop = nn.Dropout(p=drop_rate)
    
        if if_rope:
            half_head_dim = embed_dim // 2
            hw_seq_len = img_size // patch_size
            self.rope = VisionRotaryEmbeddingFast(
                dim=half_head_dim,
                pt_seq_len=pt_hw_seq_len,
                ft_seq_len=hw_seq_len
            )
    
        # TODO: release this comment
        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
        # import ipdb;ipdb.set_trace()
        inter_dpr = [0.0] + dpr
        self.drop_path = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity()
                # transformer blocks
        self.layers = nn.ModuleList(
            [
                create_block(
                    embed_dim,
                    ssm_cfg=ssm_cfg,
                    norm_epsilon=norm_epsilon,
                    rms_norm=rms_norm,
                    residual_in_fp32=residual_in_fp32,
                    fused_add_norm=fused_add_norm,
                    layer_idx=i,
                    if_bimamba=if_bimamba,
                    bimamba_type=bimamba_type,
                    drop_path=inter_dpr[i],
                    if_devide_out=if_devide_out,
                    init_layer_scale=init_layer_scale,
                    **factory_kwargs,
                )
                for i in range(depth)
            ]
        )
        
        # output head
        self.norm_f = (nn.LayerNorm if not rms_norm else RMSNorm)(
            embed_dim, eps=norm_epsilon, **factory_kwargs
        )
    
        # self.pre_logits = nn.Identity()
    
        # original init
        self.patch_embed.apply(segm_init_weights)
        # 此处与mamba不同,问题在于没有执行head相关操作
        # self.head.apply(segm_init_weights)
        # 在pos_embed中也进行了模板和搜索的划分
        if if_abs_pos_embed:
            trunc_normal_(self.pos_embed_x, std=.02)
            trunc_normal_(self.pos_embed_z, std=.02)
            
        if if_cls_token:
            if use_double_cls_token:
                trunc_normal_(self.cls_token_head, std=.02)
                trunc_normal_(self.cls_token_tail, std=.02)
            else:
                trunc_normal_(self.cls_token, std=.02)
    
        # mamba init
        self.apply(
            partial(
                _init_weights,
                n_layer=depth,
                **(initializer_cfg if initializer_cfg is not None else {}),
            )
        )
    
    
    def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs):
        return {
            i: layer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)
            for i, layer in enumerate(self.layers)
        }
    
    @torch.jit.ignore
    def no_weight_decay(self):
        return {"pos_embed", "cls_token", "dist_token", "cls_token_head", "cls_token_tail"}
    
    @torch.jit.ignore()
    def load_pretrained(self, checkpoint_path, prefix=""):
        _load_weights(self, checkpoint_path, prefix)
    
    def forward_features(self, z, x, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False):
        # taken from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py
        # with slight modifications to add the dist_token
        x = self.patch_embed(x)                  #x.shape = torch.Size([B, 3, 256, 256])  -> torch.Size([2, 256, 384])
        z = self.patch_embed(z)                  #z.shape = torch.Size([B, 3, 128, 128])  -> torch.Size([2, 64, 384])
        B, M, _ = x.shape
       
        if self.if_cls_token:                 # False
            if self.use_double_cls_token:
                cls_token_head = self.cls_token_head.expand(B, -1, -1)
                cls_token_tail = self.cls_token_tail.expand(B, -1, -1)
                token_position = [0, M + 1]
                x = torch.cat((cls_token_head, x, cls_token_tail), dim=1)
                M = x.shape[1]
            else:
                if self.use_middle_cls_token:
                    cls_token = self.cls_token.expand(B, -1, -1)
                    token_position = M // 2
                    # add cls token in the middle
                    x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1)       
                elif if_random_cls_token_position:
                    cls_token = self.cls_token.expand(B, -1, -1)
                    token_position = random.randint(0, M)
                    x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1)
                    print("token_position: ", token_position)
                else:
                    cls_token = self.cls_token.expand(B, -1, -1)  # stole cls_tokens impl from Phil Wang, thanks
                    token_position = 0
                    x = torch.cat((cls_token, x), dim=1)
                M = x.shape[1]                 
        # 此处由于划分了模板和搜索,pos_embed也需要单独执行
        if self.if_abs_pos_embed:                  # True 
            x = x + self.pos_embed_x               # x = x + positon_embemding =torch.Size([B, 256, 384]) + torch.Size([1, 256, 384]) = torch.Size([B, 256, 384])
            z = z + self.pos_embed_z               # z = z + positon_embemding =torch.Size([B, 64, 384]) + torch.Size([1, 64, 384]) = torch.Size([B, 64, 384])
            x = torch.cat((z, x), dim=1)           # torch.Size([B, 320, 384])
            x = self.pos_drop(x)                   # x.shape = torch.Size([B, 320, 384])
            
        if if_random_token_rank:                   #False
            # 生成随机 shuffle 索引
            shuffle_indices = torch.randperm(M)
    
            if isinstance(token_position, list):
                print("original value: ", x[0, token_position[0], 0], x[0, token_position[1], 0])
            else:
                print("original value: ", x[0, token_position, 0])
            print("original token_position: ", token_position)
    
            # 执行 shuffle
            x = x[:, shuffle_indices, :]
    
            if isinstance(token_position, list):
                # 找到 cls token 在 shuffle 之后的新位置
                new_token_position = [torch.where(shuffle_indices == token_position[i])[0].item() for i in range(len(token_position))]
                token_position = new_token_position
            else:
                # 找到 cls token 在 shuffle 之后的新位置
                token_position = torch.where(shuffle_indices == token_position)[0].item()
    
            if isinstance(token_position, list):
                print("new value: ", x[0, token_position[0], 0], x[0, token_position[1], 0])
            else:
                print("new value: ", x[0, token_position, 0])
            print("new token_position: ", token_position)
    
    
        if_flip_img_sequences = False
        if self.flip_img_sequences_ratio > 0 and (self.flip_img_sequences_ratio - random.random()) > 1e-5:        # False
            x = x.flip([1])
            if_flip_img_sequences = True
    
        # mamba impl
        residual = None
        hidden_states = x
        if not self.if_bidirectional:                                 # True
            for layer in self.layers:
                if if_flip_img_sequences and self.if_rope:            # False
                    hidden_states = hidden_states.flip([1])
                    if residual is not None:
                        residual = residual.flip([1])
    
                # rope about
                if self.if_rope:                                       # False
                    hidden_states = self.rope(hidden_states)
                    if residual is not None and self.if_rope_residual:
                        residual = self.rope(residual)
    
                if if_flip_img_sequences and self.if_rope:             # False
                    hidden_states = hidden_states.flip([1])
                    if residual is not None:
                        residual = residual.flip([1])
    
                hidden_states, residual = layer(
                    hidden_states, residual, inference_params=inference_params
                )
        
        else:             # False
            # get two layers in a single for-loop
            for i in range(len(self.layers) // 2):
                if self.if_rope:
                    hidden_states = self.rope(hidden_states)
                    if residual is not None and self.if_rope_residual:
                        residual = self.rope(residual)
    
                hidden_states_f, residual_f = self.layers[i * 2](
                    hidden_states, residual, inference_params=inference_params
                )
                hidden_states_b, residual_b = self.layers[i * 2 + 1](
                    hidden_states.flip([1]), None if residual == None else residual.flip([1]), inference_params=inference_params
                )
                hidden_states = hidden_states_f + hidden_states_b.flip([1])
                residual = residual_f + residual_b.flip([1])
      
        if not self.fused_add_norm:         #False
            if residual is None:
                residual = hidden_states
            else:
                residual = residual + self.drop_path(hidden_states)
            hidden_states = self.norm_f(residual.to(dtype=self.norm_f.weight.dtype))
        else:       #True
            # Set prenorm=False here since we don't need the residual
            fused_add_norm_fn = rms_norm_fn if isinstance(self.norm_f, RMSNorm) else layer_norm_fn
            hidden_states = fused_add_norm_fn(                                         # hidden_states.shape = torch.Size([B, 320, 384])
                self.drop_path(hidden_states),
                self.norm_f.weight,
                self.norm_f.bias,
                eps=self.norm_f.eps,
                residual=residual,
                prenorm=False,
                residual_in_fp32=self.residual_in_fp32,
            )
    
        # return only cls token if it exists
        if self.if_cls_token:          #False
            if self.use_double_cls_token:
                return (hidden_states[:, token_position[0], :] + hidden_states[:, token_position[1], :]) / 2
            else:
                if self.use_middle_cls_token:
                    return hidden_states[:, token_position, :]
                elif if_random_cls_token_position:
                    return hidden_states[:, token_position, :]
                else:
                    return hidden_states[:, token_position, :]
    
        if self.final_pool_type == 'none':
            return hidden_states[:, -1, :]
        elif self.final_pool_type == 'mean':         #True
            return hidden_states.mean(dim=1)         #hidden_states.shape = torch.Size([2, 384])
        elif self.final_pool_type == 'max':
            return hidden_states
        elif self.final_pool_type == 'all':
            return hidden_states
        else:
            raise NotImplementedError
    
    def forward(self, x, return_features=False, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False):
        x = self.forward_features(x, inference_params, if_random_cls_token_position=if_random_cls_token_position, if_random_token_rank=if_random_token_rank)
        if return_features:
            return x
        x = self.head(x)
        if self.final_pool_type == 'max':
            x = x.max(dim=1)[0]
        return x
    

    @register_model def vim_tiny_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model

    @register_model def vim_tiny_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model

    @register_model def vim_small_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='all', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg()

    if pretrained:
        checkpoint = torch.load(pretrained, map_location="cpu")
        missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False)
        print('Load pretrained model from: ' + pretrained)
    return model
    

    @register_model def vim_small_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model,报错:multi-feature.shape: torch.Size([4, 256, 1152]) x_proj.shape: torch.Size([4, 256, 384]) multi-feature.shape: torch.Size([4, 256, 1152]) x_proj.shape: torch.Size([4, 64, 384]) Training crashed at epoch 1 Traceback for the error! Traceback (most recent call last): File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/train/trainers/base_trainer.py", line 85, in train self.train_epoch() File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/train/trainers/ltr_trainer.py", line 133, in train_epoch self.cycle_dataset(loader) File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/train/trainers/ltr_trainer.py", line 86, in cycle_dataset loss, stats = self.actor(data) File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/train/actors/mamba_fetrack.py", line 31, in call out_dict = self.forward_pass(data) File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/train/actors/mamba_fetrack.py", line 74, in forward_pass out_dict = self.net(template=template_list, File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/models/mamba_fetrack/mamba_fetrack.py", line 54, in forward rgb_feature = self.backbone.forward_features( z=template, x=search, #[B, 320, 384] File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/models/mamba_fetrack/models_mamba.py", line 424, in forward_features z = self.patch_embed(z) #z.shape = torch.Size([B, 3, 128, 128]) -> torch.Size([2, 64, 384]) File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/models/mamba_fetrack/models_mamba.py", line 88, in forward raise ValueError(f"Inconsistent shapes for concatenation: {multi_scale_features.shape[1]} != {x_proj.shape[1]}") ValueError: Inconsistent shapes for concatenation: 256 != 64

    Restarting training from last epoch ... Finished training!

  2. Assistant

    The error you're encountering is due to a mismatch in the number of tokens in the multi-scale features and the projected features before concatenation in your MultiScaleEmbed module's forward method.

    Here's the specific problematic line in your code:

    if multi_scale_features.shape[1] != x_proj.shape[1]:
        raise ValueError(f"Inconsistent shapes for concatenation: {multi_scale_features.shape[1]} != {x_proj.shape[1]}")
    

    Understanding the Problem:

    • multi_scale_features is generated by interpolating several layers' outputs to x_proj's grid size.
    • However, it's clear from your logs (i.e., multi-feature.shape: torch.Size([4, 256, 1152]) and x_proj.shape: torch.Size([4, 256, 384])) that the channel dimensions are inconsistent, which prevents concatenation along that dimension.

    Possible Reasons and Solutions:

    1. Grid Size Mismatch:

      • From the shapes, the second dimension mismatch likely comes from the spatial dimensions in the original features being inconsistent even after ensuring the interpolation to match the grid sizes:
      • Make sure that self.grid_size[0] and self.grid_size[1] in both the x and z passes are calculated correctly (bounding box and search image) based on their respective input image sizes.
    2. Expected Output:

      • You might need a consistent approach to handle both spatial dim differences and feature channel dims as a common guide here is to either average pool, max pool, or align shapes but often resize them accordingly to some expected constant shape.
    3. Adjust Output Shapes Post-Concat:

      • A common tactic is using extra pooling or upsampling to match these expected dims before the forward function returns the tensor with aligned_multi_scale_features.shape == x_proj.shape.
    4. Verify Interpolation Strategy:

      • Ensure your multi-scale features are interpolated to the shape (self.grid_size[0], self.grid_size[1]). Print out the grid sizes for both z and x (if they differ, fix one to match the other consistently).
    5. Example Debugging Step:

    print(f"x_proj grid size (expected): {self.grid_size[0]}x{self.grid_size[1]}")
    print(f"multi_scale_features grid size (received): {multi_scale_features.shape[2]}x{multi_scale_features.shape[3]}")
    
    1. Normalization Consistency:

      • Ensure that both x_proj and multi_scale_features go through the same normalization process (if there's any extra scaling or wrapping around norm apply after feature calculation consider shifting them directly).
    2. Modification Suggestion:

      • Fix interpolation logic to a unified grid size defined at the class initialization once.
      • Change the concatenation axis or perform any necessary transformations before concatenation to ensure dimensions match.

    Applying these tactics above should help resolve the inconsistency in tensor shapes that prevents successful model training without runtime errors.