import torch import torch.nn as nn from functools import partial from torch import…
- User
import torch import torch.nn as nn from functools import partial from torch import Tensor from typing import Optional
from timm.models.vision_transformer import VisionTransformer, cfg from timm.models.registry import register_model from timm.models.layers import trunc_normal, lecun_normal_
from timm.models.layers import DropPath, to_2tuple from timm.models.vision_transformer import _load_weights
import math
from collections import namedtuple
from mamba_ssm.modules.mamba_simple import Mamba from mamba_ssm.utils.generation import GenerationMixin from mamba_ssm.utils.hf import load_config_hf, load_state_dict_hf #################### 结合当前项目引入 ######################### from lib.models.mamba_fetrack.rope import * import random from lib.models.layers.head import build_box_head import importlib import lib.train.admin.settings as ws_settings from .utils import combine_tokens, recover_tokens ################################################################# import torch.nn.functional as F ############## try: from mamba_ssm.ops.triton.layernorm import RMSNorm, layer_norm_fn, rms_norm_fn except ImportError: RMSNorm, layer_norm_fn, rms_norm_fn = None, None, None
all = [ 'vim_tiny_patch16_224', 'vim_small_patch16_224', 'vim_base_patch16_224', 'vim_tiny_patch16_384', 'vim_small_patch16_384', 'vim_base_patch16_384', ]
class MultiScaleEmbed(nn.Module): def init(self,img_size = 224, patch_size=16, stride=16, in_chans=3, embed_dim = 768, scales=[1,2,4],norm_layer=None, flatten=True): super().init() img_size = to_2tuple(img_size) patch_size = to_2tuple(patch_size) self.img_size = img_size self.patch_size = patch_size self.grid_size = ((img_size[0] - patch_size[0]) // stride + 1, (img_size[1] - patch_size[1]) // stride + 1) self.num_patches = self.grid_size[0] * self.grid_size[1] self.flatten = flatten self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=stride) self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
# 引入额外层 self.additional_convs = nn.ModuleList([ nn.Sequential( nn.Conv2d(in_chans, embed_dim, kernel_size =3, stride=scale,padding=1), nn.BatchNorm2d(embed_dim), nn.ReLU(inplace=True) ) for scale in scales ]) def forward(self, x): B, C, H, W = x.shape x_proj = self.proj(x) if self.flatten: x_proj = x_proj.flatten(2).transpose(1, 2) # BCHW -> BNC x_proj = self.norm(x_proj) # Ensure all multi-scale outputs are resized to match x_proj multi_scale_features = [F.interpolate(layer(x), size=(self.grid_size[0], self.grid_size[1]), mode='bilinear', align_corners=False) for layer in self.additional_convs] multi_scale_features = torch.cat(multi_scale_features, dim=1) if self.flatten: multi_scale_features = multi_scale_features.flatten(2).transpose(1, 2) # Ensure BNC shape multi_scale_features = self.norm(multi_scale_features) # Print shapes for debugging purposes print('multi-feature.shape: ', multi_scale_features.shape) print('x_proj.shape: ', x_proj.shape) # Ensure shapes are consistent for concatenation if multi_scale_features.shape[1] != x_proj.shape[1]: raise ValueError(f"Inconsistent shapes for concatenation: {multi_scale_features.shape[1]} != {x_proj.shape[1]}") # Concatenate features combined_features = torch.cat([x_proj, multi_scale_features], dim=-1) return combined_featuresclass PatchEmbed(nn.Module): """ 2D Image to Patch Embedding """ def init(self, img_size=224, patch_size=16, stride=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True): super().init() img_size = to_2tuple(img_size) patch_size = to_2tuple(patch_size) self.img_size = img_size self.patch_size = patch_size self.grid_size = ((img_size[0] - patch_size[0]) // stride + 1, (img_size[1] - patch_size[1]) // stride + 1) self.num_patches = self.grid_size[0] * self.grid_size[1] self.flatten = flatten
self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=stride) self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() def forward(self, x): B, C, H, W = x.shape x = self.proj(x) if self.flatten: x = x.flatten(2).transpose(1, 2) # BCHW -> BNC x = self.norm(x) print('xxxxxxxxxxxxxxxxxx: ', x.shape) return xclass Block(nn.Module): def init( self, dim, mixer_cls, norm_cls=nn.LayerNorm, fused_add_norm=False, residual_in_fp32=False,drop_path=0., ): super().init() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.mixer = mixer_cls(dim) self.norm = norm_cls(dim) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() if self.fused_add_norm: assert RMSNorm is not None, "RMSNorm import fails" assert isinstance( self.norm, (nn.LayerNorm, RMSNorm) ), "Only LayerNorm and RMSNorm are supported for fused_add_norm"
def forward( self, hidden_states: Tensor, residual: Optional[Tensor] = None, inference_params=None ): r"""Pass the input through the encoder layer. Args: hidden_states: the sequence to the encoder layer (required). residual: hidden_states = Mixer(LN(residual)) """ if not self.fused_add_norm: if residual is None: residual = hidden_states else: residual = residual + self.drop_path(hidden_states) hidden_states = self.norm(residual.to(dtype=self.norm.weight.dtype)) if self.residual_in_fp32: residual = residual.to(torch.float32) else: fused_add_norm_fn = rms_norm_fn if isinstance(self.norm, RMSNorm) else layer_norm_fn if residual is None: hidden_states, residual = fused_add_norm_fn( hidden_states, self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) else: hidden_states, residual = fused_add_norm_fn( self.drop_path(hidden_states), self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) hidden_states = self.mixer(hidden_states, inference_params=inference_params) return hidden_states, residual def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return self.mixer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)def create_block( d_model, ssm_cfg=None, norm_epsilon=1e-5, drop_path=0., rms_norm=False, residual_in_fp32=False, fused_add_norm=False, layer_idx=None, device=None, dtype=None, if_bimamba=False, bimamba_type="none", if_devide_out=False, init_layer_scale=None, ): if if_bimamba: bimamba_type = "v1" if ssm_cfg is None: ssm_cfg = {} factory_kwargs = {"device": device, "dtype": dtype} mixer_cls = partial(Mamba, layer_idx=layer_idx, bimamba_type=bimamba_type, if_devide_out=if_devide_out, init_layer_scale=init_layer_scale, **ssm_cfg, **factory_kwargs) norm_cls = partial( nn.LayerNorm if not rms_norm else RMSNorm, eps=norm_epsilon, **factory_kwargs ) block = Block( d_model, mixer_cls, norm_cls=norm_cls, drop_path=drop_path, fused_add_norm=fused_add_norm, residual_in_fp32=residual_in_fp32, ) block.layer_idx = layer_idx return block
https://github.com/huggingface/transformers/blob/c28d04e9e252a1a099944e325685f14d242ecdcd/src/transformers/models/gpt2/modeling_gpt2.py#L454
def init_weights( module, n_layer, initializer_range=0.02, # Now only used for embedding layer. rescale_prenorm_residual=True, n_residuals_per_layer=1, # Change to 2 if we have MLP ): if isinstance(module, nn.Linear): if module.bias is not None: if not getattr(module.bias, "no_reinit", False): nn.init.zeros(module.bias) elif isinstance(module, nn.Embedding): nn.init.normal(module.weight, std=initializer_range)
if rescale_prenorm_residual: for name, p in module.named_parameters(): if name in ["out_proj.weight", "fc2.weight"]: nn.init.kaiming_uniform_(p, a=math.sqrt(5)) with torch.no_grad(): p /= math.sqrt(n_residuals_per_layer * n_layer)def segm_init_weights(m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=0.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.Conv2d): # NOTE conv was left to pytorch default in my original init lecun_normal_(m.weight) if m.bias is not None: nn.init.zeros_(m.bias) elif isinstance(m, (nn.LayerNorm, nn.GroupNorm, nn.BatchNorm2d)): nn.init.zeros_(m.bias) nn.init.ones_(m.weight)
class VisionMamba(nn.Module): def init(self, img_size=224, patch_size=16, stride=16, depth=24, embed_dim=192, channels=3, num_classes=1000, ssm_cfg=None, drop_rate=0., drop_path_rate=0.1, norm_epsilon: float = 1e-5, rms_norm: bool = False, initializer_cfg=None, fused_add_norm=False, residual_in_fp32=False, device=None, dtype=None, ft_seq_len=None, pt_hw_seq_len=14, if_bidirectional=False, final_pool_type='none', if_abs_pos_embed=False, if_rope=False, if_rope_residual=False, flip_img_sequences_ratio=-1., if_bimamba=False, bimamba_type="none", if_cls_token=False, if_devide_out=False, init_layer_scale=None, use_double_cls_token=False, use_middle_cls_token=False, **kwargs): factory_kwargs = {"device": device, "dtype": dtype} # add factory_kwargs into kwargs kwargs.update(factory_kwargs) super().init() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.if_bidirectional = if_bidirectional self.final_pool_type = final_pool_type self.if_abs_pos_embed = if_abs_pos_embed self.if_rope = if_rope self.if_rope_residual = if_rope_residual self.flip_img_sequences_ratio = flip_img_sequences_ratio # self.if_cls_token = if_cls_token (此处与Mamba源代码不同) self.if_cls_token = False self.use_double_cls_token = use_double_cls_token self.use_middle_cls_token = use_middle_cls_token self.num_tokens = 1 if if_cls_token else 0
# pretrain parameters self.num_classes = num_classes self.d_model = self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models ############## 原始尺度 ################### # self.patch_embed = PatchEmbed( # img_size=img_size, patch_size=patch_size, stride=stride, in_chans=channels, embed_dim=embed_dim) ############### 多尺度 ################## self.patch_embed = MultiScaleEmbed(img_size=img_size, patch_size=patch_size, stride=stride, in_chans=channels, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches if if_cls_token: if use_double_cls_token: self.cls_token_head = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) self.cls_token_tail = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) self.num_tokens = 2 else: self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) # self.num_tokens = 1 if if_abs_pos_embed: # self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, self.embed_dim)) # 此处与mamba不同,划分了x和z的位置编码 self.pos_embed_x = nn.Parameter(torch.zeros(1, 256, self.embed_dim)) self.pos_embed_z = nn.Parameter(torch.zeros(1, 64, self.embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) if if_rope: half_head_dim = embed_dim // 2 hw_seq_len = img_size // patch_size self.rope = VisionRotaryEmbeddingFast( dim=half_head_dim, pt_seq_len=pt_hw_seq_len, ft_seq_len=hw_seq_len ) # TODO: release this comment dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule # import ipdb;ipdb.set_trace() inter_dpr = [0.0] + dpr self.drop_path = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity() # transformer blocks self.layers = nn.ModuleList( [ create_block( embed_dim, ssm_cfg=ssm_cfg, norm_epsilon=norm_epsilon, rms_norm=rms_norm, residual_in_fp32=residual_in_fp32, fused_add_norm=fused_add_norm, layer_idx=i, if_bimamba=if_bimamba, bimamba_type=bimamba_type, drop_path=inter_dpr[i], if_devide_out=if_devide_out, init_layer_scale=init_layer_scale, **factory_kwargs, ) for i in range(depth) ] ) # output head self.norm_f = (nn.LayerNorm if not rms_norm else RMSNorm)( embed_dim, eps=norm_epsilon, **factory_kwargs ) # self.pre_logits = nn.Identity() # original init self.patch_embed.apply(segm_init_weights) # 此处与mamba不同,问题在于没有执行head相关操作 # self.head.apply(segm_init_weights) # 在pos_embed中也进行了模板和搜索的划分 if if_abs_pos_embed: trunc_normal_(self.pos_embed_x, std=.02) trunc_normal_(self.pos_embed_z, std=.02) if if_cls_token: if use_double_cls_token: trunc_normal_(self.cls_token_head, std=.02) trunc_normal_(self.cls_token_tail, std=.02) else: trunc_normal_(self.cls_token, std=.02) # mamba init self.apply( partial( _init_weights, n_layer=depth, **(initializer_cfg if initializer_cfg is not None else {}), ) ) def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return { i: layer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs) for i, layer in enumerate(self.layers) } @torch.jit.ignore def no_weight_decay(self): return {"pos_embed", "cls_token", "dist_token", "cls_token_head", "cls_token_tail"} @torch.jit.ignore() def load_pretrained(self, checkpoint_path, prefix=""): _load_weights(self, checkpoint_path, prefix) def forward_features(self, z, x, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False): # taken from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py # with slight modifications to add the dist_token x = self.patch_embed(x) #x.shape = torch.Size([B, 3, 256, 256]) -> torch.Size([2, 256, 384]) z = self.patch_embed(z) #z.shape = torch.Size([B, 3, 128, 128]) -> torch.Size([2, 64, 384]) B, M, _ = x.shape if self.if_cls_token: # False if self.use_double_cls_token: cls_token_head = self.cls_token_head.expand(B, -1, -1) cls_token_tail = self.cls_token_tail.expand(B, -1, -1) token_position = [0, M + 1] x = torch.cat((cls_token_head, x, cls_token_tail), dim=1) M = x.shape[1] else: if self.use_middle_cls_token: cls_token = self.cls_token.expand(B, -1, -1) token_position = M // 2 # add cls token in the middle x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1) elif if_random_cls_token_position: cls_token = self.cls_token.expand(B, -1, -1) token_position = random.randint(0, M) x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1) print("token_position: ", token_position) else: cls_token = self.cls_token.expand(B, -1, -1) # stole cls_tokens impl from Phil Wang, thanks token_position = 0 x = torch.cat((cls_token, x), dim=1) M = x.shape[1] # 此处由于划分了模板和搜索,pos_embed也需要单独执行 if self.if_abs_pos_embed: # True x = x + self.pos_embed_x # x = x + positon_embemding =torch.Size([B, 256, 384]) + torch.Size([1, 256, 384]) = torch.Size([B, 256, 384]) z = z + self.pos_embed_z # z = z + positon_embemding =torch.Size([B, 64, 384]) + torch.Size([1, 64, 384]) = torch.Size([B, 64, 384]) x = torch.cat((z, x), dim=1) # torch.Size([B, 320, 384]) x = self.pos_drop(x) # x.shape = torch.Size([B, 320, 384]) if if_random_token_rank: #False # 生成随机 shuffle 索引 shuffle_indices = torch.randperm(M) if isinstance(token_position, list): print("original value: ", x[0, token_position[0], 0], x[0, token_position[1], 0]) else: print("original value: ", x[0, token_position, 0]) print("original token_position: ", token_position) # 执行 shuffle x = x[:, shuffle_indices, :] if isinstance(token_position, list): # 找到 cls token 在 shuffle 之后的新位置 new_token_position = [torch.where(shuffle_indices == token_position[i])[0].item() for i in range(len(token_position))] token_position = new_token_position else: # 找到 cls token 在 shuffle 之后的新位置 token_position = torch.where(shuffle_indices == token_position)[0].item() if isinstance(token_position, list): print("new value: ", x[0, token_position[0], 0], x[0, token_position[1], 0]) else: print("new value: ", x[0, token_position, 0]) print("new token_position: ", token_position) if_flip_img_sequences = False if self.flip_img_sequences_ratio > 0 and (self.flip_img_sequences_ratio - random.random()) > 1e-5: # False x = x.flip([1]) if_flip_img_sequences = True # mamba impl residual = None hidden_states = x if not self.if_bidirectional: # True for layer in self.layers: if if_flip_img_sequences and self.if_rope: # False hidden_states = hidden_states.flip([1]) if residual is not None: residual = residual.flip([1]) # rope about if self.if_rope: # False hidden_states = self.rope(hidden_states) if residual is not None and self.if_rope_residual: residual = self.rope(residual) if if_flip_img_sequences and self.if_rope: # False hidden_states = hidden_states.flip([1]) if residual is not None: residual = residual.flip([1]) hidden_states, residual = layer( hidden_states, residual, inference_params=inference_params ) else: # False # get two layers in a single for-loop for i in range(len(self.layers) // 2): if self.if_rope: hidden_states = self.rope(hidden_states) if residual is not None and self.if_rope_residual: residual = self.rope(residual) hidden_states_f, residual_f = self.layers[i * 2]( hidden_states, residual, inference_params=inference_params ) hidden_states_b, residual_b = self.layers[i * 2 + 1]( hidden_states.flip([1]), None if residual == None else residual.flip([1]), inference_params=inference_params ) hidden_states = hidden_states_f + hidden_states_b.flip([1]) residual = residual_f + residual_b.flip([1]) if not self.fused_add_norm: #False if residual is None: residual = hidden_states else: residual = residual + self.drop_path(hidden_states) hidden_states = self.norm_f(residual.to(dtype=self.norm_f.weight.dtype)) else: #True # Set prenorm=False here since we don't need the residual fused_add_norm_fn = rms_norm_fn if isinstance(self.norm_f, RMSNorm) else layer_norm_fn hidden_states = fused_add_norm_fn( # hidden_states.shape = torch.Size([B, 320, 384]) self.drop_path(hidden_states), self.norm_f.weight, self.norm_f.bias, eps=self.norm_f.eps, residual=residual, prenorm=False, residual_in_fp32=self.residual_in_fp32, ) # return only cls token if it exists if self.if_cls_token: #False if self.use_double_cls_token: return (hidden_states[:, token_position[0], :] + hidden_states[:, token_position[1], :]) / 2 else: if self.use_middle_cls_token: return hidden_states[:, token_position, :] elif if_random_cls_token_position: return hidden_states[:, token_position, :] else: return hidden_states[:, token_position, :] if self.final_pool_type == 'none': return hidden_states[:, -1, :] elif self.final_pool_type == 'mean': #True return hidden_states.mean(dim=1) #hidden_states.shape = torch.Size([2, 384]) elif self.final_pool_type == 'max': return hidden_states elif self.final_pool_type == 'all': return hidden_states else: raise NotImplementedError def forward(self, x, return_features=False, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False): x = self.forward_features(x, inference_params, if_random_cls_token_position=if_random_cls_token_position, if_random_token_rank=if_random_token_rank) if return_features: return x x = self.head(x) if self.final_pool_type == 'max': x = x.max(dim=1)[0] return x@register_model def vim_tiny_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model
@register_model def vim_tiny_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model
@register_model def vim_small_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='all', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg()
if pretrained: checkpoint = torch.load(pretrained, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False) print('Load pretrained model from: ' + pretrained) return model@register_model def vim_small_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model,报错:multi-feature.shape: torch.Size([4, 256, 1152]) x_proj.shape: torch.Size([4, 256, 384]) multi-feature.shape: torch.Size([4, 256, 1152]) x_proj.shape: torch.Size([4, 64, 384]) Training crashed at epoch 1 Traceback for the error! Traceback (most recent call last): File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/train/trainers/base_trainer.py", line 85, in train self.train_epoch() File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/train/trainers/ltr_trainer.py", line 133, in train_epoch self.cycle_dataset(loader) File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/train/trainers/ltr_trainer.py", line 86, in cycle_dataset loss, stats = self.actor(data) File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/train/actors/mamba_fetrack.py", line 31, in call out_dict = self.forward_pass(data) File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/train/actors/mamba_fetrack.py", line 74, in forward_pass out_dict = self.net(template=template_list, File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/models/mamba_fetrack/mamba_fetrack.py", line 54, in forward rgb_feature = self.backbone.forward_features( z=template, x=search, #[B, 320, 384] File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/models/mamba_fetrack/models_mamba.py", line 424, in forward_features z = self.patch_embed(z) #z.shape = torch.Size([B, 3, 128, 128]) -> torch.Size([2, 64, 384]) File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "/data2/local_userdata/heyuting/Mamba/FETrack/Mamba_FETrack/lib/train/../../lib/models/mamba_fetrack/models_mamba.py", line 88, in forward raise ValueError(f"Inconsistent shapes for concatenation: {multi_scale_features.shape[1]} != {x_proj.shape[1]}") ValueError: Inconsistent shapes for concatenation: 256 != 64
Restarting training from last epoch ... Finished training!
- Assistant
The error you're encountering is due to a mismatch in the number of tokens in the multi-scale features and the projected features before concatenation in your
MultiScaleEmbedmodule'sforwardmethod.Here's the specific problematic line in your code:
if multi_scale_features.shape[1] != x_proj.shape[1]: raise ValueError(f"Inconsistent shapes for concatenation: {multi_scale_features.shape[1]} != {x_proj.shape[1]}")Understanding the Problem:
multi_scale_featuresis generated by interpolating several layers' outputs tox_proj's grid size.- However, it's clear from your logs (i.e.,
multi-feature.shape: torch.Size([4, 256, 1152])andx_proj.shape: torch.Size([4, 256, 384])) that the channel dimensions are inconsistent, which prevents concatenation along that dimension.
Possible Reasons and Solutions:
-
Grid Size Mismatch:
- From the shapes, the second dimension mismatch likely comes from the spatial dimensions in the original features being inconsistent even after ensuring the interpolation to match the grid sizes:
- Make sure that
self.grid_size[0]andself.grid_size[1]in both the x and z passes are calculated correctly (bounding box and search image) based on their respective input image sizes.
-
Expected Output:
- You might need a consistent approach to handle both spatial dim differences and feature channel dims as a common guide here is to either average pool, max pool, or align shapes but often resize them accordingly to some expected constant shape.
-
Adjust Output Shapes Post-Concat:
- A common tactic is using extra pooling or upsampling to match these expected dims before the forward function returns the tensor with
aligned_multi_scale_features.shape == x_proj.shape.
- A common tactic is using extra pooling or upsampling to match these expected dims before the forward function returns the tensor with
-
Verify Interpolation Strategy:
- Ensure your multi-scale features are interpolated to the shape
(self.grid_size[0], self.grid_size[1]). Print out the grid sizes for both z and x (if they differ, fix one to match the other consistently).
- Ensure your multi-scale features are interpolated to the shape
-
Example Debugging Step:
print(f"x_proj grid size (expected): {self.grid_size[0]}x{self.grid_size[1]}") print(f"multi_scale_features grid size (received): {multi_scale_features.shape[2]}x{multi_scale_features.shape[3]}")-
Normalization Consistency:
- Ensure that both
x_projandmulti_scale_featuresgo through the same normalization process (if there's any extra scaling or wrapping around norm apply after feature calculation consider shifting them directly).
- Ensure that both
-
Modification Suggestion:
- Fix interpolation logic to a unified grid size defined at the class initialization once.
- Change the concatenation axis or perform any necessary transformations before concatenation to ensure dimensions match.
Applying these tactics above should help resolve the inconsistency in tensor shapes that prevents successful model training without runtime errors.