Skip to content
The Chat Archive
gpt-4o-2024-05-13· OpenAI WildChat-4.8M (AI2)

模型# 将 4输入分开,构建新的相同模态结合的2输入,2分支 import math import logging from functools import partial…

  1. User

    模型# 将 4输入分开,构建新的相同模态结合的2输入,2分支 import math import logging from functools import partial from collections import OrderedDict from copy import deepcopy

    import torch import torch.nn as nn import torch.nn.functional as F

    from timm.models.layers import to_2tuple

    from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise from .utils import combine_tokens, recover_tokens from .vit import VisionTransformer from ..layers.attn_blocks import CEBlock from .new_counter_guide import Counter_Guide from .ad_counter_guide import Counter_Guide_Enhanced from ..mamba_fetrack.models_mamba import VisionMamba, vim_small_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2

    集成CrossMamba模块进行不同模态特征交互

    from ..mamba_fetrack.mamba_cross import CrossMamba _logger = logging.getLogger(name)

    class VisionTransformerCE(VisionTransformer): """ Vision Transformer with candidate elimination (CE) module

    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
        - https://arxiv.org/abs/2010.11929
    
    Includes distillation token & head support for `DeiT: Data-efficient Image Transformers`
        - https://arxiv.org/abs/2012.12877
    """
    def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
                 num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False,
                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None,
                 act_layer=None, weight_init='',
                 ce_loc=None, ce_keep_ratio=None):
        super().__init__()
        if isinstance(img_size, tuple):
            self.img_size = img_size
        else:
            self.img_size = to_2tuple(img_size)
        self.patch_size = patch_size
        self.in_chans = in_chans
    
        self.num_classes = num_classes
        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
        self.num_tokens = 2 if distilled else 1
        norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
        act_layer = act_layer or nn.GELU
    
        self.patch_embed = embed_layer(
            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
        num_patches = self.patch_embed.num_patches
    
        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
        self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None
        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
        self.pos_drop = nn.Dropout(p=drop_rate)
        self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4)
        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
        blocks = []
        ce_index = 0
        self.ce_loc = ce_loc
        for i in range(depth):
            ce_keep_ratio_i = 1.0
            if ce_loc is not None and i in ce_loc:
                ce_keep_ratio_i = ce_keep_ratio[ce_index]
                ce_index += 1
    
            blocks.append(
                CEBlock(
                    dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate,
                    attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer,
                    keep_ratio_search=ce_keep_ratio_i)
            )
    
        self.blocks = nn.Sequential(*blocks)
        self.norm = norm_layer(embed_dim)
    
        self.init_weights(weight_init)
    
        # 添加交互模块counter_guide
        self.counter_guide = Counter_Guide_Enhanced(768, 768)
        # 初始化CrossMamba模块
        self.cross_mamba = CrossMamba(dim=768)
        #########################################
        
    def forward_features(self, z, x, event_z, event_x,
                         mask_z=None, mask_x=None,
                         ce_template_mask=None, ce_keep_rate=None,
                         return_last_attn=False
                         ):
        # 分支1 处理流程
        B, H, W = x.shape[0], x.shape[2], x.shape[3]
    
        x = self.patch_embed(x)
        z = self.patch_embed(z)
        z += self.pos_embed_z
        x += self.pos_embed_x        
    
        if mask_z is not None and mask_x is not None:
            mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_z = mask_z.flatten(1).unsqueeze(-1)
    
            mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_x = mask_x.flatten(1).unsqueeze(-1)
    
            mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode)
            mask_x = mask_x.squeeze(-1)
    
        if self.add_cls_token:
            cls_tokens = self.cls_token.expand(B, -1, -1)
            cls_tokens = cls_tokens + self.cls_pos_embed
        if self.add_sep_seg:
            x += self.search_segment_pos_embed
            z += self.template_segment_pos_embed
        x = combine_tokens(z, x, mode=self.cat_mode)
        if self.add_cls_token:
            x = torch.cat([cls_tokens, x], dim=1)
        x = self.pos_drop(x)
    
        lens_z = self.pos_embed_z.shape[1]
        lens_x = self.pos_embed_x.shape[1]
        global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device)
        global_index_t = global_index_t.repeat(B, 1)
        global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device)
        global_index_s = global_index_s.repeat(B, 1)
        removed_indexes_s = []
        # # 分支2 处理流程
        event_x = self.pos_embed_event(event_x)
        event_z = self.pos_embed_event(event_z)
        event_x += self.pos_embed_x
        event_z += self.pos_embed_z
    
        event_x = combine_tokens(event_z, event_x, mode=self.cat_mode)
    
        if self.add_cls_token:
            event_x = torch.cat([cls_tokens, event_x], dim=1)
    
        lens_z = self.pos_embed_z.shape[1]
        lens_x = self.pos_embed_x.shape[1]
    
        global_index_t1 = torch.linspace(0, lens_z - 1, lens_z).to(event_x.device)
        global_index_t1 = global_index_t1.repeat(B, 1)
        global_index_s1 = torch.linspace(0, lens_x - 1, lens_x).to(event_x.device)
        global_index_s1 = global_index_s1.repeat(B, 1)
    
        removed_indexes_s1 = []
    
        for i, blk in enumerate(self.blocks):
            # 第一个分支处理
            x, global_index_t, global_index_s, removed_index_s, attn = \
                blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate)
            # 第二个分支处理
            event_x, global_index_t1, global_index_s1, removed_index_s1, attn1 = \
                blk(event_x, global_index_t1, global_index_s1, mask_x, ce_template_mask, ce_keep_rate)
    
    
            if self.ce_loc is not None and i in self.ce_loc:
                removed_indexes_s.append(removed_index_s)
                removed_indexes_s1.append(removed_index_s1)
    
            # 在第1层增加counter_guide模块,验证早期融合效果
            if i == 0 :
                # 引入counter_guide进行特征增强
                enhanced_x, enhanced_event_x = self.counter_guide(x, event_x)
                # 将增强后的特征与原特征相加
                x = x + enhanced_x
                event_x = event_x + enhanced_event_x
        ## 引入Crossmamba进行事件和图像特征融合,不采用concatenate
        x = self.norm(x)
        event_x = self.norm(event_x)
        
        residual_rgb_f = 0
        x = self.cross_mamba(x, residual_rgb_f, event_x)
    
    
        aux_dict = {
            "attn": attn,
            'attn1': attn1,
            "removed_indexes_s": removed_indexes_s,  # used for visualization
            'removed_indexes_s1': removed_indexes_s1,
        }
        
        return x, aux_dict 
    
    def forward(self, z, x, event_z, event_x,
                ce_template_mask=None, ce_keep_rate=None,
                tnc_keep_rate=None,
                return_last_attn=False):
    
        x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,)
    
        return x, aux_dict
    

    def _create_vision_transformer(pretrained=False, **kwargs): model = VisionTransformerCE(**kwargs)

    if pretrained:
        if 'npz' in pretrained:
            model.load_pretrained(pretrained, prefix='')
        else:
            checkpoint = torch.load(pretrained, map_location="cpu")
            missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False)
            print('Load pretrained model from: ' + pretrained)
    
    return model
    

    def vit_base_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model

    def vit_large_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model中引入的crossmamba定义是:import math import torch import torch.nn as nn import torch.nn.functional as F from einops import rearrange import numbers from .mamba_cross_simple import Mamba

    将输入张量从(b,c,h,w)重排为(b,h*w,c),常用于将图像张量转换为序列格式

    def to_3d(x): return rearrange(x, 'b c h w -> b (h w) c')

    将序列张量从(b,h*w,c)重排回(b,c,h,w),这是to_3d的反向操作,通常在完成序列处理之后用于恢复原始形状。

    def to_4d(x, h, w): return rearrange(x, 'b (h w) c -> b c h w', h=h, w=w)

    无偏LayerNorm

    实现了不包含偏置项的层标准化,计算输入的方差,用于对输入进行规范化处理,再通过权重进行缩放。

    class BiasFree_LayerNorm(nn.Module): def init(self, normalized_shape): super(BiasFree_LayerNorm, self).init() if isinstance(normalized_shape, numbers.Integral): normalized_shape = (normalized_shape,) normalized_shape = torch.Size(normalized_shape)

        assert len(normalized_shape) == 1
    
        self.weight = nn.Parameter(torch.ones(normalized_shape))
        self.normalized_shape = normalized_shape
    
    def forward(self, x):
        sigma = x.var(-1, keepdim=True, unbiased=False)
        return x / torch.sqrt(sigma+1e-5) * self.weight
    

    包含偏置的层标准化

    class WithBias_LayerNorm(nn.Module): def init(self, normalized_shape): super(WithBias_LayerNorm, self).init() if isinstance(normalized_shape, numbers.Integral): normalized_shape = (normalized_shape,) normalized_shape = torch.Size(normalized_shape)

        assert len(normalized_shape) == 1
    
        self.weight = nn.Parameter(torch.ones(normalized_shape))
        self.bias = nn.Parameter(torch.zeros(normalized_shape))
        self.normalized_shape = normalized_shape
    
    def forward(self, x):
        mu = x.mean(-1, keepdim=True)
        sigma = x.var(-1, keepdim=True, unbiased=False)
        return (x - mu) / torch.sqrt(sigma+1e-5) * self.weight + self.bias
    

    对2种不同的层标准化进行的封装 LayerNorm_type参数决定了使用哪种层标准化

    同时通过先把二维图像数据转换为3维序列,应用层标准化再转换回原来的2维结构。

    class LayerNorm(nn.Module): def init(self, dim, LayerNorm_type): super(LayerNorm, self).init() if LayerNorm_type =='BiasFree': self.body = BiasFree_LayerNorm(dim) else: self.body = WithBias_LayerNorm(dim)

    def forward(self, x):
        h, w = x.shape[-2:]
        return to_4d(self.body(to_3d(x)), h, w)
    

    CrossMamba是Mamba的变体,bimamba_type='V3'指明了具体的Mmaba版本

    self.cross_mamba()用于处理来自两个不同模态的特征,m1和m2分别是两个模态的特征, m1_resi是m1的残差,并输出融合后的全局特征global_f

    self.norm1和self.norm2分别对m1和m2进行层标准化处理(包含偏置)

    self.dwconv是一个深度可分离卷积层,用于局部空间内增强特征表达能力。

    先 to_3d 再 to_4d 操作,避免了直接在多维空间上操作的复杂性,并确保了层标准化可以适用于不同位置的特征向量,保持处理的一致性和独立性。

    class CrossMamba(nn.Module): def init(self, dim): super(CrossMamba, self).init() self.cross_mamba = Mamba(dim,bimamba_type="v3") # 使用v3版本,会使用额外的输入extra_emb进行特殊处理 self.norm1 = LayerNorm(dim,'with_bias') self.norm2 = LayerNorm(dim,'with_bias') self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim) def forward(self,m1,m1_resi,m2):
    # print('m1.shape',m1.shape) # m1.shape torch.Size([4, 320, 384]) # print('m2.shape',m2.shape) # m2.shape torch.Size([4, 320, 384]) # print(m1_resi) # 0 m1_resi = m1+m1_resi
    # print('m1_resi.shape',m1_resi.shape) # m2.shape torch.Size([4, 320, 384])

        global_f = self.cross_mamba(m1, extra_emb=m2)
        # print('global_f.shape',global_f.shape)  # global_f.shape torch.Size([4, 320, 384])
        return global_f,那么直接说明crossmamba的作用是什么?
    
  2. Assistant

    CrossMamba 是一个模块,用于融合来自两个不同模态(例如,图像模态和事件模态)的特征信息,通过特定的机制进行跨模态特征互交。

    它的主要功能和作用包括:

    1. 融合来自不同模态的特征:

      • CrossMamba 模块接受来自两个不同模态的特征(m1 和 m2)以及一个残差(m1_resi)。
      • 通过将模态1的特征m1 和 残差m1_resi 相加来得到融合后的模态1特征,这保证了模态1特征和其残差的直接交互,保持了模态1的稳定性。
      • 使用一个称为 Mamba 的子模块(内部使用一些复杂机制进行跨模态特征的增强和融合),应用在模态1特征和模态2特征上,得到融合后的全局特征 global_f。
    2. 层规范化处理:

      • CrossMamba 内部包含两个 LayerNorm 模块 (self.norm1 和 self.norm2),分别对模态1特征和模态2特征进行层规范化处理,确保特征分布均匀,提升模型的稳定性和训练效果。
      • 这种层规范化是以 "with_bias" 的方式进行的,即包含偏置项的处理方式,可以更加灵活地适应不同特征分布。
    3. 局部空间增强:

      • 还包含一个深度可分离卷积层 (self.dwconv),进一步增强局部特征表达能力。深度可分离卷积有助于减少参数量和计算量,同时捕获更细粒度的空间特征信息,提升模型效率和性能。
    4. 适应不同形状的特征数据:

      • CrossMamba 的实现过程中利用了 Einops 库提供的 rearrange 功能,通过 to_3d 和 to_4d 操作,可以方便地处理输入特征的维度变化。从二维图像张量转换为三维序列格式,再恢复回二维结构。这种处理方式简化了在多维空间上操作的复杂性,确保特征向量处理的一致性。

    综上,CrossMamba 主要作用在于融合来自不同模态的特征,通过层规范化和深度可分离卷积等机制增强特征的表达能力,提升跨模态特征互交的效果,从而提高模型在多模态任务上的表现。

  3. User

    那么既然crossmamba中包含了两个LayerNorm操作,在vit_ce中是否需要在crossmamba引入之前增加两个norm操作?

  4. Assistant

    在 CrossMamba 中已经包含了两个 LayerNorm 操作,对传入的两个模态特征分别进行规范化。因此在引入 CrossMamba 之前再增加类似的 norm 操作,可能会导致重复的规范化处理。

    在 ViT-CE 模型中,如果你观察到在 CrossMamba 引入之前模型已经对传入特征进行了 LayerNorm 规范化,那么在 CrossMamba 中可以直接使用传入的特征而不必再次规范化。这可以通过以下方法实现:

    1. 检查 ViT-CE 中 CrossMamba 引入前的特征处理流程,看是否已经包含 LayerNorm 操作。
    2. 如果已经包含 LayerNorm,则不需要在 CrossMamba 中再重复进行。

    具体实现可以调整为如下:

    class VisionTransformerCE(VisionTransformer):
        ...
    
        def forward_features(self, z, x, event_z, event_x,
                             mask_z=None, mask_x=None,
                             ce_template_mask=None, ce_keep_rate=None,
                             return_last_attn=False):
            # 分支1 处理流程
            B, H, W = x.shape[0], x.shape[2], x.shape[3]
    
            x = self.patch_embed(x)
            z = self.patch_embed(z)
            z += self.pos_embed_z
            x += self.pos_embed_x        
    
            if mask_z is not None and mask_x is not None:
                mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
                mask_z = mask_z.flatten(1).unsqueeze(-1)
    
                mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
                mask_x = mask_x.flatten(1).unsqueeze(-1)
    
                mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode)
                mask_x = mask_x.squeeze(-1)
    
            if self.add_cls_token:
                cls_tokens = self.cls_token.expand(B, -1, -1)
                cls_tokens = cls_tokens + self.cls_pos_embed
            if self.add_sep_seg:
                x += self.search_segment_pos_embed
                z += self.template_segment_pos_embed
            x = combine_tokens(z, x, mode=self.cat_mode)
            if self.add_cls_token:
                x = torch.cat([cls_tokens, x], dim=1)
            x = self.pos_drop(x)
    
            lens_z = self.pos_embed_z.shape[1]
            lens_x = self.pos_embed_x.shape[1]
            global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device)
            global_index_t = global_index_t.repeat(B, 1)
            global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device)
            global_index_s = global_index_s.repeat(B, 1)
            removed_indexes_s = []
            # 分支2 处理流程
            event_x = self.pos_embed_event(event_x)
            event_z = self.pos_embed_event(event_z)
            event_x += self.pos_embed_x
            event_z += self.pos_embed_z
    
            event_x = combine_tokens(event_z, event_x, mode=self.cat_mode)
    
            if self.add_cls_token:
                event_x = torch.cat([cls_tokens, event_x], dim=1)
    
            lens_z = self.pos_embed_z.shape[1]
            lens_x = self.pos_embed_x.shape[1]
    
            global_index_t1 = torch.linspace(0, lens_z - 1, lens_z).to(event_x.device)
            global_index_t1 = global_index_t1.repeat(B, 1)
            global_index_s1 = torch.linspace(0, lens_x - 1, lens_x).to(event_x.device)
            global_index_s1 = global_index_s1.repeat(B, 1)
    
            removed_indexes_s1 = []
    
            for i, blk in enumerate(self.blocks):
                # 第一个分支处理
                x, global_index_t, global_index_s, removed_index_s, attn = \
                    blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate)
                # 第二个分支处理
                event_x, global_index_t1, global_index_s1, removed_index_s1, attn1 = \
                    blk(event_x, global_index_t1, global_index_s1, mask_x, ce_template_mask, ce_keep_rate)
    
                if self.ce_loc is not None and i in self.ce_loc:
                    removed_indexes_s.append(removed_index_s)
                    removed_indexes_s1.append(removed_index_s1)
    
                if i == 0:
                    enhanced_x, enhanced_event_x = self.counter_guide(x, event_x)
                    x = x + enhanced_x
                    event_x = event_x + enhanced_event_x
            
            # 交叉模态融合前的规范化 (如果必要的话,可以添加这段代码)
            # x = self.norm(x)
            # event_x = self.norm(event_x)
            
            # Crossover module
            x = self.cross_mamba(x, 0, event_x)
    
            aux_dict = {
                "attn": attn,
                'attn1': attn1,
                "removed_indexes_s": removed_indexes_s,  # used for visualization
                'removed_indexes_s1': removed_indexes_s1,
            }
            
            return x, aux_dict
    

    只需要在 CrossMamba 的 forward 中加入 norm 操作即可,避免重复的 LayerNorm 操作,以确保特征的一致性和减少不必要的计算。

  5. User

    在引入crossmamba之前训练配置是:DATA: MAX_SAMPLE_INTERVAL: 100 MEAN:

    • 0.485
    • 0.456
    • 0.406 SEARCH: CENTER_JITTER: 3 FACTOR: 4.0 SCALE_JITTER: 0.25 SIZE: 256 NUMBER: 1 STD:
    • 0.229
    • 0.224
    • 0.225 TEMPLATE: CENTER_JITTER: 0 FACTOR: 2.0 SCALE_JITTER: 0 SIZE: 128 TRAIN: DATASETS_NAME:
      • COESOT DATASETS_RATIO:
      • 1 SAMPLE_PER_EPOCH: 60000 VAL: DATASETS_NAME:
      • COESOT_VAL DATASETS_RATIO:
      • 1 SAMPLE_PER_EPOCH: 5000 MODEL: PRETRAIN_FILE: "mae_pretrain_vit_base.pth" EXTRA_MERGER: False RETURN_INTER: False BACKBONE: TYPE: vit_base_patch16_224_ce STRIDE: 16 CE_LOC: [3, 6, 9] CE_KEEP_RATIO: [1, 1, 1] CE_TEMPLATE_RANGE: 'CTR_POINT' HEAD: TYPE: CENTER NUM_CHANNELS: 256 TRAIN: BACKBONE_MULTIPLIER: 0.1 DROP_PATH_RATE: 0.1 CE_START_EPOCH: 0 # candidate elimination start epoch 20 CE_WARM_EPOCH: 0 # candidate elimination warm up epoch 50 BATCH_SIZE: 16 EPOCH: 50 GIOU_WEIGHT: 1 # 2.0 L1_WEIGHT: 14 # 5.0 FOCAL_WEIGHT: 1.0 # 1.5 GRAD_CLIP_NORM: 0.1 LR: 0.0001 LR_DROP_EPOCH: 40 NUM_WORKER: 8 OPTIMIZER: ADAMW PRINT_INTERVAL: 50 SCHEDULER: TYPE: step DECAY_RATE: 0.1

    TYPE: Mstep

    MILESTONES: [40, 60]

    GAMMA: 0.1

    VAL_EPOCH_INTERVAL: 1 WEIGHT_DECAY: 0.0001 AMP: False TEST: EPOCH: 50 SEARCH_FACTOR: 4.0 SEARCH_SIZE: 256 TEMPLATE_FACTOR: 2.0 TEMPLATE_SIZE: 128,那么引入crossmamba之后,是否需要对训练配置进行修改?

  6. Assistant

    在引入 CrossMamba 模块之后,执行跨模态特征融合时,可能需要调整一些训练配置,以确保新模块的有效性和训练过程的稳定性。然而,是否需要进行配置修改以及修改的具体内容可能取决于以下几个因素:

    1. 模型复杂度: 引入 CrossMamba 会增加模型的复杂度。可以适当调整学习率、优化器参数、和调度器策略,以适应新增模块。

    2. 正则化和数据增广: 考虑加强正则化策略(如权重衰减)和数据增广措施,以防止模型过拟合。

    3. 超参数调整: 包括批次大小、训练次数、损失函数的权重等,在引入新模块后可以进行重新调整和验证。

    以下是可能需要调整的配置项:

    1. 学习率及其调度器: 随着模型复杂度的增加,适当的学习率调整(如减小初始学习率)和调度策略可能有助于稳定训练。

    2. 正则化策略: 更新 WEIGHT_DECAY,增加正则化效果,避免过拟合。

    3. 数据集平衡与采样策略: 确保新的特征融合后,各模态数据能够均衡地参与训练,适当调整 DATASETS_RATIO。

    4. 损失函数的权重: 根据新的特征融合方式,重新权衡不同损失的权重。

    以下示例展示了可能的配置调整:

    DATA:
      MAX_SAMPLE_INTERVAL: 100
      MEAN:
      - 0.485
      - 0.456
      - 0.406
      SEARCH:
        CENTER_JITTER: 3
        FACTOR: 4.0
        SCALE_JITTER: 0.25
        SIZE: 256
        NUMBER: 1
      STD:
      - 0.229
      - 0.224
      - 0.225
      TEMPLATE:
        CENTER_JITTER: 0
        FACTOR: 2.0
        SCALE_JITTER: 0
        SIZE: 128
      TRAIN:
        DATASETS_NAME:
        - COESOT
        DATASETS_RATIO:  # 确保比例适当
        - 1
        SAMPLE_PER_EPOCH: 60000
      VAL:
        DATASETS_NAME:
        - COESOT_VAL
        DATASETS_RATIO:
        - 1
        SAMPLE_PER_EPOCH: 5000
    MODEL:
      PRETRAIN_FILE: "mae_pretrain_vit_base.pth"
      EXTRA_MERGER: False
      RETURN_INTER: False
      BACKBONE:
        TYPE: vit_base_patch16_224_ce
        STRIDE: 16
        CE_LOC: [3, 6, 9]
        CE_KEEP_RATIO: [1, 1, 1]
        CE_TEMPLATE_RANGE: 'CTR_POINT'
      HEAD:
        TYPE: CENTER
        NUM_CHANNELS: 256
    TRAIN:
      BACKBONE_MULTIPLIER: 0.1
      DROP_PATH_RATE: 0.1
      CE_START_EPOCH: 0  # candidate elimination start epoch  20
      CE_WARM_EPOCH: 0  # candidate elimination warm up epoch 50
      BATCH_SIZE: 16
      EPOCH: 50
      GIOU_WEIGHT: 1     # 2.0
      L1_WEIGHT: 14       # 5.0
      FOCAL_WEIGHT: 1.0   # 1.5
      GRAD_CLIP_NORM: 0.1
      LR: 0.00005  # 降低学习率以适应新模块
      LR_DROP_EPOCH: 40
      NUM_WORKER: 8
      OPTIMIZER: ADAMW
      PRINT_INTERVAL: 50
      SCHEDULER:
        TYPE: step
        DECAY_RATE: 0.1
      VAL_EPOCH_INTERVAL: 1
      WEIGHT_DECAY: 0.0001  # 增加或保持正则化
      AMP: False
    TEST:
      EPOCH: 50
      SEARCH_FACTOR: 4.0
      SEARCH_SIZE: 256
      TEMPLATE_FACTOR: 2.0
      TEMPLATE_SIZE: 128
    

    具体修改需要根据以下步骤进行实验验证和调整:

    1. 初实验: 采用推荐配置进行初实验,观察损失变化和训练稳定性。

    2. 监控指标: 关注验证集上的指标变化,如果发现过拟合或欠拟合,适当调整学习率和正则化参数。

    3. 微调策略: 根据初实验结果进行微调,特别是学习率、权重衰减、批次大小、和损失函数权重等超参数。

    总之,引入 CrossMamba 后,合理的训练配置调整是确保新模块有效融合进整体模型的关键,需要在实验过程中不断优化。

  7. User

    为什么降低学习率,原理是什么?

  8. Assistant

    降低学习率在引入新模块(如 CrossMamba)后是一个常见的实验策略,主要原因如下:

    1. 模型复杂度增加:

      • 新模块的引入增加了网络的复杂度和参数量,较高的学习率可能会导致训练不稳定,参数更新过大,容易导致训练过程中的数值振荡或不收敛。
    2. 梯度更新的稳健性:

      • 降低学习率可以使梯度更新更加平滑,这对于有新模块的复杂网络结构尤其重要。较低的学习率可以使模型在参数空间中更稳定地找到收敛路径,避免跳过全局最优解的情况。
    3. 避免过拟合:

      • 新模块的增加使模型表达能力增强,但也带来过拟合风险。降低学习率可以延缓模型过快的学习速度,使模型逐步学到更稳健的特征,而不是快速拟合到训练数据的噪声上。
    4. 初期适应性:

      • 在网络一开始训练的时候可选择较高的学习率进行快速学习,但随着新模块的引入特别是在整体模型结构调整之后,推荐降低学习率让模型有一个适应和缓和的过程,这更有助于新的网络部分与已有部分更好地融合。

    具体应用实例

    假设原学习率是 1e-4,在引入 CrossMamba 之后,建议降低学习率至 5e-5 或 1e-5。这种减半或十倍减小是实践中常见的策略。

    学习率调整的示例步骤:

    1. 初步实验:使用降低后的学习率(例如从 1e-4 降低到 5e-5),观察损失函数变化、精度、和验证集的性能。

    2. 观察与调整:

      • 检查训练曲线和验证集上的性能指标,确保没有数值振荡,收敛速度相对平稳。
      • 如果发现训练速度过慢且没有收敛,可以适度提高学习率,同时注意监控过拟合现象。
    3. 逐步微调:在确定合理范围后,可以在前期用稍大的学习率进行快速收敛,到后期逐步减小学习率,调度策略可以使用预定义的 Scheduler,如 step-decay, cosine decay 等。

    4. 正则化处理:结合正则化(如 weight decay)和梯度剪裁(如 GRAD_CLIP_NORM),确保梯度更新的稳健性,特别在复杂模型结构下,避免梯度爆炸和过拟合。

    修改后的配置示例

    TRAIN:
      BACKBONE_MULTIPLIER: 0.1
      DROP_PATH_RATE: 0.1
      CE_START_EPOCH: 0  # candidate elimination start epoch  20
      CE_WARM_EPOCH: 0  # candidate elimination warm up epoch 50
      BATCH_SIZE: 16
      EPOCH: 50
      GIOU_WEIGHT: 1     # 2.0
      L1_WEIGHT: 14       # 5.0
      FOCAL_WEIGHT: 1.0   # 1.5
      GRAD_CLIP_NORM: 0.1
      LR: 0.00005  # 降低学习率以适应新模块
      LR_DROP_EPOCH: 40
      NUM_WORKER: 8
      OPTIMIZER: ADAMW
      PRINT_INTERVAL: 50
      SCHEDULER:
        TYPE: step
        DECAY_RATE: 0.1
      VAL_EPOCH_INTERVAL: 1
      WEIGHT_DECAY: 0.0001  # 增加或保持正则化
      AMP: False
    

    学习率调度器

    可以进一步配置学习率调度器,使其动态调整学习率,例如:

    SCHEDULER:
      TYPE: CosineAnnealingLR
      T_max: 10  # 每10个 epoch 完成一个学习率循环
      eta_min: 1e-5
    

    通过降低初始学习率,结合学习率调度器和正则化策略,可以帮助你在引入 CrossMamba 模块之后有效地稳定训练过程,提高模型性能。

  9. User

    翻译并总结:Visual tracking is a widely exploited research topic in computer vision that aims to predict the locations of the target object in the subsequent video frames given its initial localization. It has been widely used in many practical scenarios, including smart video surveillance, UAVs, autonomous vehicles, military fields, etc. Current tracking algorithms are usually developed based on RGB frames captured using the visible camera and the overall tracking performance has been boosted by deep learning techniques greatly in recent years. These deep learning based trackers can be divided into four main streams, including proposal classification based tracking (e.g., MDNet~\cite{Nam2015LearningMC}, RT-MDNet~\cite{jung2018real}), Siamese network based tracking (e.g., SiamFC~\cite{bertinetto2021fullyconvolutional}, SiamRPN++\cite{xu2020siamfc++}, SINT\cite{Tao2016SiameseIS}, SINT++\cite{Wang2018SINTRV}), Transformer network based tracking (e.g., TransT\cite{Chen2021TransformerT}, STARK~\cite{Yan2021LearningST}, MixFormer~\cite{Cui2022MixFormerET}, AiATrack~\cite{gao2022aiatrack} ). There are also some works that attempt to address the issues caused by the local search strategy used in most tracking frameworks, such as TANet~\cite{liu2020tanet}, DeepMTA~\cite{yang2020interpretable}, and GlobalTrack~\cite{huang2020globaltrack}. Although the trackers already achieved good performance on existing benchmark datasets, the tracking results under extremely challenging scenarios (e.g., fast motion and low illumination) are still far from satisfying.

    To further improve the tracking performance in challenging scenarios, this paper resorts to the newly developed Event cameras (e.g., DVS346, Prophesee, CeleX-V) to provide the external cues. Event camera can capture motion cues well due to their unique imaging principle. To be specific, the Event camera asynchronously records the pixels and it records the pixel only when its variation of light intensity exceeds a given threshold. Event camera also performs better than the visible camera on high dynamic range, low energy consumption, low data storage, etc~\cite{Gallego2019EventBasedVA}. However, the Event camera captures nothing or very few Event points when the target object is motionless. It also fails to record the color or detailed texture information of the tracked target object. Thanks to the good results provided by the visible camera in these cases, we can combine the two cameras to achieve robust and high-performance tracking.

    Based on this idea, some RGB-Event visual trackers have already been proposed in recent years. Specifically, the CMT~\cite{Wang2021VisEventRO} proposed by Wang et al. introduces a new cross-modality Transformer module for the fusion of RGB and Event data. Tang et al.\cite{tang2022coesot} propose a unified adapter-based Transformer network as their backbone which achieves feature extraction, fusion, and tracking simultaneously. Zhang et al.\cite{Zhang2022SpikingTF} propose a hybrid SNN-Transformer framework for energy-efficient visual tracking. Although these works achieve good results on existing benchmark datasets, however, these tracking algorithms are still limited by the following issues: \textbf{1).} These trackers are all developed based on the Transformer network whose computational complexity is very high, as the complexity of the attention computation is O(N2)\mathcal{O}(N^2). \textbf{2).} The memory requirement on the GPU is very high in the training and tracking phase, as the KV cache needs to be stored for subsequent processing. Obviously, the tracking speed of these trackers is limited due to the aforementioned issues. Thus, it is natural to raise the following question: \textit{How can we design a new RGB-Event tracking framework that achieves a good balance between high-performance and low model complexity?}

    Recently, the state space model (SSM) has drawn more and more attention in the artificial intelligence community, especially in natural language processing and computer vision~\cite{wang2024SSMSurvey}. The SSM is an attention-free model with the complexity O(N)\mathcal{O}(N). On the other hand, current SSM based vision models~\cite{Nguyen2022S4NDMI, Smith2022SimplifiedSS, Liu2024VMambaVS,zhu2024vision} also demonstrate that they can achieve similar or even higher performance on large-scale classification tasks, such as ImageNet-1K~\cite{Deng2009ImageNetAL}. The FLOPs and memory requirement are also decreased significantly compared with the Transformer based networks. It has been widely used in many domains and tasks, such as RGB/Video-based recognition~\cite{Li2024MambaNDSS}, segmentation~\cite{Xing2024SegMambaLS,Ma2024UMambaEL,Ruan2024VMUNetVM}, graph-based modeling~\cite{tang2023modeling,Wang2024GraphMambaTL,Behrouz2024GraphMT}, point cloud~\cite{Liang2024PointMambaAS,Zhang2024PointCM,Liu2024PointMA}, Event stream~\cite{Zubic2024StateSM}, time series data~\cite{islam2022long,Wang2023SelectiveSS}, etc. However, the performance of the state space model based RGB-Event tracking task is still unknown. In this paper, we propose a novel RGB-Event tracking framework based on the State Space Model (SSM) to replace the Transformer network. For the details of our framework, it takes the RGB frames and Event stream as the input, then, two modality-specific Mamba backbone networks are designed to encode the RGB and Event inputs. Also, we introduce the Mamba to boost the interactive learning between dual modalities. The fused features will be fed into the tracking head for target object localization. As shown in Fig.\ref{fig:ComparisonParams}, our Mamba-FETrack achieves much lower FLOPs and parameters, meanwhile, maintaining comparable or even better results than the ViT-B and ViT-S based OSTrack RGB-Event tracker. An overview of our proposed Mamba based RGB-Event tracking framework is illustrated in Fig.\ref{fig:framework}.

    To sum up, we draw the contributions of this paper as the following three aspects:

    1). We propose the \textit{first} state space model based RGB-Event tracking framework which achieves a good tradeoff between high performance and low complexity, termed Mamba-FETrack. It improves the GPU memory cost, FLOPs, and parameters of ViT-S based OSTrack (RGB-Event version) by 9.5%9.5\%, 94.5%94.5\% and 88.3%88.3\%, respectively, meanwhile, exceeds this tracker on tracking performance.

    2). We introduce a simple but effective \textit{FusionMamba block} which boosts the interactive feature learning between the RGB and Event features effectively.

    3). We conduct extensive experiments on two large-scale RGB-Event tracking datasets (i.e., FELT~\cite{wang2024longterm}, and FE108~\cite{Zhang2021ObjectTB}) which fully validated the effectiveness of our proposed Mamba based RGB-Event tracker.

  10. Assistant

    摘要和总结

    背景和现状: 视觉跟踪(Visual Tracking)是计算机视觉中的一个重要研究领域,旨在给定目标物体的初始位置,预测其在后续视频帧中的位置。这在智能视频监控、无人机、自动驾驶和军事领域等实际场景中有广泛应用。当前的跟踪算法主要基于使用普通相机捕获的RGB帧,近年来深度学习技术极大地提升了整体跟踪性能。这些基于深度学习的跟踪器可以分为四大类:

    1. 提议分类跟踪(如MDNet, RT-MDNet)
    2. Siamese网络跟踪(如SiamFC, SiamRPN++, SINT, SINT++)
    3. Transformer网络跟踪(如TransT, STARK, MixFormer, AiATrack)
    4. 解决局部搜索策略问题的工作(如TANet, DeepMTA, GlobalTrack)

    尽管这些跟踪器在现有基准数据集上已经取得了不错的性能,但在极具挑战性的场景(如快速运动和低照度)下,跟踪结果仍不理想。

    引入Event相机: 为了解决这些问题,本文引入了新开发的Event相机(如DVS346, Prophesee, CeleX-V),提供额外的运动线索。Event相机由于其独特的成像原理,可以更好地捕捉到运动线索。尽管Event相机在高动态范围、低能耗和低数据存储方面表现优于普通可见光相机,但在目标物体静止时几乎捕捉不到任何信息,也无法记录颜色或细节纹理。通过结合可见光相机,能够实现更加鲁棒和高性能的跟踪。

    现有RGB-Event视觉跟踪工作: 近年来,一些RGB-Event视觉跟踪器已经被提出:

    • CMT: 提出了一种跨模态Transformer模块,用于融合RGB和Event数据。
    • CoESOT: 采用了基于Transformer的框架,实现特征提取、融合和跟踪。
    • Hybrid SNN-Transformer框架: 用于节能型视觉跟踪。

    尽管这些工作在现有基准数据集上取得了不错的结果,但仍存在以下问题:

    1. 这些跟踪器都基于计算量很大的Transformer网络。
    2. 在训练和跟踪阶段,KV缓存的存储需求很高,导致跟踪速度受到限制。

    基于此,本文提出了一个新的问题:如何设计一个新的RGB-Event跟踪框架,在实现高性能的同时保持低复杂度?

    提出的新方法: 本文提出了一个新的基于State Space Model (SSM)的RGB-Event跟踪框架,替代Transformer网络。本文的具体贡献有三方面:

    1. 提出了第一个基于SSM的RGB-Event跟踪框架Mamba-FETrack,实现了高性能和低复杂度的良好平衡。与基于ViT-S的OSTrack (RGB-Event版本)相比,GPU内存成本、FLOPs和参数分别降低了9.5%、94.5%9.5\%、94.5\%和88.3%88.3\%,同时性能超越该跟踪器。
    2. 引入了一个简单但有效的FusionMamba block,有效地增强了RGB和Event特征之间的交互学习。
    3. 在两个大规模RGB-Event跟踪数据集(FELT和FE108)上进行了广泛的实验,验证了所提出的Mamba-FETrack的有效性。

    总结 本文提出并验证了一种基于SSM的RGB-Event跟踪框架,大大降低了复杂度和计算量,同时在多个基准数据集上取得了优异的性能。通过引入FusionMamba block增强了不同模态特征的交互学习能力,为未来的多模态跟踪算法提供了新的思路和方向。