在vitce中,设置了import math import logging from functools import partial from collections…
- User
在vit_ce中,设置了import math import logging from functools import partial from collections import OrderedDict from copy import deepcopy
import torch import torch.nn as nn import torch.nn.functional as F
from timm.models.layers import to_2tuple
from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise from .utils import combine_tokens, recover_tokens from .vit import VisionTransformer from ..layers.attn_blocks import CEBlock
_logger = logging.getLogger(name)
class VisionTransformerCE(VisionTransformer): """ Vision Transformer with candidate elimination (CE) module
A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` - https://arxiv.org/abs/2010.11929 Includes distillation token & head support for `DeiT: Data-efficient Image Transformers` - https://arxiv.org/abs/2012.12877 """ def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None, act_layer=None, weight_init='', ce_loc=None, ce_keep_ratio=None): """ Args: img_size (int, tuple): input image size patch_size (int, tuple): patch size in_chans (int): number of input channels num_classes (int): number of classes for classification head embed_dim (int): embedding dimension depth (int): depth of transformer num_heads (int): number of attention heads mlp_ratio (int): ratio of mlp hidden dim to embedding dim qkv_bias (bool): enable bias for qkv if True representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set distilled (bool): model includes a distillation token and head as in DeiT models drop_rate (float): dropout rate attn_drop_rate (float): attention dropout rate drop_path_rate (float): stochastic depth rate embed_layer (nn.Module): patch embedding layer norm_layer: (nn.Module): normalization layer weight_init: (str): weight init scheme """ # super().__init__() super().__init__() if isinstance(img_size, tuple): self.img_size = img_size else: self.img_size = to_2tuple(img_size) self.patch_size = patch_size self.in_chans = in_chans self.num_classes = num_classes self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models self.num_tokens = 2 if distilled else 1 norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) act_layer = act_layer or nn.GELU self.patch_embed = embed_layer( img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4) # self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4) # self.pos_embed_event_z = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=3, stride=1) # attn = CrossAttn(768, 4, 3072, 0.1, 'relu') # self.cross_attn = Iter_attn(attn, 2) dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule blocks = [] ce_index = 0 self.ce_loc = ce_loc for i in range(depth): ce_keep_ratio_i = 1.0 if ce_loc is not None and i in ce_loc: ce_keep_ratio_i = ce_keep_ratio[ce_index] ce_index += 1 blocks.append( CEBlock( dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer, keep_ratio_search=ce_keep_ratio_i) ) self.blocks = nn.Sequential(*blocks) self.norm = norm_layer(embed_dim) self.init_weights(weight_init) def forward_features(self, z, x, event_z, event_x, mask_z=None, mask_x=None, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False ): B, H, W = x.shape[0], x.shape[2], x.shape[3] event_z = self.pos_embed_event(event_z) # [:,:,:,:1000] event_x = self.pos_embed_event(event_x) # B 768 1024 x = self.patch_embed(x) z = self.patch_embed(z) event_z += self.pos_embed_z event_x += self.pos_embed_x z += self.pos_embed_z x += self.pos_embed_x # attention mask handling # B, H, W if mask_z is not None and mask_x is not None: mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_z = mask_z.flatten(1).unsqueeze(-1) mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_x = mask_x.flatten(1).unsqueeze(-1) mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode) mask_x = mask_x.squeeze(-1) if self.add_cls_token: cls_tokens = self.cls_token.expand(B, -1, -1) cls_tokens = cls_tokens + self.cls_pos_embed if self.add_sep_seg: x += self.search_segment_pos_embed z += self.template_segment_pos_embed x = combine_tokens(z, event_z, x, event_x, mode=self.cat_mode) # 64+64+256+256=640 # x = combine_tokens(z, x, event_z, event_x, mode=self.cat_mode) # 64+64+256+256=640 if self.add_cls_token: x = torch.cat([cls_tokens, x], dim=1) x = self.pos_drop(x) lens_z = self.pos_embed_z.shape[1] lens_x = self.pos_embed_x.shape[1] global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device) global_index_t = global_index_t.repeat(B, 1) global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device) global_index_s = global_index_s.repeat(B, 1) removed_indexes_s = [] for i, blk in enumerate(self.blocks): x, global_index_t, global_index_s, removed_index_s, attn = \ blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate) if self.ce_loc is not None and i in self.ce_loc: removed_indexes_s.append(removed_index_s) x = self.norm(x) lens_x_new = global_index_s.shape[1] lens_z_new = global_index_t.shape[1] z = x[:, :lens_z_new*2] x = x[:, lens_z_new*2:] if removed_indexes_s and removed_indexes_s[0] is not None: removed_indexes_cat = torch.cat(removed_indexes_s, dim=1) pruned_lens_x = lens_x - lens_x_new pad_x = torch.zeros([B, pruned_lens_x, x.shape[2]], device=x.device) x = torch.cat([x, pad_x], dim=1) index_all = torch.cat([global_index_s, removed_indexes_cat], dim=1) # recover original token order C = x.shape[-1] x = torch.zeros_like(x).scatter_(dim=1, index=index_all.unsqueeze(-1).expand(B, -1, C).to(torch.int64), src=x) x = recover_tokens(x, lens_z_new, lens_x, mode=self.cat_mode) x = x[:, :lens_x] # RGB head x = torch.cat([event_x, x], dim=1) # x = x[:, lens_x//2:] # event head # x = torch.cat([z, x], dim=1) # re-concatenate with the template, which may be further used by other modules # x, event_x = x[:, :lens_x//2], x[:, lens_x//2:] # x = x[:, -lens_x//2:] aux_dict = { "attn": attn, "removed_indexes_s": removed_indexes_s, # used for visualization } return x, aux_dict def forward(self, z, x, event_z, event_x, ce_template_mask=None, ce_keep_rate=None, tnc_keep_rate=None, return_last_attn=False): x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,) return x, aux_dictdef _create_vision_transformer(pretrained=False, **kwargs): model = VisionTransformerCE(**kwargs)
if pretrained: if 'npz' in pretrained: model.load_pretrained(pretrained, prefix='') else: checkpoint = torch.load(pretrained, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False) print('Load pretrained model from: ' + pretrained) return modeldef vit_base_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model
def vit_large_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model,然后在""" Basic ceutrack model. """ import math import os from typing import List
import torch from torch import nn from torch.nn.modules.transformer import _get_clones
from lib.models.layers.head import build_box_head from lib.models.ceutrack.vit import vit_base_patch16_224 from lib.models.ceutrack.vit_ce import vit_large_patch16_224_ce, vit_base_patch16_224_ce
from lib.models.ceutrack.vit_ce_BACKUPS import vit_large_patch16_224_ce, vit_base_patch16_224_ce
from lib.utils.box_ops import box_xyxy_to_cxcywh
class CEUTrack(nn.Module): """ This is the base class for ceutrack """
def __init__(self, transformer, box_head, aux_loss=False, head_type="CORNER"): """ Initializes the model. Parameters: transformer: torch module of the transformer architecture. aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. """ super().__init__() self.backbone = transformer self.box_head = box_head self.aux_loss = aux_loss self.head_type = head_type if head_type == "CORNER" or head_type == "CENTER": self.feat_sz_s = int(box_head.feat_sz) self.feat_len_s = int(box_head.feat_sz ** 2) if self.aux_loss: self.box_head = _get_clones(self.box_head, 6) def forward(self, template: torch.Tensor, search: torch.Tensor, event_template: torch.Tensor, # torch.Size([4, 1, 19, 10000]) event_search: torch.Tensor, # torch.Size([4, 1, 19, 10000]) ce_template_mask=None, ce_keep_rate=None, return_last_attn=False, ): # before feeding into backbone, we need to concat four vectors, or two two concat; x, aux_dict = self.backbone(z=template, x=search, event_z=event_template, event_x=event_search, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate, return_last_attn=return_last_attn, ) # Forward head feat_last = x if isinstance(x, list): feat_last = x[-1] out = self.forward_head(feat_last, None) out.update(aux_dict) out['backbone_feat'] = x return out def forward_head(self, cat_feature, gt_score_map=None): """ cat_feature: output embeddings of the backbone, it can be (HW1+HW2, B, C) or (HW2, B, C) """ # output the last 256 # enc_opt = cat_feature[:, -self.feat_len_s:] # encoder output for the search region (B, HW, C) 768*256 # opt = (enc_opt.unsqueeze(-1)).permute((0, 3, 2, 1)).contiguous() # bs, Nq, C, HW = opt.size() # opt_feat = opt.view(-1, C, self.feat_sz_s, self.feat_sz_s) ## dual head 768+768)*256 enc_opt1 = cat_feature[:, -self.feat_len_s:] enc_opt2 = cat_feature[:, :self.feat_len_s] enc_opt = torch.cat([enc_opt1, enc_opt2], dim=-1) opt = (enc_opt.unsqueeze(-1)).permute((0, 3, 2, 1)).contiguous() bs, Nq, C, HW = opt.size() opt_feat = opt.view(-1, C, self.feat_sz_s, self.feat_sz_s) if self.head_type == "CORNER": # run the corner head pred_box, score_map = self.box_head(opt_feat, True) outputs_coord = box_xyxy_to_cxcywh(pred_box) outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map, } return out elif self.head_type == "CENTER": # run the center head score_map_ctr, bbox, size_map, offset_map = self.box_head(opt_feat, gt_score_map) # outputs_coord = box_xyxy_to_cxcywh(bbox) outputs_coord = bbox outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map_ctr, 'size_map': size_map, 'offset_map': offset_map} return out else: raise NotImplementedErrordef build_ceutrack(cfg, training=True): current_dir = os.path.dirname(os.path.abspath(file)) # This is your Project Root pretrained_path = os.path.join(current_dir, '../../../pretrained_models') if cfg.MODEL.PRETRAIN_FILE and ('CEUTrack' not in cfg.MODEL.PRETRAIN_FILE) and training: pretrained = os.path.join(pretrained_path, cfg.MODEL.PRETRAIN_FILE) else: pretrained = ''
if cfg.MODEL.BACKBONE.TYPE == 'vit_base_patch16_224': backbone = vit_base_patch16_224(pretrained, drop_path_rate=cfg.TRAIN.DROP_PATH_RATE) hidden_dim = backbone.embed_dim patch_start_index = 1 elif cfg.MODEL.BACKBONE.TYPE == 'vit_base_patch16_224_ce': backbone = vit_base_patch16_224_ce(pretrained, drop_path_rate=cfg.TRAIN.DROP_PATH_RATE, ce_loc=cfg.MODEL.BACKBONE.CE_LOC, ce_keep_ratio=cfg.MODEL.BACKBONE.CE_KEEP_RATIO, ) # hidden_dim = backbone.embed_dim hidden_dim = backbone.embed_dim * 2 patch_start_index = 1 elif cfg.MODEL.BACKBONE.TYPE == 'vit_large_patch16_224_ce': backbone = vit_large_patch16_224_ce(pretrained, drop_path_rate=cfg.TRAIN.DROP_PATH_RATE, ce_loc=cfg.MODEL.BACKBONE.CE_LOC, ce_keep_ratio=cfg.MODEL.BACKBONE.CE_KEEP_RATIO, ) hidden_dim = backbone.embed_dim patch_start_index = 1 else: raise NotImplementedError backbone.finetune_track(cfg=cfg, patch_start_index=patch_start_index) box_head = build_box_head(cfg, hidden_dim) model = CEUTrack( backbone, box_head, aux_loss=False, head_type=cfg.MODEL.HEAD.TYPE, ) if 'CEUTrack' in cfg.MODEL.PRETRAIN_FILE and training: checkpoint = torch.load(cfg.MODEL.PRETRAIN_FILE, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["net"], strict=False) print('Load pretrained model from: ' + cfg.MODEL.PRETRAIN_FILE) return model中采用了vit_ce_base,那么我们可以将其替换为PVT吗? - Assistant
是的,您可以尝试将Vision Transformer (ViT) 模型替换为Pyramid Vision Transformer (PVT) 来用于CEUTrack模型中。PVT作为一种分层的Transformer架构,它将图像划分为多尺度的patch,并通过Transformer编码器处理这些来自不同尺度的patch,从而能更好地捕获不同尺度的特征信息。这种能力可能对于各种视觉任务,包括视觉跟踪,是有益的。
进行这种替换需要以下几步:
-
引入PVT模型: 首先,您需要确保您有PVT的PyTorch实现。这可能需要您从相关的论文或公开的代码库中获取PVT模型的实现。
-
定义PVT作为骨干网络: 替换CEUTrack中的
backbone部分,使用PVT模型作为新的骨干网络。您需要调整代码中的build_ceutrack函数,以创建PVT模型实例并用作骨干网络。同时,确保PVT的输出维度与原始ViT模型对齐,或者您需要相应地调整下游任务的输入维度。 -
适应性改变: PVT由于其多尺度特性,可能需要对原模型的某些部分进行调整。例如,可能需要特别注意如何合并不同尺度的特征,以及如何处理自注意力模块的输入输出,以确保信息能从不同层和尺度有效融合。
-
调参与优化: 替换骨干网络后需要重新调整超参数和优化设置。PVT和ViT在行为上可能存在差异,因此为了最大化新模型的性能,你可能需要尝试不同的学习率、权重衰减等超参数。
-
实验与评估: 在完成以上步骤后,需要进行大量的实验以评估新模型的性能。对于视觉跟踪任务,这包括但不限于跟踪精度、鲁棒性和速度。实验结果将帮助您了解替换骨干网络是否有益于模型的整体性能。
请记住,任何架构更改都需要深入理解原始模型的工作机制以及新模型的特性。此外,处理各种兼容性问题可能需要时间和耐心。
-
- User
PVT是如何构建多尺度的?
- Assistant
Pyramid Vision Transformer (PVT) 是一种转换器结构,专为视觉任务设计,具有多尺度处理的能力。PVT的核心理念在于,和传统的Transformer模型(如Vision Transformer,ViT)在单一尺度上处理图像不同,PVT采用分层的方法逐渐减小特征图的空间维度(即尺寸),同时增加其通道数,实现了对不同尺度特征的提取。这种设计使得PVT能够更有效地捕获从局部到整体的信息,适应不同大小的物体和场景的识别任务。下面是PVT构建多尺度特征的基本方式:
-
分层设计:PVT被设计为多个阶段,每个阶段都处理不同尺度的输入。通常在进入下一阶段之前,特征的空间分辨率会降低,而特征的通道数会增加。这种设计模仿了CNN中常见的金字塔结构。
-
分块(Patch):在每个阶段开始时,PVT将输入的特征图(对于第一阶段为原始图像)划分为更小的块(或称“Patch”)。每个块通过一个线性层或卷积层被映射(或“嵌入”)到一个较高维度的特征空间。
-
逐层下采样:在传统的Transformer模型中,每个Transformer块的输入和输出保持相同的尺寸。而在PVT中,它通过调整每个阶段的块大小和步长,逐渐缩减特征图的空间维度,这样便可以在保持计算复杂度可控的前提下处理更大的输入图像。
-
逐阶段特征提取:在每个阶段的内部,采用多头自注意力(Multi-Head Self-Attention, MHSA)机制处理划分好的块,以及前馈神经网络(Feed-Forward Network, FFN)提取特征。由于每个阶段处理的特征图尺寸不同,PVT能够捕获多种尺度上的信息。
-
重构和降采样:在过渡到下一阶段之前,PVT通过重构和降采样操作减小特征图的空间尺寸,为处理下一尺度的特征图做准备。这通常是通过应用卷积操作实现的。
PVT通过这种层次化的设计,有效地生成了不同尺度的特征表示,对于处理不同分辨率的视觉信息非常有效。这使得PVT在多种视觉任务上,如图像分类、目标检测和语义分割等,表现出色。
-
- User
vit_ce中的输入patches是固定的,这样有什么优劣?
- Assistant
在Vision Transformer (ViT) 及其变种(例如ViT-CE)中,输入图像被切割成等大的patches是一种设计选择。这个设计选择带来了一系列的优缺点,基于这种处理方式和固定尺寸的patches,我们可以分析如下:
优点:
-
简化模型设计:通过将输入图像切割成固定尺寸的patches,并将每个patch线性嵌入到一个高维空间,这种方式简化了模型的输入处理流程。每个patch可以被视为一个“词”(类比于NLP中的词嵌入),这使得Transformer模型可以直接应用于视觉领域。
-
并行处理:由于所有patches的尺寸相同,它允许模型并行地处理这些patches,显著提高了计算效率。这种并行处理的特性是Transformer架构的核心优势之一。
-
模型泛化能力:固定尺寸的patches处理允许模型无需调整即可应对不同尺寸的输入图像,这为模型的泛化能力和灵活应用提供了便利。
缺点:
-
空间信息缺失:将图像分成若干独立的patches可能导致丢失部分局部细节和边界上的空间信息。尽管Transformer通过自注意力机制在全局范围内建立起patch间的联系,局部连续性的损失仍是其中一个限制。
-
效率问题:对于高分辨率的输入图像,生成大量的小patches会导致计算开销显著增加,特别是在自注意力机制中,计算复杂度会随着序列长度的平方线性增长。这使得处理高分辨率图像变得不够高效。
-
尺寸敏感性:虽然ViT在处理不同尺寸的图像方面表现出了灵活性,但模型通常在特定尺寸的数据集上进行预训练,并优化了那个尺寸级别的任务。在应用于大幅不同尺寸的图像时,不同尺寸的patches可能影响模型性能,因此可能需要调整模型或重新训练模型适应新的输入尺寸。
-
缺乏多尺度能力:固定尺寸的patches不利于模型捕获多尺度的特征,这在一些需要感知不同尺度物体的任务(如目标检测和语义分割)中尤为重要。相较之下,多尺度或金字塔式的结构(如PVT所采用)能更好地处理这类问题。
总的来说,尽管采用固定尺寸的patches简化了模型设计,并使得ViT及其变种在多种视觉任务上取得了显著成就,但这种设计也带来了一系列挑战和限制,需要针对特定任务进行适应性修改和优化。
-