在import math import logging from functools import partial from collections import…
- User
在import math import logging from functools import partial from collections import OrderedDict from copy import deepcopy
import torch import torch.nn as nn import torch.nn.functional as F
from timm.models.layers import to_2tuple
from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise from .utils import combine_tokens, recover_tokens from .vit import VisionTransformer from ..layers.attn_blocks import CEBlock import random import numpy as np from .DCM import Counter_Guide_Enhanced
_logger = logging.getLogger(name)
class VisionTransformerCE(VisionTransformer): """ Vision Transformer with candidate elimination (CE) module
A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` - https://arxiv.org/abs/2010.11929 Includes distillation token & head support for `DeiT: Data-efficient Image Transformers` - https://arxiv.org/abs/2012.12877 """ def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None, act_layer=None, weight_init='', ce_loc=None, ce_keep_ratio=None): super().__init__() if isinstance(img_size, tuple): self.img_size = img_size else: self.img_size = to_2tuple(img_size) self.patch_size = patch_size self.in_chans = in_chans self.num_classes = num_classes self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models self.num_tokens = 2 if distilled else 1 norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) act_layer = act_layer or nn.GELU self.patch_embed = embed_layer( img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4) # self.pos_embed_event = PatchEmbed_event_new(in_chans=32, embed_dim=768, kernel_size=4, stride=4) dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule blocks = [] ce_index = 0 self.ce_loc = ce_loc for i in range(depth): ce_keep_ratio_i = 1.0 if ce_loc is not None and i in ce_loc: ce_keep_ratio_i = ce_keep_ratio[ce_index] ce_index += 1 blocks.append( CEBlock( dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer, keep_ratio_search=ce_keep_ratio_i) ) self.blocks = nn.Sequential(*blocks) self.norm = norm_layer(embed_dim) self.init_weights(weight_init) # 添加交互模块counter_guide-此处对模块已经做了更新 self.counter_guide = Counter_Guide_Enhanced(768, 768) # 新增随机遮蔽特征操作 def masking_fea(self,z, event_z, x, event_x, ratio=0.8 ): b,nz,c = z.shape b,nez,c = event_z.shape b,nx,c = x.shape b,nex,c = event_x.shape assert(nz == nez) assert(nx == nex) lenz_out = int(nz*ratio) lenx_out = int(nx*ratio) mask_nz = torch.rand(b,nz).float() mask_ez = torch.rand(b,nez).float() mask_nx = torch.rand(b,nx).float() mask_ex = torch.rand(b,nex).float() mask_nz = mask_nz>0.4 mask_ez = mask_ez>0.4 mask_ez = ~mask_nz + mask_ez mask_nz_idx = mask_nz.float().sort(1,descending=True)[-1].to(device = z.device) mask_ez_idx = mask_ez.float().sort(1,descending=True)[-1].to(device = z.device) mask_nx = mask_nx>0.4 mask_ex = mask_ex>0.4 mask_ex = ~mask_nx + mask_ex mask_nx_idx = mask_nx.float().sort(1,descending=True)[-1].to(device = z.device) mask_ex_idx = mask_ex.float().sort(1,descending=True)[-1].to(device = z.device) masked_z = torch.gather(z, 1, mask_nz_idx[:,:lenz_out,None].repeat([1,1,c])) masked_ez = torch.gather(event_z, 1, mask_ez_idx[:,:lenz_out,None].repeat([1,1,c])) masked_x = torch.gather(x, 1, mask_nx_idx[:,:lenx_out,None].repeat([1,1,c])) masked_ex = torch.gather(event_x, 1, mask_ex_idx[:,:lenx_out,None].repeat([1,1,c])) return masked_z, masked_ez, masked_x, masked_ex,{'x1':mask_nx_idx[:,:lenx_out],'x0':mask_nx_idx[:,lenx_out:], 'ex1':mask_ex_idx[:,:lenx_out],'ex0':mask_ex_idx[:,lenx_out:], } # 随机遮蔽特征操作完成 def forward_features(self, z, x, event_z, event_x, mask_z=None, mask_x=None, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False ): # 分支1 处理流程 B, H, W = x.shape[0], x.shape[2], x.shape[3] x = self.patch_embed(x) z = self.patch_embed(z) z += self.pos_embed_z x += self.pos_embed_x event_x = self.pos_embed_event(event_x) event_z = self.pos_embed_event(event_z) event_x += self.pos_embed_x event_z += self.pos_embed_z masked_z, masked_event_z, masked_x, masked_event_x, token_idx = self.masking_fea(z, event_z, x, event_x, ratio=0.9) z = masked_z x = masked_x event_x = masked_event_x event_z = masked_event_z if mask_z is not None and mask_x is not None: mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_z = mask_z.flatten(1).unsqueeze(-1) mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_x = mask_x.flatten(1).unsqueeze(-1) mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode) mask_x = mask_x.squeeze(-1) if self.add_cls_token: cls_tokens = self.cls_token.expand(B, -1, -1) cls_tokens = cls_tokens + self.cls_pos_embed if self.add_sep_seg: x += self.search_segment_pos_embed z += self.template_segment_pos_embed x = combine_tokens(z, x, mode=self.cat_mode) if self.add_cls_token: x = torch.cat([cls_tokens, x], dim=1) x = self.pos_drop(x) lens_z = self.pos_embed_z.shape[1] lens_x = self.pos_embed_x.shape[1] global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device) global_index_t = global_index_t.repeat(B, 1) global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device) global_index_s = global_index_s.repeat(B, 1) removed_indexes_s = [] # # 分支2 处理流程 event_x = combine_tokens(event_z, event_x, mode=self.cat_mode) if self.add_cls_token: event_x = torch.cat([cls_tokens, event_x], dim=1) lens_z = self.pos_embed_z.shape[1] lens_x = self.pos_embed_x.shape[1] global_index_t1 = torch.linspace(0, lens_z - 1, lens_z).to(event_x.device) global_index_t1 = global_index_t1.repeat(B, 1) global_index_s1 = torch.linspace(0, lens_x - 1, lens_x).to(event_x.device) global_index_s1 = global_index_s1.repeat(B, 1) removed_indexes_s1 = [] out_attn = [] out_attn1 = [] for i, blk in enumerate(self.blocks): # 第一个分支处理 x, global_index_t, global_index_s, removed_index_s, attn = \ blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate) # 第二个分支处理 event_x, global_index_t1, global_index_s1, removed_index_s1, attn1 = \ blk(event_x, global_index_t1, global_index_s1, mask_x, ce_template_mask, ce_keep_rate) if self.ce_loc is not None and i in self.ce_loc: removed_indexes_s.append(removed_index_s) removed_indexes_s1.append(removed_index_s1) out_attn.append(attn) out_attn1.append(attn1) out_attn_idx = random.choice(np.arange(len(out_attn))) out_attn = out_attn[out_attn_idx] out_attn_idx1 = random.choice(np.arange(len(out_attn1))) out_attn1 = out_attn1[out_attn_idx1] # 应用LayerNorm归一化处理 x = self.norm(x) event_x = self.norm(event_x) x_cat = torch.cat([event_x, x], dim=1) x = x_cat aux_dict = { "attn": out_attn, "attn1": out_attn1, "removed_indexes_s": removed_indexes_s, # used for visualization 'removed_indexes_s1': removed_indexes_s1, } return x, aux_dict def forward(self, z, x, event_z, event_x, ce_template_mask=None, ce_keep_rate=None, tnc_keep_rate=None, return_last_attn=False): x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,) return x, aux_dictdef _create_vision_transformer(pretrained=False, **kwargs): model = VisionTransformerCE(**kwargs)
if pretrained: if 'npz' in pretrained: model.load_pretrained(pretrained, prefix='') else: checkpoint = torch.load(pretrained, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False) print('Load pretrained model from: ' + pretrained) return modeldef vit_base_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model
def vit_large_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model 和 from . import BaseActor from lib.utils.misc import NestedTensor from lib.utils.box_ops import box_cxcywh_to_xyxy, box_xywh_to_xyxy import torch from lib.utils.merge import merge_template_search from ...utils.heapmap_utils import generate_heatmap from ...utils.ce_utils import generate_mask_cond, adjust_keep_rate
import torch.nn.functional as F
class CEUTrackActor(BaseActor): """ Actor for training CEUTrack models """
def __init__(self, net, objective, loss_weight, settings, cfg=None): super().__init__(net, objective) self.loss_weight = loss_weight self.settings = settings self.bs = self.settings.batchsize # batch size self.cfg = cfg def __call__(self, data): """ args: data - The input data, should contain the fields 'template', 'search', 'gt_bbox'. template_images: (N_t, batch, 3, H, W) search_images: (N_s, batch, 3, H, W) returns: loss - the training loss status - dict containing detailed losses """ # forward pass out_dict = self.forward_pass(data) # compute losses loss, status = self.compute_losses(out_dict, data) return loss, status def forward_pass(self, data): # currently only support 1 template and 1 search region assert len(data['template_images']) == 1 assert len(data['search_images']) == 1 assert len(data['template_event']) == 1 assert len(data['search_event']) == 1 template_list = [] for i in range(self.settings.num_template): template_img_i = data['template_images'][i].view(-1, *data['template_images'].shape[2:]) # (batch, 3, 128, 128) # template_att_i = data['template_att'][i].view(-1, *data['template_att'].shape[2:]) # (batch, 128, 128) template_list.append(template_img_i) search_img = data['search_images'][0].view(-1, *data['search_images'].shape[2:]) # (batch, 3, 320, 320) # search_att = data['search_att'][0].view(-1, *data['search_att'].shape[2:]) # (batch, 320, 320) template_event = data['template_event'][0].view(-1, *data['template_event'].shape[2:]) search_event = data['search_event'][0].view(-1, *data['search_event'].shape[2:]) box_mask_z = None ce_keep_rate = None if self.cfg.MODEL.BACKBONE.CE_LOC: box_mask_z = generate_mask_cond(self.cfg, template_list[0].shape[0], template_list[0].device, data['template_anno'][0]) ce_start_epoch = self.cfg.TRAIN.CE_START_EPOCH ce_warm_epoch = self.cfg.TRAIN.CE_WARM_EPOCH ce_keep_rate = adjust_keep_rate(data['epoch'], warmup_epochs=ce_start_epoch, total_epochs=ce_start_epoch + ce_warm_epoch, ITERS_PER_EPOCH=1, base_keep_rate=self.cfg.MODEL.BACKBONE.CE_KEEP_RATIO[0]) if len(template_list) == 1: template_list = template_list[0] out_dict = self.net(template=template_list, search=search_img, event_template=template_event, event_search=search_event, ce_template_mask=box_mask_z, ce_keep_rate=ce_keep_rate, return_last_attn=False) return out_dict def compute_losses(self, pred_dict, gt_dict, return_status=True): # gt gaussian map gt_bbox = gt_dict['search_anno'][-1] # (Ns, batch, 4) (x1,y1,w,h) -> (batch, 4) gt_gaussian_maps = generate_heatmap(gt_dict['search_anno'], self.cfg.DATA.SEARCH.SIZE, self.cfg.MODEL.BACKBONE.STRIDE) gt_gaussian_maps = gt_gaussian_maps[-1].unsqueeze(1) # Get boxes pred_boxes = pred_dict['pred_boxes'] if torch.isnan(pred_boxes).any(): raise ValueError("Network outputs is NAN! Stop Training") num_queries = pred_boxes.size(1) pred_boxes_vec = box_cxcywh_to_xyxy(pred_boxes).view(-1, 4) # (B,N,4) --> (BN,4) (x1,y1,x2,y2) gt_boxes_vec = box_xywh_to_xyxy(gt_bbox)[:, None, :].repeat((1, num_queries, 1)).view(-1, 4).clamp(min=0.0, max=1.0) # (B,4) --> (B,1,4) --> (B,N,4) # compute giou and iou try: giou_loss, iou = self.objective['giou'](pred_boxes_vec, gt_boxes_vec) # (BN,4) (BN,4) except: giou_loss, iou = torch.tensor(0.0).cuda(), torch.tensor(0.0).cuda() # compute l1 loss l1_loss = self.objective['l1'](pred_boxes_vec, gt_boxes_vec) # (BN,4) (BN,4) # compute location loss if 'score_map' in pred_dict: location_loss = self.objective['focal'](pred_dict['score_map'], gt_gaussian_maps) else: location_loss = torch.tensor(0.0, device=l1_loss.device) # 按照HRCEUTrack,新增loss约束 rank_rgb_loss = self.rank_rgb_loss(pred_dict,gt_dict['search_anno'], gt_dict['template_anno']) rank_event_loss= self.rank_event_loss(pred_dict,gt_dict['search_anno'], gt_dict['template_anno']) ####### # 计算特征一致性损失 # feature_consistency_loss = F.mse_loss(pred_dict['A_enhanced_x'], pred_dict['A_original_x']) + F.mse_loss(pred_dict['A_enhanced_event_x'], pred_dict['A_original_event_x']) # # 标量化特征一致性损失 # feature_consistency_loss = feature_consistency_loss.mean() # weighted sum loss = self.loss_weight['giou'] * giou_loss + self.loss_weight['l1'] * l1_loss + self.loss_weight['focal'] * location_loss + rank_rgb_loss * 1.2 + rank_event_loss * 1.2 # + 1 * feature_consistency_loss if return_status: # status for log mean_iou = iou.detach().mean() status = {"Loss/total": loss.item(), "Loss/giou": giou_loss.item(), "Loss/l1": l1_loss.item(), "Loss/location": location_loss.item(), #'Loss/feature_consistency': feature_consistency_loss.item(), "rank_rgb_loss": rank_rgb_loss.item(), "rank_event_loss": rank_event_loss.item(), "IoU": mean_iou.item()} return loss, status else: return loss#################################### 按照 HRCEUTrack新增 loss #########################################
def _random_permute(self,matrix): # matrix = random.choice(matrix) b, c, h, w = matrix.shape idx = [ torch.randperm(c).to(matrix.device) for i in range(b)] idx = torch.stack(idx, dim=0)[:, :, None, None].repeat([1,1,h,w]) # idx = torch.randperm(c)[None,:,None,None].repeat([b,1,h,w]).to(matrix.device) matrix01 = torch.gather(matrix, 1, idx) return matrix01 def crop_flag(self, flag, global_index_s, global_index_t,H1 = 64, H2 = 256): B,Ls = global_index_s.shape B, Lt = global_index_t.shape B,C,L1,L2 = flag.shape flag_t = flag[:,:,:H1,:] flag_s = flag[:,:,H1:,:] flag_t = torch.gather(flag_t,2,global_index_t[:,None,:,None].repeat([1,C,1,L2]).long()) flag_s = torch.gather(flag_s,2,global_index_s[:,None,:,None].repeat([1,C,1,L2]).long()) flag = torch.cat([flag_t, flag_s], dim = 2) flag_t = flag[:,:,:,:H1] flag_s = flag[:,:,:,H1:] flag_t = torch.gather(flag_t,3,global_index_t[:,None,None,:].repeat([1,C,int(Ls+Lt),1]).long()) flag_s = torch.gather(flag_s,3,global_index_s[:,None,None,:].repeat([1,C,int(Ls+Lt),1]).long()) flag = torch.cat([flag_t, flag_s], dim = 3) B, C, L11, L12 = flag.shape try: assert(L11 == int(Lt + Ls)) assert(L12 == int(Lt + Ls)) except: print('L11:{}, L12:{}, L1:{}, L2:{}'.format(L11, L12, L1, L2)) return flag def crop_fusion(self, flag, attn, global_index_s, global_index_t,H1 = 64, H2 = 256 ): flag = self.crop_flag(flag=flag, global_index_s=global_index_s, global_index_t=global_index_t) B,C,L1,L2 = flag.shape Ba, Ca, La, La2 = attn.shape _,idx1 = flag.mean(dim=3,keepdim=False).sort(dim=2,descending=True) # print('shape of flag:{}, idx1:{}'.format(flag.shape, idx1[:,:,:32,None].repeat([1,Ca,1,L2]).shape)) flag = torch.gather(flag,2,idx1[:,:,:32,None].repeat([1,C,1,L2]).long()) attn = torch.gather(attn,2,idx1[:,:,:32,None].repeat([1,Ca,1,L2]).long()) _,idx2 = flag.mean(dim=2,keepdim=False).sort(dim=2,descending=True) flag = torch.gather(flag,3,idx2[:,:,None,:32].repeat([1,C,32,1]).long()) attn = torch.gather(attn,3,idx2[:,:,None,:32].repeat([1,Ca,32,1]).long()) return attn * flag def loss_rank_ori(self, outputs, targetsi, temp_annoi=None): """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] The target boxes are expected in format (center_x, center_y, h, w), normalized by the image size. """ attn = outputs['attn'] # print('attn shape:',attn.shape) attn1 = torch.cat([attn[:,:,114:344,57:114], attn[:,:,114:344,344:]],dim=3) attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # print('attn1 shape',attn1.shape) attn2 = torch.cat([attn[:,:,344:,:57], attn[:,:,344:,114:344]],dim=3) attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # print('attn2 shape',attn2.shape) attn = torch.cat([attn1, attn2],dim=1) B, C, H, W = attn.shape _,s1,_ = torch.svd(attn.reshape([B*C, H, W])) s01 = torch.abs(s1 - 1) return torch.mean(s01) def rank_rgb_loss(self, outputs, targetsi, temp_annoi=None): attn = outputs['attn'] # print('attn.shape',attn.shape) # attn.shape torch.Size([2, 12, 320, 320]) attn1 = torch.cat([attn[:,:,57:172,28:57], attn[:,:,57:172,172:]], dim=3) attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) attn2 = torch.cat([attn[:,:,172:,:28], attn[:,:,172:,57:172]], dim=3) attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) attn2 = F.interpolate(attn2, size=(115, 144), mode='bilinear', align_corners=False) attn = torch.cat([attn1, attn2], dim=1) B, C, H, W = attn.shape _, s1, _ = torch.svd(attn.reshape([B*C, H, W])) s01 = torch.abs(s1 - 1) return torch.mean(s01) def rank_event_loss(self, outputs, targetsi, temp_annoi=None): attn = outputs['attn1'] attn1 = torch.cat([attn[:,:,57:172,28:57], attn[:,:,57:172,172:]], dim=3) attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) attn2 = torch.cat([attn[:,:,172:,:28], attn[:,:,172:,57:172]], dim=3) attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) attn2 = F.interpolate(attn2, size=(115, 144), mode='bilinear', align_corners=False) attn = torch.cat([attn1, attn2], dim=1) B, C, H, W = attn.shape _, s1, _ = torch.svd(attn.reshape([B*C, H, W])) s02 = torch.abs(s1 - 1) return torch.mean(s02)中关于 attn和attn1,我们期望进行 内部矩阵的可视化,应该如何处理? - Assistant
要对
attn和attn1进行可视化,通常的方法是将这些注意力矩阵绘制为热图,这可以提供关于模型焦点位置的直观信息。由于attn和attn1的形状可能为[batch_size, num_heads, height, width](或类似形状),以下是一个步骤和示例代码,解释如何将这些注意力矩阵进行可视化:1. 选择特定批次和头部
通常,您可能希望查看特定批次或特定头部的注意力图。针对简化,可以选择第一个批次和第一个头部:
batch_index = 0 head_index = 0 attn_to_plot = attn[batch_index, head_index] attn1_to_plot = attn1[batch_index, head_index]2. 绘制热图
使用 matplotlib 创建一个热图以可视化注意力权重。这可以帮助理解模型在处理输入时的关注区域。
import matplotlib.pyplot as plt def plot_attention(attention_map, title="Attention Map"): plt.figure(figsize=(10, 10)) plt.imshow(attention_map, cmap='viridis') plt.colorbar() plt.title(title) plt.show() plot_attention(attn_to_plot.cpu().detach().numpy(), title="Attention Map for attn") plot_attention(attn1_to_plot.cpu().detach().numpy(), title="Attention Map for attn1")3. 特殊情况处理
如果您的注意力图有特定的结构或者需要特殊处理(例如,调整尺寸或者应用非线性标准化等),请在绘图之前进行这些处理。例如,有时您可能需要归一化注意力分数。
attn_to_plot = attn_to_plot / attn_to_plot.max() attn1_to_plot = attn1_to_plot / attn1_to_plot.max()4. 保存或导出
如果需要将生成的可视化保存为文件或进行进一步分析,可以使用
plt.save()方法。问题复杂性
注意力机制在视觉Transformer结构中是用来选取图像的哪些部分是重要的。提供这些视觉化有助于调试和解释模型如何理解数据。输出的形状、头部数量等因素可能会根据实际使用的模型架构有所不同,确保理解自己模型的细节以适当调整上述代码。
- User
attn和attn1的维度都是 torch.Size([4, 12, 287, 287])
- Assistant
给定
attn和attn1的形状为[4, 12, 287, 287](其中 4 是批次大小,12 是头的数量,287x287 是注意力矩阵的空间维度),我们可以使用下面的步骤来可视化特定的注意力图。选择数据
如果想可视化第一个样本(batch_index = 0)的第一个头部(head_index = 0)的注意力图:
batch_index = 0 head_index = 0 attn_to_plot = attn[batch_index, head_index] attn1_to_plot = attn1[batch_index, head_index]绘制和显示注意力图
继续使用 matplotlib 来绘制这些选择的注意力图,如下:
import matplotlib.pyplot as plt def plot_attention(attention_map, title="Attention Map"): plt.figure(figsize=(12, 12)) # 图的大小可以根据需要调整 plt.imshow(attention_map, cmap='viridis') # 使用“viridis”色彩映射,也可尝试其他色彩映射如 'hot', 'coolwarm', etc. plt.colorbar() # 显示颜色条 plt.title(title) plt.grid(False) # 隐藏网格线,更好地观察图 plt.show() # 绘制注意力图 plot_attention(attn_to_plot.cpu().detach().numpy(), title="Attention Map for attn - Head 1, Sample 1") plot_attention(attn1_to_plot.cpu().detach().numpy(), title="Attention Map for attn1 - Head 1, Sample 1")探索不同头部的注意力
可以通过循环来查看同一批次中不同头部的注意力分布情况:
for head_index in range(attn.shape[1]): # attn.shape[1] 是头部的数量 attn_to_plot = attn[batch_index, head_index].cpu().detach().numpy() attn1_to_plot = attn1[batch_index, head_index].cpu().detach().numpy() plot_attention(attn_to_plot, title=f"Attention Map for attn - Head {head_index + 1}, Sample 1") plot_attention(attn1_to_plot, title=f"Attention Map for attn1 - Head {head_index + 1}, Sample 1")通过这种方式,你可以快速检查每个头部的注意力模式,并评估他们是否在抓取你认为重要的视觉特征。注意力图的解释需要基于模型的具体应用上下文,并且可能需要领域专业知识。
- User
那么在上述代码中,直接可视化attn和attn1,那么应该在源代码中加在什么位置?
- Assistant
要在你的代码中直接集成可视化
attn和attn1,你需要确定何时和在哪里执行这些可视化操作最有意义。通常,将这种可视化放在模型的推理或测试阶段是更好的选择,这样不会干扰正常的训练流程。如果你的目的是调试或理解模型的注意力机制,可以将这些代码放在你执行推理或评估的地方,例如在模型测试函数或在执行单一样本推理后。示例:在测试或评估函数中集成
如果你有一个专门用于模型评估的函数,在那里添加可视化代码通常是最合适的。这里有一个简化的例子说明如何集成:
def evaluate_model(model, dataloader): model.eval() # 将模型设置为评估模式 with torch.no_grad(): # 关闭梯度计算 for data in dataloader: # 获得输入数据 inputs = data['input'].to(device) labels = data['label'].to(device) # 执行模型推理 outputs, attn_maps = model(inputs) attn = attn_maps['attn'] attn1 = attn_maps['attn1'] # 可视化第一批数据中第一个头的注意力图 plot_attention(attn[0, 0].cpu().numpy(), "Attention Map for attn - Head 1, Sample 1") plot_attention(attn1[0, 0].cpu().numpy(), "Attention Map for attn1 - Head 1, Sample 1") # 你可以在这里添加更多的可视化或其他后处理步骤 break # 如果只想看第一批的结果,加上这个break # 假设 you have 'model' 和 'test_loader' evaluate_model(model, test_loader)细节注意事项
- 实际应用位置:确保在模型输出
attn和attn1变量的地方调用可视化函数。你可能需要稍微修改模型的前向传播函数,以确保它返回注意力矩阵。 - 性能考虑:可视化通常是耗时的操作,尤其是在大规模的数据集上。你可能想限制可视化的样本数量,或者只在特定的验证步骤中执行可视化。
- 设备问题:在进行可视化之前,确保将张量从 GPU 移到 CPU,并转换为 NumPy 数组(如上例所示)。
确保你理解何时调用你的评估或测试函数,并相应地安排可视化的执行。如果是在训练循环中调用,注意要区分训练和验证阶段,以避免影响训练过程的性能。
- 实际应用位置:确保在模型输出
- User
在from . import BaseActor from lib.utils.misc import NestedTensor from lib.utils.box_ops import box_cxcywh_to_xyxy, box_xywh_to_xyxy import torch from lib.utils.merge import merge_template_search from ...utils.heapmap_utils import generate_heatmap from ...utils.ce_utils import generate_mask_cond, adjust_keep_rate
import torch.nn.functional as F
class CEUTrackActor(BaseActor): """ Actor for training CEUTrack models """
def __init__(self, net, objective, loss_weight, settings, cfg=None): super().__init__(net, objective) self.loss_weight = loss_weight self.settings = settings self.bs = self.settings.batchsize # batch size self.cfg = cfg def __call__(self, data): """ args: data - The input data, should contain the fields 'template', 'search', 'gt_bbox'. template_images: (N_t, batch, 3, H, W) search_images: (N_s, batch, 3, H, W) returns: loss - the training loss status - dict containing detailed losses """ # forward pass out_dict = self.forward_pass(data) # compute losses loss, status = self.compute_losses(out_dict, data) return loss, status def forward_pass(self, data): # currently only support 1 template and 1 search region assert len(data['template_images']) == 1 assert len(data['search_images']) == 1 assert len(data['template_event']) == 1 assert len(data['search_event']) == 1 template_list = [] for i in range(self.settings.num_template): template_img_i = data['template_images'][i].view(-1, *data['template_images'].shape[2:]) # (batch, 3, 128, 128) # template_att_i = data['template_att'][i].view(-1, *data['template_att'].shape[2:]) # (batch, 128, 128) template_list.append(template_img_i) search_img = data['search_images'][0].view(-1, *data['search_images'].shape[2:]) # (batch, 3, 320, 320) # search_att = data['search_att'][0].view(-1, *data['search_att'].shape[2:]) # (batch, 320, 320) template_event = data['template_event'][0].view(-1, *data['template_event'].shape[2:]) search_event = data['search_event'][0].view(-1, *data['search_event'].shape[2:]) box_mask_z = None ce_keep_rate = None if self.cfg.MODEL.BACKBONE.CE_LOC: box_mask_z = generate_mask_cond(self.cfg, template_list[0].shape[0], template_list[0].device, data['template_anno'][0]) ce_start_epoch = self.cfg.TRAIN.CE_START_EPOCH ce_warm_epoch = self.cfg.TRAIN.CE_WARM_EPOCH ce_keep_rate = adjust_keep_rate(data['epoch'], warmup_epochs=ce_start_epoch, total_epochs=ce_start_epoch + ce_warm_epoch, ITERS_PER_EPOCH=1, base_keep_rate=self.cfg.MODEL.BACKBONE.CE_KEEP_RATIO[0]) if len(template_list) == 1: template_list = template_list[0] out_dict = self.net(template=template_list, search=search_img, event_template=template_event, event_search=search_event, ce_template_mask=box_mask_z, ce_keep_rate=ce_keep_rate, return_last_attn=False) return out_dict def compute_losses(self, pred_dict, gt_dict, return_status=True): # gt gaussian map gt_bbox = gt_dict['search_anno'][-1] # (Ns, batch, 4) (x1,y1,w,h) -> (batch, 4) gt_gaussian_maps = generate_heatmap(gt_dict['search_anno'], self.cfg.DATA.SEARCH.SIZE, self.cfg.MODEL.BACKBONE.STRIDE) gt_gaussian_maps = gt_gaussian_maps[-1].unsqueeze(1) # Get boxes pred_boxes = pred_dict['pred_boxes'] if torch.isnan(pred_boxes).any(): raise ValueError("Network outputs is NAN! Stop Training") num_queries = pred_boxes.size(1) pred_boxes_vec = box_cxcywh_to_xyxy(pred_boxes).view(-1, 4) # (B,N,4) --> (BN,4) (x1,y1,x2,y2) gt_boxes_vec = box_xywh_to_xyxy(gt_bbox)[:, None, :].repeat((1, num_queries, 1)).view(-1, 4).clamp(min=0.0, max=1.0) # (B,4) --> (B,1,4) --> (B,N,4) # compute giou and iou try: giou_loss, iou = self.objective['giou'](pred_boxes_vec, gt_boxes_vec) # (BN,4) (BN,4) except: giou_loss, iou = torch.tensor(0.0).cuda(), torch.tensor(0.0).cuda() # 新增 compute ciou and iou # try: # ciou_loss, iou = self.objective['ciou'](pred_boxes_vec, gt_boxes_vec) # (BN,4) (BN,4) # except: # ciou_loss, iou = torch.tensor(0.0).cuda(), torch.tensor(0.0).cuda() ############################### # compute l1 loss l1_loss = self.objective['l1'](pred_boxes_vec, gt_boxes_vec) # (BN,4) (BN,4) # compute location loss if 'score_map' in pred_dict: location_loss = self.objective['focal'](pred_dict['score_map'], gt_gaussian_maps) else: location_loss = torch.tensor(0.0, device=l1_loss.device) # 按照HRCEUTrack,新增loss约束 rank_loss = self.loss_rank(pred_dict,gt_dict['search_anno'], gt_dict['template_anno']) ####### # 计算特征一致性损失 # feature_consistency_loss = F.mse_loss(pred_dict['A_enhanced_x'], pred_dict['A_original_x']) + F.mse_loss(pred_dict['A_enhanced_event_x'], pred_dict['A_original_event_x']) # # 标量化特征一致性损失 # feature_consistency_loss = feature_consistency_loss.mean() # weighted sum loss = self.loss_weight['giou'] * giou_loss + self.loss_weight['l1'] * l1_loss + self.loss_weight['focal'] * location_loss + rank_loss_rgb * 1.2 + rank_loss_event * 1.2 # + 1 * feature_consistency_loss if return_status: # status for log mean_iou = iou.detach().mean() status = {"Loss/total": loss.item(), "Loss/giou": giou_loss.item(), "Loss/l1": l1_loss.item(), "Loss/location": location_loss.item(), #'Loss/feature_consistency': feature_consistency_loss.item(), "IoU": mean_iou.item()} return loss, status else: return loss#################################### 按照 HRCEUTrack新增 loss #########################################
def _random_permute(self,matrix): # matrix = random.choice(matrix) b, c, h, w = matrix.shape idx = [ torch.randperm(c).to(matrix.device) for i in range(b)] idx = torch.stack(idx, dim=0)[:, :, None, None].repeat([1,1,h,w]) # idx = torch.randperm(c)[None,:,None,None].repeat([b,1,h,w]).to(matrix.device) matrix01 = torch.gather(matrix, 1, idx) return matrix01 def crop_flag(self, flag, global_index_s, global_index_t,H1 = 64, H2 = 256): B,Ls = global_index_s.shape B, Lt = global_index_t.shape B,C,L1,L2 = flag.shape flag_t = flag[:,:,:H1,:] flag_s = flag[:,:,H1:,:] flag_t = torch.gather(flag_t,2,global_index_t[:,None,:,None].repeat([1,C,1,L2]).long()) flag_s = torch.gather(flag_s,2,global_index_s[:,None,:,None].repeat([1,C,1,L2]).long()) flag = torch.cat([flag_t, flag_s], dim = 2) flag_t = flag[:,:,:,:H1] flag_s = flag[:,:,:,H1:] flag_t = torch.gather(flag_t,3,global_index_t[:,None,None,:].repeat([1,C,int(Ls+Lt),1]).long()) flag_s = torch.gather(flag_s,3,global_index_s[:,None,None,:].repeat([1,C,int(Ls+Lt),1]).long()) flag = torch.cat([flag_t, flag_s], dim = 3) B, C, L11, L12 = flag.shape try: assert(L11 == int(Lt + Ls)) assert(L12 == int(Lt + Ls)) except: print('L11:{}, L12:{}, L1:{}, L2:{}'.format(L11, L12, L1, L2)) return flag def crop_fusion(self, flag, attn, global_index_s, global_index_t,H1 = 64, H2 = 256 ): flag = self.crop_flag(flag=flag, global_index_s=global_index_s, global_index_t=global_index_t) B,C,L1,L2 = flag.shape Ba, Ca, La, La2 = attn.shape _,idx1 = flag.mean(dim=3,keepdim=False).sort(dim=2,descending=True) # print('shape of flag:{}, idx1:{}'.format(flag.shape, idx1[:,:,:32,None].repeat([1,Ca,1,L2]).shape)) flag = torch.gather(flag,2,idx1[:,:,:32,None].repeat([1,C,1,L2]).long()) attn = torch.gather(attn,2,idx1[:,:,:32,None].repeat([1,Ca,1,L2]).long()) _,idx2 = flag.mean(dim=2,keepdim=False).sort(dim=2,descending=True) flag = torch.gather(flag,3,idx2[:,:,None,:32].repeat([1,C,32,1]).long()) attn = torch.gather(attn,3,idx2[:,:,None,:32].repeat([1,Ca,32,1]).long()) return attn * flag # def loss_rank(self, outputs, targetsi, temp_annoi=None): # """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss # targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] # The target boxes are expected in format (center_x, center_y, h, w), normalized by the image size. # """ # attn = outputs['attn'] # print('attn shape:{}'.format(attn.shape)) # attn shape:torch.Size([4, 12, 574, 574]) # attn1 = torch.cat([attn[:,:,114:344,57:114], attn[:,:,114:344,344:]],dim=3) # print('attn1 shape:{}'.format(attn1.shape)) # attn1 shape:torch.Size([4, 12, 230, 287]) # attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # print('attn1 shape:{}'.format(attn1.shape)) # attn1 shape:torch.Size([1, 1, 230, 287]) # attn2 = torch.cat([attn[:,:,344:,:57], attn[:,:,344:,114:344]],dim=3) # print('attn2 shape:{}'.format(attn2.shape)) # attn2 shape:torch.Size([4, 12, 230, 287]) # attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # print('attn2 shape:{}'.format(attn1.shape)) # attn2 shape:torch.Size([1, 1, 230, 287]) # attn = torch.cat([attn1, attn2],dim=1) # print('cat_attn shape:',attn.shape) # cat_attn shape: torch.Size([1, 2, 230, 287]) # B, C, H, W = attn.shape # # _,s1,_ = torch.svd(attn1.reshape([B*C, H, W])) # _,s1,_ = torch.svd(attn.reshape([B*C, H, W])) # s01 = torch.abs(s1 - 1) # return torch.mean(s01) # def loss_rank(self, outputs, targetsi, temp_annoi=None): # """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss # targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] # The target boxes are expected in format (center_x, center_y, h, w), normalized by the image size. # """ # attn = outputs['attn'] # print('attn shape:{}'.format(attn.shape)) # # attn1 = torch.cat([attn[:,:,114:344,57:114], attn[:,:,114:344,344:]],dim=3) # print('attn1 shape:{}'.format(attn1.shape)) # attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # print('attn1 shape:{}'.format(attn1.shape)) # attn2 = torch.cat([attn[:,:,344:,:57], attn[:,:,344:,114:344]],dim=3) # print('attn2 shape:{}'.format(attn2.shape)) # attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # print('attn2 shape:{}'.format(attn1.shape)) # attn = torch.cat([attn1, attn2],dim=1) # print('cat_attn shape:',attn.shape) # B, C, H, W = attn.shape # # _,s1,_ = torch.svd(attn1.reshape([B*C, H, W])) # _,s1,_ = torch.svd(attn.reshape([B*C, H, W])) # s01 = torch.abs(s1 - 1) # return torch.mean(s01) def loss_rank_ori(self, outputs, targetsi, temp_annoi=None): """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] The target boxes are expected in format (center_x, center_y, h, w), normalized by the image size. """ attn = outputs['attn'] print('attn shape:',attn.shape) # # attn shape: torch.Size([4, 12, 574, 574]) attn1 = torch.cat([attn[:,:,114:344,57:114], attn[:,:,114:344,344:]],dim=3) attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) print('attn1 shape',attn1.shape) attn2 = torch.cat([attn[:,:,344:,:57], attn[:,:,344:,114:344]],dim=3) attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) print('attn2 shape',attn2.shape) attn = torch.cat([attn1, attn2],dim=1) B, C, H, W = attn.shape _,s1,_ = torch.svd(attn.reshape([B*C, H, W])) s01 = torch.abs(s1 - 1) return torch.mean(s01) def loss_rank_rgb(self, outputs, targetsi, temp_annoi=None): attn = outputs['attn'] print('attn shape:',attn.shape) # attn shape: torch.Size([4, 12, 287, 287]) # 假设保持切割比例,大致对应双模态下的分隔 attn1 = torch.cat([attn[:,:,57:172,28:57], attn[:,:,57:172,172:]], dim=3) attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # print('attn1 shape',attn1.shape) # attn1 shape torch.Size([1, 1, 115, 144]) attn2 = torch.cat([attn[:,:,172:,:28], attn[:,:,172:,57:172]], dim=3) attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) attn2 = F.interpolate(attn2, size=(115, 144), mode=‘bilinear’, align_corners=False) # print('attn2 shape',attn2.shape) # attn2 shape torch.Size([1, 1, 115, 143]) # attn2 = torch.nn.functional.pad(attn2, (0, 1)) # print('attn2 shape',attn2.shape) # attn2 shape torch.Size([1, 1, 115, 144]) attn = torch.cat([attn1, attn2], dim=1) B, C, H, W = attn.shape _, s1, _ = torch.svd(attn.reshape([B*C, H, W])) s01 = torch.abs(s1 - 1) return torch.mean(s01) def loss_rank_event(self, outputs, targetsi, temp_annoi=None): attn = outputs['attn1'] print('attn shape:',attn.shape) # attn shape: torch.Size([4, 12, 287, 287]) # 假设保持切割比例,大致对应双模态下的分隔 attn1 = torch.cat([attn[:,:,57:172,28:57], attn[:,:,57:172,172:]], dim=3) attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) print('attn1 shape',attn1.shape) # attn1 shape torch.Size([1, 1, 115, 144]) attn2 = torch.cat([attn[:,:,172:,:28], attn[:,:,172:,57:172]], dim=3) attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) attn2 = torch.nn.functional.pad(attn2, (0, 1)) print('attn2 shape',attn2.shape) # attn2 shape torch.Size([1, 1, 115, 144]) attn = torch.cat([attn1, attn2], dim=1) B, C, H, W = attn.shape _, s1, _ = torch.svd(attn.reshape([B*C, H, W])) s01 = torch.abs(s1 - 1) return torch.mean(s01)中,attn = outputs['attn'],那么将此处的注意力图可视化出来,因为attn1 = torch.cat([attn[:,:,57:172,28:57], attn[:,:,57:172,172:]], dim=3) attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True)主要选择的是attn中交互部分的注意力区域,但是我们的模型变化了,所以我需要查看新的attn,然后重新确定新的交互区域在哪里 - Assistant
为了有效地确定和可视化您的新模型的注意力区域,您可以首先在模型的正向传递或评估阶段直接打印和可视化注意力矩阵
attn。这将帮助您了解哪些区域正在受到模型的关注,并据此调整对attn1的切割和处理策略。以下是一个简化的步骤,描述如何在您现有的代码框架中集成注意力图的可视化:
步骤 1: 修改
forward_pass方法以返回注意力矩阵确保
self.net模型的输出包括attn:def forward_pass(self, data): # Prepare the input data as before # ... your original setup code out_dict = self.net( # Ensure that this call returns not only outputs but also attn template=template_list, search=search_img, event_template=template_event, event_search=search_event, ce_template_mask=box_mask_z, ce_keep_rate=ce_keep_rate, return_last_attn=True # Assuming this flag makes your network return attn matrices ) return out_dict步骤 2: 可视化注意力矩阵
在
__call__或其他适当的方法中添加可视化代码,以便在进行预测时直接观察注意力矩阵:def __call__(self, data): out_dict = self.forward_pass(data) # Add visualization here for debugging attn = out_dict['attn'] # Ensure attn is returned correctly self.visualize_attention(attn[0, 0]) # Visualize the first attention map from the first head and first sample # Compute losses and so on loss, status = self.compute_losses(out_dict, data) return loss, status def visualize_attention(self, attention_map): # Simple plotting function, can be expanded based on needs import matplotlib.pyplot as plt plt.figure(figsize=(10, 10)) plt.imshow(attention_map.detach().cpu(), cmap='hot', interpolation='nearest') plt.colorbar() plt.title("Attention Map Visualization") plt.show()这个
visualize_attention方法简单地绘制了从第一个样本、第一个头部抽取的注意力图。根据模型的输出和您的具体需要,您可能会调整这部分代码来查看更多样本或不同头部的注意力图。注意点
- 确保模型在训练过程中或在需要时返回正确的注意力矩阵。
- 可视化代码会降低运行速度,特别是在训练时,所以建议仅在调试时开启。
- 根据
attn矩阵的维度调整可视化函数,确保正确显示。
这种集成方式应当仅用于开发和调试阶段,以免影响模型的正常训练和评估性能。当您了解了注意力集中的区域之后,可以据此优化模型的相关部分或数据处理步骤。