在import math import logging from functools import partial from collections import…
- User
在import math import logging from functools import partial from collections import OrderedDict from copy import deepcopy
import torch import torch.nn as nn import torch.nn.functional as F
from timm.models.layers import to_2tuple
from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise from .utils import combine_tokens, recover_tokens from .vit import VisionTransformer from …layers.attn_blocks import CEBlock import random import numpy as np
logger = logging.getLogger(name)
class VisionTransformerCE(VisionTransformer): “”" Vision Transformer with candidate elimination (CE) module
A PyTorch impl of : An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale
Includes distillation token & head support for DeiT: Data-efficient Image Transformers
def init(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None, act_layer=None, weight_init=‘’, ce_loc=None, ce_keep_ratio=None): “”” Args: img_size (int, tuple): input image size patch_size (int, tuple): patch size in_chans (int): number of input channels num_classes (int): number of classes for classification head embed_dim (int): embedding dimension depth (int): depth of transformer num_heads (int): number of attention heads mlp_ratio (int): ratio of mlp hidden dim to embedding dim qkv_bias (bool): enable bias for qkv if True representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set distilled (bool): model includes a distillation token and head as in DeiT models drop_rate (float): dropout rate attn_drop_rate (float): attention dropout rate drop_path_rate (float): stochastic depth rate embed_layer (nn.Module): patch embedding layer norm_layer: (nn.Module): normalization layer weight_init: (str): weight init scheme “”"
super().init()
super().init() if isinstance(img_size, tuple): self.img_size = img_size else: self.img_size = to_2tuple(img_size) self.patch_size = patch_size self.in_chans = in_chans
self.num_classes = num_classes self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models self.num_tokens = 2 if distilled else 1 norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) act_layer = act_layer or nn.GELU
self.patch_embed = embed_layer( img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches
self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4)
self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4)
self.pos_embed_event_z = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=3, stride=1)
attn = CrossAttn(768, 4, 3072, 0.1, ‘relu’)
self.cross_attn = Iter_attn(attn, 2)
dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule blocks = [] ce_index = 0 self.ce_loc = ce_loc for i in range(depth): ce_keep_ratio_i = 1.0 if ce_loc is not None and i in ce_loc: ce_keep_ratio_i = ce_keep_ratio[ce_index] ce_index += 1
blocks.append( CEBlock( dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer, keep_ratio_search=ce_keep_ratio_i) )
self.blocks = nn.Sequential(blocks) self.norm = norm_layer(embed_dim)
self.init_weights(weight_init) def masking_fea(self,z, event_z, x, event_x, ratio=0.8 ): b,nz,c = z.shape b,nez,c = event_z.shape b,nx,c = x.shape b,nex,c = event_x.shape assert(nz == nez) assert(nx == nex) lenz_out = int(nzratio) lenx_out = int(nxratio)
mask_nz = torch.rand(b,nz).float() mask_ez = torch.rand(b,nez).float() mask_nx = torch.rand(b,nx).float() mask_ex = torch.rand(b,nex).float() mask_nz = mask_nz>0.4 mask_ez = mask_ez>0.4 mask_ez = ~mask_nz + mask_ez mask_nz_idx = mask_nz.float().sort(1,descending=True)[-1].to(device = z.device) mask_ez_idx = mask_ez.float().sort(1,descending=True)[-1].to(device = z.device) mask_nx = mask_nx>0.4 mask_ex = mask_ex>0.4 mask_ex = ~mask_nx + mask_ex mask_nx_idx = mask_nx.float().sort(1,descending=True)[-1].to(device = z.device) mask_ex_idx = mask_ex.float().sort(1,descending=True)[-1].to(device = z.device)
masked_z = torch.gather(z, 1, mask_nz_idx[:,:lenz_out,None].repeat([1,1,c])) masked_ez = torch.gather(event_z, 1, mask_ez_idx[:,:lenz_out,None].repeat([1,1,c])) masked_x = torch.gather(x, 1, mask_nx_idx[:,:lenx_out,None].repeat([1,1,c])) masked_ex = torch.gather(event_x, 1, mask_ex_idx[:,:lenx_out,None].repeat([1,1,c])) return masked_z, masked_ez, masked_x, masked_ex,{‘x1’:mask_nx_idx[:,:lenx_out],‘x0’:mask_nx_idx[:,lenx_out:], ‘ex1’:mask_ex_idx[:,:lenx_out],‘ex0’:mask_ex_idx[:,lenx_out:], }
def forward_features(self, z, x, event_z, event_x, mask_z=None, mask_x=None, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False,Track=False ): B, H, W = x.shape[0], x.shape[2], x.shape[3]
print(‘shape of event_z before projection:{}, event_x:{}’.format(event_z.shape, event_x.shape))
event_z = self.pos_embed_event(event_z) # [:,:,:,:1000] event_x = self.pos_embed_event(event_x) # B 768 1024 x = self.patch_embed(x) z = self.patch_embed(z)
print(‘shape of event_z:{}, event_x:{}, x:{}, z:{}’.format(event_z.shape,event_x.shape,x.shape,z.shape ))
event_z += self.pos_embed_z event_x += self.pos_embed_x z += self.pos_embed_z x += self.pos_embed_x
attention mask handling # B, H, W
if mask_z is not None and mask_x is not None: mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_z = mask_z.flatten(1).unsqueeze(-1)
mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_x = mask_x.flatten(1).unsqueeze(-1)
mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode) mask_x = mask_x.squeeze(-1)
if self.add_cls_token: cls_tokens = self.cls_token.expand(B, -1, -1) cls_tokens = cls_tokens + self.cls_pos_embed
if self.add_sep_seg: x += self.search_segment_pos_embed z += self.template_segment_pos_embed if Track == False: z, event_z, x, event_x, token_idx = self.masking_fea(z, event_z, x, event_x, ratio=0.9) x = combine_tokens(z, event_z, x, event_x, mode=self.cat_mode) # 64+64+256+256=640
x = combine_tokens(z, x, event_z, event_x, mode=self.cat_mode) # 64+64+256+256=640
if self.add_cls_token: x = torch.cat([cls_tokens, x], dim=1)
x = self.pos_drop(x)
lens_z = self.pos_embed_z.shape[1]
lens_x = self.pos_embed_x.shape[1]
lens_z = z.shape[1] lens_x = x.shape[1] global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device) global_index_t = global_index_t.repeat(B, 1)
global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device) global_index_s = global_index_s.repeat(B, 1) removed_indexes_s = [] out_attn = [] out_attn5 = [] fifth_layer_attn = None for i, blk in enumerate(self.blocks):
out_global_s.append(global_index_s)
out_global_t.append(global_index_t)
x, global_index_t, global_index_s, removed_index_s, attn = blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate)
if i == 4:
fifth_layer_attn = attn
if self.ce_loc is not None and i in self.ce_loc: removed_indexes_s.append(removed_index_s) out_attn.append(attn) out_attn5.append(fifth_layer_attn)
out_attn_idx = random.choice(np.arange(len(out_attn))) out_attn = out_attn[out_attn_idx] x = self.norm(x) lens_x_new = global_index_s.shape[1] lens_z_new = global_index_t.shape[1]
z = x[:, :lens_z_new2] x = x[:, lens_z_new*2:] if removed_indexes_s and removed_indexes_s[0] is not None: removed_indexes_cat = torch.cat(removed_indexes_s, dim=1)
pruned_lens_x = lens_x - lens_x_new pad_x = torch.zeros([B, pruned_lens_x, x.shape[2]], device=x.device) x = torch.cat([x, pad_x], dim=1) index_all = torch.cat([global_index_s, removed_indexes_cat], dim=1)
recover original token order
C = x.shape[-1] x = torch.zeros_like(x).scatter(dim=1, index=index_all.unsqueeze(-1).expand(B, -1, C).to(torch.int64), src=x)
x = recover_tokens(x, lens_z_new, lens_x, mode=self.cat_mode)
event_x = x[:, lens_x:] # RGB head
x = x[:, :lens_x] # RGB head x = torch.cat([event_x, x], dim=1)
aux_dict = {
“attn”: attn,
“attn”: out_attn,
“fifth_layer_attn”: out_attn5,
“removed_indexes_s”: removed_indexes_s, # used for visualization }
return x, aux_dict
def forward(self, z, x, event_z, event_x, ce_template_mask=None, ce_keep_rate=None, tnc_keep_rate=None, return_last_attn=False,Track=False):
x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,Track=Track)
return x, aux_dict
def _create_vision_transformer(pretrained=False, **kwargs): model = VisionTransformerCE(**kwargs)
if pretrained: if ‘npz’ in pretrained: model.load_pretrained(pretrained, prefix=‘’) else: checkpoint = torch.load(pretrained, map_location=“cpu”) missing_keys, unexpected_keys = model.load_state_dict(checkpoint[“model”], strict=False) print(‘Load pretrained model from: ’ + pretrained)
return model
def vit_base_patch16_224_ce(pretrained=False, **kwargs): “”" ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). “”“ model_kwargs = dict( patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model
def vit_large_patch16_224_ce(pretrained=False, **kwargs): “”” ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). “”“ model_kwargs = dict( patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model 损失是:from . import BaseActor from lib.utils.misc import NestedTensor from lib.utils.box_ops import box_cxcywh_to_xyxy, box_xywh_to_xyxy import torch from lib.utils.merge import merge_template_search from …utils.heapmap_utils import generate_heatmap from …utils.ce_utils import generate_mask_cond, adjust_keep_rate import matplotlib.pyplot as plt import numpy as np import torch.nn.functional as F from skimage.measure import label, regionprops import matplotlib.pyplot as plt import matplotlib.patches as patches import numpy as np class CEUTrackActor(BaseActor): “”” Actor for training CEUTrack models “”“
def init(self, net, objective, loss_weight, settings, cfg=None): super().init(net, objective) self.loss_weight = loss_weight self.settings = settings self.bs = self.settings.batchsize # batch size self.cfg = cfg
def call(self, data): “”” args: data - The input data, should contain the fields ‘template’, ‘search’, ‘gt_bbox’. template_images: (N_t, batch, 3, H, W) search_images: (N_s, batch, 3, H, W) returns: loss - the training loss status - dict containing detailed losses “”"
forward pass
out_dict = self.forward_pass(data)
compute losses
loss, status = self.compute_losses(out_dict, data)
return loss, status
def forward_pass(self, data):
currently only support 1 template and 1 search region
assert len(data[‘template_images’]) == 1 assert len(data[‘search_images’]) == 1 assert len(data[‘template_event’]) == 1 assert len(data[‘search_event’]) == 1
template_list = [] for i in range(self.settings.num_template): template_img_i = data[‘template_images’][i].view(-1, data[‘template_images’].shape[2:]) # (batch, 3, 128, 128)
template_att_i = data[‘template_att’][i].view(-1, data[‘template_att’].shape[2:]) # (batch, 128, 128)
template_list.append(template_img_i)
search_img = data[‘search_images’][0].view(-1, data[‘search_images’].shape[2:]) # (batch, 3, 320, 320)
search_att = data[‘search_att’][0].view(-1, data[‘search_att’].shape[2:]) # (batch, 320, 320)
template_event = data[‘template_event’][0].view(-1, data[‘template_event’].shape[2:]) search_event = data[‘search_event’][0].view(-1, data[‘search_event’].shape[2:])
box_mask_z = None ce_keep_rate = None if self.cfg.MODEL.BACKBONE.CE_LOC: box_mask_z = generate_mask_cond(self.cfg, template_list[0].shape[0], template_list[0].device, data[‘template_anno’][0])
ce_start_epoch = self.cfg.TRAIN.CE_START_EPOCH ce_warm_epoch = self.cfg.TRAIN.CE_WARM_EPOCH ce_keep_rate = adjust_keep_rate(data[‘epoch’], warmup_epochs=ce_start_epoch, total_epochs=ce_start_epoch + ce_warm_epoch, ITERS_PER_EPOCH=1, base_keep_rate=self.cfg.MODEL.BACKBONE.CE_KEEP_RATIO[0])
if len(template_list) == 1: template_list = template_list[0]
out_dict = self.net(template=template_list, search=search_img, event_template=template_event, event_search=search_event, ce_template_mask=box_mask_z, ce_keep_rate=ce_keep_rate, return_last_attn=False)
return out_dict
def compute_losses(self, pred_dict, gt_dict, return_status=True):
gt gaussian map
gt_bbox = gt_dict[‘search_anno’][-1] # (Ns, batch, 4) (x1,y1,w,h) -> (batch, 4) gt_gaussian_maps = generate_heatmap(gt_dict[‘search_anno’], self.cfg.DATA.SEARCH.SIZE, self.cfg.MODEL.BACKBONE.STRIDE) gt_gaussian_maps = gt_gaussian_maps[-1].unsqueeze(1)
Get boxes
pred_boxes = pred_dict[‘pred_boxes’] if torch.isnan(pred_boxes).any(): raise ValueError(“Network outputs is NAN! Stop Training”) num_queries = pred_boxes.size(1) pred_boxes_vec = box_cxcywh_to_xyxy(pred_boxes).view(-1, 4) # (B,N,4) --> (BN,4) (x1,y1,x2,y2) gt_boxes_vec = box_xywh_to_xyxy(gt_bbox)[:, None, :].repeat((1, num_queries, 1)).view(-1, 4).clamp(min=0.0, max=1.0) # (B,4) --> (B,1,4) --> (B,N,4)
compute giou and iou
try: giou_loss, iou = self.objective[‘giou’](pred_boxes_vec, gt_boxes_vec) # (BN,4) (BN,4) except: giou_loss, iou = torch.tensor(0.0).cuda(), torch.tensor(0.0).cuda()
compute l1 loss
l1_loss = self.objective[‘l1’](pred_boxes_vec, gt_boxes_vec) # (BN,4) (BN,4)
compute location loss
if ‘score_map’ in pred_dict: location_loss = self.objective[‘focal’](pred_dict[‘score_map’], gt_gaussian_maps) else: location_loss = torch.tensor(0.0, device=l1_loss.device)
按照HRCEUTrack,新增loss约束
rank_rgb_loss = self.rank_rgb_loss(pred_dict,gt_dict[‘search_anno’], gt_dict[‘template_anno’])
rank_event_loss= self.rank_event_loss(pred_dict,gt_dict[‘search_anno’], gt_dict[‘template_anno’])
#######
计算特征一致性损失
feature_consistency_loss = F.mse_loss(pred_dict[‘A_enhanced_x’], pred_dict[‘A_original_x’]) + F.mse_loss(pred_dict[‘A_enhanced_event_x’], pred_dict[‘A_original_event_x’])
# 标量化特征一致性损失
feature_consistency_loss = feature_consistency_loss.mean()
weighted sum
loss = self.loss_weight[‘giou’] * giou_loss + self.loss_weight[‘l1’] * l1_loss + self.loss_weight[‘focal’] * location_loss + rank_event_loss * 1.2 # + 1 * feature_consistency_loss if return_status:
status for log
mean_iou = iou.detach().mean() status = {“Loss/total”: loss.item(), “Loss/giou”: giou_loss.item(), “Loss/l1”: l1_loss.item(), “Loss/location”: location_loss.item(), #‘Loss/feature_consistency’: feature_consistency_loss.item(),
“rank_rgb_loss”: rank_rgb_loss.item(),
“rank_event_loss”: rank_event_loss.item(), “IoU”: mean_iou.item()} return loss, status else: return loss
#################################### 按照 HRCEUTrack新增 loss #########################################
def _random_permute(self,matrix):
matrix = random.choice(matrix)
b, c, h, w = matrix.shape idx = [ torch.randperm©.to(matrix.device) for i in range(b)] idx = torch.stack(idx, dim=0)[:, :, None, None].repeat([1,1,h,w])
idx = torch.randperm©[None,:,None,None].repeat([b,1,h,w]).to(matrix.device)
matrix01 = torch.gather(matrix, 1, idx) return matrix01 def crop_flag(self, flag, global_index_s, global_index_t,H1 = 64, H2 = 256): B,Ls = global_index_s.shape B, Lt = global_index_t.shape B,C,L1,L2 = flag.shape flag_t = flag[:,:,:H1,:] flag_s = flag[:,:,H1:,:]
flag_t = torch.gather(flag_t,2,global_index_t[:,None,:,None].repeat([1,C,1,L2]).long()) flag_s = torch.gather(flag_s,2,global_index_s[:,None,:,None].repeat([1,C,1,L2]).long()) flag = torch.cat([flag_t, flag_s], dim = 2)
flag_t = flag[:,:,:,:H1] flag_s = flag[:,:,:,H1:] flag_t = torch.gather(flag_t,3,global_index_t[:,None,None,:].repeat([1,C,int(Ls+Lt),1]).long()) flag_s = torch.gather(flag_s,3,global_index_s[:,None,None,:].repeat([1,C,int(Ls+Lt),1]).long()) flag = torch.cat([flag_t, flag_s], dim = 3) B, C, L11, L12 = flag.shape try: assert(L11 == int(Lt + Ls)) assert(L12 == int(Lt + Ls)) except: print(‘L11:{}, L12:{}, L1:{}, L2:{}’.format(L11, L12, L1, L2)) return flag def crop_fusion(self, flag, attn, global_index_s, global_index_t,H1 = 64, H2 = 256 ): flag = self.crop_flag(flag=flag, global_index_s=global_index_s, global_index_t=global_index_t) B,C,L1,L2 = flag.shape Ba, Ca, La, La2 = attn.shape _,idx1 = flag.mean(dim=3,keepdim=False).sort(dim=2,descending=True)
print(‘shape of flag:{}, idx1:{}’.format(flag.shape, idx1[:,:,:32,None].repeat([1,Ca,1,L2]).shape))
flag = torch.gather(flag,2,idx1[:,:,:32,None].repeat([1,C,1,L2]).long()) attn = torch.gather(attn,2,idx1[:,:,:32,None].repeat([1,Ca,1,L2]).long()) _,idx2 = flag.mean(dim=2,keepdim=False).sort(dim=2,descending=True) flag = torch.gather(flag,3,idx2[:,:,None,:32].repeat([1,C,32,1]).long()) attn = torch.gather(attn,3,idx2[:,:,None,:32].repeat([1,Ca,32,1]).long()) return attn * flag
def loss_rank_ori(self, outputs, targetsi, temp_annoi=None): “”“Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss targets dicts must contain the key “boxes” containing a tensor of dim [nb_target_boxes, 4] The target boxes are expected in format (center_x, center_y, h, w), normalized by the image size. “”” attn = outputs[‘attn’]
print(‘attn shape:’,attn.shape)
attn1 = torch.cat([attn[:,:,114:344,57:114], attn[:,:,114:344,344:]],dim=3) attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True)
print(‘attn1 shape’,attn1.shape)
attn2 = torch.cat([attn[:,:,344:,:57], attn[:,:,344:,114:344]],dim=3) attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True)
print(‘attn2 shape’,attn2.shape)
attn = torch.cat([attn1, attn2],dim=1) B, C, H, W = attn.shape ,s1, = torch.svd(attn.reshape([BC, H, W])) s01 = torch.abs(s1 - 1) return torch.mean(s01)
def rank_rgb_loss(self, outputs, targetsi, temp_annoi=None):
attn = outputs[‘attn’]
# print(‘attn.shape’,attn.shape) # attn.shape torch.Size([2, 12, 320, 320])
attn1 = torch.cat([attn[:,:,57:172,28:57], attn[:,:,57:172,172:]], dim=3)
attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True)
attn2 = torch.cat([attn[:,:,172:,:28], attn[:,:,172:,57:172]], dim=3)
attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True)
attn2 = F.interpolate(attn2, size=(115, 144), mode=‘bilinear’, align_corners=False)
attn = torch.cat([attn1, attn2], dim=1)
B, C, H, W = attn.shape
_, s1, _ = torch.svd(attn.reshape([BC, H, W]))
s01 = torch.abs(s1 - 1)
return torch.mean(s01)
def rank_event_loss(self, outputs, targetsi, temp_annoi=None): attn = outputs[‘attn’] print('‘) print(attn) print(’***') print(‘attn.shape’,attn.shape) attn_show = attn # 复制一份attn用于显示
整体归一化处理
attn_min = attn_show.min(dim=2, keepdim=True)[0].min(dim=3, keepdim=True)[0] attn_max = attn_show.max(dim=2, keepdim=True)[0].max(dim=3, keepdim=True)[0] attn_show = (attn_show - attn_min) / (attn_max - attn_min)
attn1 = torch.cat([attn[:,:,114:344,57:114], attn[:,:,114:344,344:]],dim=3) attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) attn2 = torch.cat([attn[:,:,344:,:57], attn[:,:,344:,114:344]],dim=3) attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True)
attn1 = torch.cat([attn[:,:,57:172,28:57], attn[:,:,57:172,172:]], dim=3)
attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True)
attn2 = torch.cat([attn[:,:,172:,:28], attn[:,:,172:,57:172]], dim=3)
attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True)
attn2 = F.interpolate(attn2, size=(115, 144), mode=‘bilinear’, align_corners=False)
attn = torch.cat([attn1, attn2], dim=1) B, C, H, W = attn.shape _, s1, _ = torch.svd(attn.reshape([BC, H, W])) s02 = torch.abs(s1 - 1)
设置阈值找到高响应区域
threshold = 0.8 * attn_show.max().item() high_resp_mask = attn_show > threshold
labeled, num_features = label(high_resp_mask[0].cpu().numpy())
labeled, num_features = label(high_resp_mask[0].cpu().numpy(), return_num=True)
输出高响应区域的边界框
props = regionprops(labeled) plt.figure(figsize=(10, 10)) ax = plt.gca() for prop in props: bbox = prop.bbox print(f’High response region: {bbox}’) # 输出边界框信息 rect = patches.Rectangle((bbox[1], bbox[0]), bbox[3] - bbox[1], bbox[2] - bbox[0], linewidth=2, edgecolor=‘r’, facecolor=‘none’) ax.add_patch(rect)
展示注意力图和标记的高响应区域
for i in range(attn_show.shape[1]): plt.subplot(attn_show.shape[1] // 3 + 1, 3, i + 1) plt.imshow(attn_show[0, i].detach().cpu().numpy(), cmap=‘viridis’, interpolation=‘nearest’) plt.title(f’Channel {i+1}') plt.axis(‘off’) plt.suptitle(‘Attention Maps’) plt.show() return torch.mean(s02) ,其中我们将attn可视化之后,发现注意力特征可视化大致可以分为16个部分,然后作者将横轴定义为key,纵轴定义为qurey,然后key分为:模板(RGB),模板(event),搜索(RGB),搜索(event),query轴从上到下分别是模板(RGB),模板(event),搜索(RGB),搜索(event),那么现在就分析注意力矩阵,在attn1 = torch.cat([attn[:,:,114:344,57:114], attn[:,:,114:344,344:]],dim=3) attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) attn2 = torch.cat([attn[:,:,344:,:57], attn[:,:,344:,114:344]],dim=3) attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True)中,作者解释是取的注意力矩阵区域是跨模态交互部分,那么如何确定这个数值?
- Assistant
在您的代码中,注意力矩阵的特定区域选择基于对注意力矩阵结构的理解和模型设计的具体需求。
解析注意力矩阵的结构
注意力矩阵通常表示为
[batch_size, num_heads, query_length, key_length]。对于图像和视觉Transformer,query_length和key_length都可以链接到图像的不同部分或不同的输入模态(例如RGB和event)。在跨模态学习或融合的场景中,不同的区域在矩阵中代表不同类型的交互,如模板与搜索之间,或RGB与event之间的交互。确定注意力矩阵的特定区域
关于您提到的注意力矩阵分区选择(attn1的
[[114:344, 57:114], [114:344, 344:]]和 attn2得[[344:, :57], [344:, 114:344]]):-
选区出处:配合模型结构,这些索引区间可能由数据的特性或任务的具体需求定义。如,假设模型被训练以处理具有不同空间特征的不同模态的数据,那么注意力矩阵中的这些特定切片可能指向负责处理特定交互的部分(如跨模态信息融合)。
-
确立数值含义:
- 值如
114, 344, 57等,可能是基于输入数据的维度、处理后的patch大小或经过多次实验得出的最优解析区段。 - 可以通过实验来判断这些参数。例如,调节这些参数,并通过验证性能(比如交叉验证)来看哪些参数设置可以带来更好的模型性能。
- 值如
实验调整与优化
因为这些数值和区块很可能是针对特定数据集或任务优化的,要找到最合适的值或区块,可能需要按照下述步骤进行尝试:
- 分析注意力热图:将注意力矩阵可视化,查看不同头部的关注点在哪些区域,进一步理解模型是如何在不同的部分强化或忽略信号的。
- A/B 测试:试验不同的切片参数,看看它们在验证集上的表现,选择最优的设置。
- 深入理解数据:查看输入数据的特性可能帮助揭示为什么这些参数或区块是有意义的,可以进一步分析输入图像的尺寸、特征分布等。
结论
至于“如何确定这个数值”,关键在于理解和实验。要准确获取这些值,需要基于深入的数据分析、理解模型结构以及大量的实验验证。如果原始论文或这部分代码的开发者没有给出直接解释,你可能需要自己通过试错和验证来确定最优解。
-
- User
那么现在就需要将attn中不同区域的坐标位置输出,应该如何显示?
- Assistant
若要可视化注意力矩阵中不同区域的坐标位置,你可以采用如下几种方法:
1. 使用热图显示注意力权重
可以利用热图直观展现不同区域在注意力矩阵中的权重分布。这里我们使用
matplotlib库进行显示。import matplotlib.pyplot as plt import numpy as np def plot_attention_map(attention_map, title="Attention Map"): """ 绘制一个注意力矩阵的热图。 Args: attention_map (numpy.ndarray): 注意力权重矩阵。 title (str): 图像的标题。 """ plt.figure(figsize=(10, 8)) plt.imshow(attention_map, cmap='viridis', aspect='auto') plt.colorbar() plt.title(title) plt.xlabel('Key') plt.ylabel('Query') plt.show() # 假设'attn'是一个从模型中提取的注意力张量, attn.shape: [batch_size, n_heads, query_length, key_length] # 选择第一个样本,第一个头 sample_attn = attn[0, 0].detach().cpu().numpy() # 调整索引以选择不同的样本和头部 # 显示完整的注意力矩阵 plot_attention_map(sample_attn, "Complete Attention Map") # 显示特定区域的注意力矩阵 region_1 = sample_attn[114:344, 57:114] plot_attention_map(region_1, "Attention Map Region 1") region_2 = sample_attn[344:, 114:344] plot_attention_map(region_2, "Attention Map Region 2")2. 注释特定区域
在热图上加上特定区域的边界和注释,可以帮助更直观地理解这些区域的意义。
def annotate_region(attention_map, regions, title="Annotated Attention Map"): """ 在注意力矩阵热图上标记特定区域。 Args: attention_map (numpy.ndarray): 注意力权重矩阵。 regions (list of tuples): 每个元组包含(x_min, x_max, y_min, y_max)定义区域边界。 title (str): 图像的标题。 """ plt.figure(figsize=(10, 8)) plt.imshow(attention_map, cmap='viridis', aspect='auto') for x_min, x_max, y_min, y_max in regions: plt.plot([y_min, y_max, y_max, y_min, y_min], [x_min, x_min, x_max, x_max, x_min], 'r-') plt.colorbar() plt.title(title) plt.xlabel('Key') plt.ylabel('Query') plt.show() regions = [ (114, 344, 57, 114), # Region 1 (344, attn.shape[2], 114, 344) # Region 2 ] annotate_region(sample_attn, regions)这两种方法可以帮助你直观地分析和演示注意力矩阵中不同区域的行为和作用。通过调整上述代码中的参数,你可以探索不同样本、不同头部或不同的注意力区块。
- User
在 def rank_event_loss(self, outputs, targetsi, temp_annoi=None): attn = outputs['attn'] attn_show = attn
# 整体归一化处理 attn_min = attn_show.min(dim=2, keepdim=True)[0].min(dim=3, keepdim=True)[0] attn_max = attn_show.max(dim=2, keepdim=True)[0].max(dim=3, keepdim=True)[0] attn_show = (attn_show - attn_min) / (attn_max - attn_min) attn1 = torch.cat([attn[:,:,114:344,57:114], attn[:,:,114:344,344:]],dim=3) attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) attn2 = torch.cat([attn[:,:,344:,:57], attn[:,:,344:,114:344]],dim=3) attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) attn = torch.cat([attn1, attn2], dim=1) B, C, H, W = attn.shape _, s1, _ = torch.svd(attn.reshape([B*C, H, W])) s02 = torch.abs(s1 - 1) threshold = 0.8 * attn_show.max().item() high_resp_mask = attn_show > threshold labeled, num_features = label(high_resp_mask[0].cpu().numpy(), return_num=True) props = regionprops(labeled) plt.figure(figsize=(10, 10)) ax = plt.gca() for prop in props: bbox = prop.bbox print(f'High response region: {bbox}') # 输出边界框信息 rect = patches.Rectangle((bbox[1], bbox[0]), bbox[3] - bbox[1], bbox[2] - bbox[0], linewidth=2, edgecolor='r', facecolor='none') ax.add_patch(rect) # 展示注意力图和标记的高响应区域 for i in range(attn_show.shape[1]): plt.subplot(attn_show.shape[1] // 3 + 1, 3, i + 1) plt.imshow(attn_show[0, i].detach().cpu().numpy(), cmap='viridis', interpolation='nearest') plt.title(f'Channel {i+1}') plt.axis('off') plt.suptitle('Attention Maps') plt.show() return torch.mean(s02)上,假设我们不知道[attn[:,:,114:344,57:114], attn[:,:,114:344,344:]和[attn[:,:,344:,:57], attn[:,:,344:,114:344]],那么仅凭attn如何知道跨模态区域的范围,然后得到attn1和attn2? - Assistant
如果您不知道预先确定的跨模态区域的具体范围,并想从注意力矩阵中发现这些区域,您可以尝试以下几种方法来探索和识别区域:
1. 可视化注意力热图
首先,通过可视化整个注意力矩阵的热图来观察哪些区域有较高的响应。这可以帮助直观地确定潜在的重要交互区域。
import matplotlib.pyplot as plt def visualize_attention_heatmap(attn): for head_index in range(attn.shape[1]): # 遍历所有头 plt.figure(figsize=(6, 5)) plt.imshow(attn[0, head_index].detach().cpu(), cmap='viridis') plt.colorbar() plt.title(f'Head {head_index + 1}') plt.xlabel('Key') plt.ylabel('Query') plt.show() visualize_attention_heatmap(outputs['attn'])2. 自动识别区域
自动识别高响应区域可以通过应用不同的图像处理技术如阈值、轮廓发现等来实现:
from skimage.filters import threshold_otsu from skimage.measure import label, regionprops def identify_high_response_regions(attn, display=False): high_resp_regions = [] attn = attn.detach().cpu().numpy() thresh = threshold_otsu(attn) binary_mask = attn > thresh labeled_mask = label(binary_mask) regions = regionprops(labeled_mask) for region in regions: if region.area >= 10: # 可以调整最小区域大小 high_resp_regions.append(region.bbox) if display: print(f'Region bounding box (min_row, min_col, max_row, max_col): {region.bbox}') return high_resp_regions high_resp_regions = identify_high_response_regions(outputs['attn'][0][0], True) # 查看第一个样本第一个头3. 使用机器学习聚类方法
使用如k-means, DBSCAN等聚类算法来分组注意力矩阵中的相似区域,可能可以暴露出有意义的跨模态区。
from sklearn.cluster import KMeans def cluster_attention_regions(attn, n_clusters=5): reshaped_attn = attn.reshape(-1, attn.shape[-1]) # 将矩阵平铺 kmeans = KMeans(n_clusters=n_clusters) labels = kmeans.fit_predict(reshaped_attn) clustered = labels.reshape(attn.shape[:-1]) # 重塑回原来的二维形状 return clustered # 可视化聚类结果 clusters = cluster_attention_regions(outputs['attn'][0, 0].detach().cpu().numpy()) plt.imshow(clusters, cmap='viridis') plt.colorbar() plt.title('Clustered Attention Regions') plt.show()结合专业知识
这些方法可以为您提供一个起点,但最佳的区域选择可能需要依据具体的应用场景和模型结构进行调整。结合模型的具体设计和任务需求(如模态间的互操作性),以及通过实验反复验证来确定最有效的方法。