模型1是:import math import logging from functools import partial from collections import…
- User
模型1是:import math import logging from functools import partial from collections import OrderedDict from copy import deepcopy
import torch import torch.nn as nn import torch.nn.functional as F
from timm.models.layers import to_2tuple
from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise from .utils import combine_tokens, recover_tokens from .vit import VisionTransformer from ..layers.attn_blocks import CEBlock import random import numpy as np
_logger = logging.getLogger(name)
class VisionTransformerCE(VisionTransformer): """ Vision Transformer with candidate elimination (CE) module
A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` - https://arxiv.org/abs/2010.11929 Includes distillation token & head support for `DeiT: Data-efficient Image Transformers` - https://arxiv.org/abs/2012.12877 """ def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None, act_layer=None, weight_init='', ce_loc=None, ce_keep_ratio=None): """ Args: img_size (int, tuple): input image size patch_size (int, tuple): patch size in_chans (int): number of input channels num_classes (int): number of classes for classification head embed_dim (int): embedding dimension depth (int): depth of transformer num_heads (int): number of attention heads mlp_ratio (int): ratio of mlp hidden dim to embedding dim qkv_bias (bool): enable bias for qkv if True representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set distilled (bool): model includes a distillation token and head as in DeiT models drop_rate (float): dropout rate attn_drop_rate (float): attention dropout rate drop_path_rate (float): stochastic depth rate embed_layer (nn.Module): patch embedding layer norm_layer: (nn.Module): normalization layer weight_init: (str): weight init scheme """ # super().__init__() super().__init__() if isinstance(img_size, tuple): self.img_size = img_size else: self.img_size = to_2tuple(img_size) self.patch_size = patch_size self.in_chans = in_chans self.num_classes = num_classes self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models self.num_tokens = 2 if distilled else 1 norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) act_layer = act_layer or nn.GELU self.patch_embed = embed_layer( img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4) # self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4) # self.pos_embed_event_z = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=3, stride=1) # attn = CrossAttn(768, 4, 3072, 0.1, 'relu') # self.cross_attn = Iter_attn(attn, 2) dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule blocks = [] ce_index = 0 self.ce_loc = ce_loc for i in range(depth): ce_keep_ratio_i = 1.0 if ce_loc is not None and i in ce_loc: ce_keep_ratio_i = ce_keep_ratio[ce_index] ce_index += 1 blocks.append( CEBlock( dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer, keep_ratio_search=ce_keep_ratio_i) ) self.blocks = nn.Sequential(*blocks) self.norm = norm_layer(embed_dim) self.init_weights(weight_init) def masking_fea(self,z, event_z, x, event_x, ratio=0.8 ): b,nz,c = z.shape b,nez,c = event_z.shape b,nx,c = x.shape b,nex,c = event_x.shape assert(nz == nez) assert(nx == nex) lenz_out = int(nz*ratio) lenx_out = int(nx*ratio) mask_nz = torch.rand(b,nz).float() mask_ez = torch.rand(b,nez).float() mask_nx = torch.rand(b,nx).float() mask_ex = torch.rand(b,nex).float() mask_nz = mask_nz>0.4 mask_ez = mask_ez>0.4 mask_ez = ~mask_nz + mask_ez mask_nz_idx = mask_nz.float().sort(1,descending=True)[-1].to(device = z.device) mask_ez_idx = mask_ez.float().sort(1,descending=True)[-1].to(device = z.device) mask_nx = mask_nx>0.4 mask_ex = mask_ex>0.4 mask_ex = ~mask_nx + mask_ex mask_nx_idx = mask_nx.float().sort(1,descending=True)[-1].to(device = z.device) mask_ex_idx = mask_ex.float().sort(1,descending=True)[-1].to(device = z.device) masked_z = torch.gather(z, 1, mask_nz_idx[:,:lenz_out,None].repeat([1,1,c])) masked_ez = torch.gather(event_z, 1, mask_ez_idx[:,:lenz_out,None].repeat([1,1,c])) masked_x = torch.gather(x, 1, mask_nx_idx[:,:lenx_out,None].repeat([1,1,c])) masked_ex = torch.gather(event_x, 1, mask_ex_idx[:,:lenx_out,None].repeat([1,1,c])) return masked_z, masked_ez, masked_x, masked_ex,{'x1':mask_nx_idx[:,:lenx_out],'x0':mask_nx_idx[:,lenx_out:], 'ex1':mask_ex_idx[:,:lenx_out],'ex0':mask_ex_idx[:,lenx_out:], } def forward_features(self, z, x, event_z, event_x, mask_z=None, mask_x=None, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False,Track=False ): B, H, W = x.shape[0], x.shape[2], x.shape[3] # print('shape of event_z before projection:{}, event_x:{}'.format(event_z.shape, event_x.shape)) event_z = self.pos_embed_event(event_z) # [:,:,:,:1000] event_x = self.pos_embed_event(event_x) # B 768 1024 x = self.patch_embed(x) z = self.patch_embed(z) # print('shape of event_z:{}, event_x:{}, x:{}, z:{}'.format(event_z.shape,event_x.shape,x.shape,z.shape )) event_z += self.pos_embed_z event_x += self.pos_embed_x z += self.pos_embed_z x += self.pos_embed_x # attention mask handling # B, H, W if mask_z is not None and mask_x is not None: mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_z = mask_z.flatten(1).unsqueeze(-1) mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_x = mask_x.flatten(1).unsqueeze(-1) mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode) mask_x = mask_x.squeeze(-1) if self.add_cls_token: cls_tokens = self.cls_token.expand(B, -1, -1) cls_tokens = cls_tokens + self.cls_pos_embed if self.add_sep_seg: x += self.search_segment_pos_embed z += self.template_segment_pos_embed if Track == False: z, event_z, x, event_x, token_idx = self.masking_fea(z, event_z, x, event_x, ratio=0.9) x = combine_tokens(z, event_z, x, event_x, mode=self.cat_mode) # 64+64+256+256=640 # x = combine_tokens(z, x, event_z, event_x, mode=self.cat_mode) # 64+64+256+256=640 if self.add_cls_token: x = torch.cat([cls_tokens, x], dim=1) x = self.pos_drop(x) # lens_z = self.pos_embed_z.shape[1] # lens_x = self.pos_embed_x.shape[1] lens_z = z.shape[1] lens_x = x.shape[1] global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device) global_index_t = global_index_t.repeat(B, 1) global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device) global_index_s = global_index_s.repeat(B, 1) removed_indexes_s = [] out_attn = [] for i, blk in enumerate(self.blocks): # out_global_s.append(global_index_s) # out_global_t.append(global_index_t) x, global_index_t, global_index_s, removed_index_s, attn = \ blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate) if self.ce_loc is not None and i in self.ce_loc: removed_indexes_s.append(removed_index_s) out_attn.append(attn) # print('shape of attn:{}, lens_z:{}, lens_x:{}'.format(attn.shape, lens_z, lens_x)) out_attn_idx = random.choice(np.arange(len(out_attn))) out_attn = out_attn[out_attn_idx] x = self.norm(x) lens_x_new = global_index_s.shape[1] lens_z_new = global_index_t.shape[1] z = x[:, :lens_z_new*2] x = x[:, lens_z_new*2:] if Track == False: idx1 = token_idx['x1'] idx0 = token_idx['x0'] idex1 = token_idx['ex1'] idex0 = token_idx['ex0'] ex = x[:,idex1.shape[1]:] x = x[:,:idex1.shape[1]] # if removed_indexes_s and removed_indexes_s[0] is not None: # removed_indexes_cat = torch.cat(removed_indexes_s, dim=1) pruned_lens_x = idx0.shape[1] pad_x = torch.zeros([B, pruned_lens_x, x.shape[2]], device=x.device) x = torch.cat([x, pad_x], dim=1) index_all = torch.cat([idx1, idx0], dim=1) # recover original token order C = x.shape[-1] x = torch.zeros_like(x).scatter_(dim=1, index=index_all.unsqueeze(-1).expand(B, -1, C).to(torch.int64), src=x) ex = torch.cat([ex, pad_x], dim=1) index_all = torch.cat([idex1, idex0], dim=1) # recover original token order C = ex.shape[-1] ex = torch.zeros_like(ex).scatter_(dim=1, index=index_all.unsqueeze(-1).expand(B, -1, C).to(torch.int64), src=ex) x = torch.cat([x,ex],dim=1) x = recover_tokens(x, lens_z_new, lens_x, mode=self.cat_mode) event_x = x[:, lens_x:] # RGB head x = x[:, :lens_x] # RGB head x = torch.cat([event_x, x], dim=1) aux_dict = { # "attn": attn, "attn": out_attn, "removed_indexes_s": removed_indexes_s, # used for visualization } return x, aux_dict def forward(self, z, x, event_z, event_x, ce_template_mask=None, ce_keep_rate=None, tnc_keep_rate=None, return_last_attn=False,Track=False): x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,Track=Track) return x, aux_dictdef _create_vision_transformer(pretrained=False, **kwargs): model = VisionTransformerCE(**kwargs)
if pretrained: if 'npz' in pretrained: model.load_pretrained(pretrained, prefix='') else: checkpoint = torch.load(pretrained, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False) print('Load pretrained model from: ' + pretrained) return modeldef vit_base_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model 对应的loss是:from . import BaseActor from lib.utils.misc import NestedTensor from lib.utils.box_ops import box_cxcywh_to_xyxy, box_xywh_to_xyxy import torch from lib.utils.merge import merge_template_search from ...utils.heapmap_utils import generate_heatmap from ...utils.ce_utils import generate_mask_cond, adjust_keep_rate
class CEUTrackActor(BaseActor): """ Actor for training CEUTrack models """
def __init__(self, net, objective, loss_weight, settings, cfg=None): super().__init__(net, objective) self.loss_weight = loss_weight self.settings = settings self.bs = self.settings.batchsize # batch size self.cfg = cfg def __call__(self, data): """ args: data - The input data, should contain the fields 'template', 'search', 'gt_bbox'. template_images: (N_t, batch, 3, H, W) search_images: (N_s, batch, 3, H, W) returns: loss - the training loss status - dict containing detailed losses """ # forward pass out_dict = self.forward_pass(data) # compute losses loss, status = self.compute_losses(out_dict, data) return loss, status def forward_pass(self, data): # currently only support 1 template and 1 search region assert len(data['template_images']) == 1 assert len(data['search_images']) == 1 assert len(data['template_event']) == 1 assert len(data['search_event']) == 1 template_list = [] for i in range(self.settings.num_template): template_img_i = data['template_images'][i].view(-1, *data['template_images'].shape[2:]) # (batch, 3, 128, 128) # template_att_i = data['template_att'][i].view(-1, *data['template_att'].shape[2:]) # (batch, 128, 128) template_list.append(template_img_i) search_img = data['search_images'][0].view(-1, *data['search_images'].shape[2:]) # (batch, 3, 320, 320) # search_att = data['search_att'][0].view(-1, *data['search_att'].shape[2:]) # (batch, 320, 320) template_event = data['template_event'][0].view(-1, *data['template_event'].shape[2:]) search_event = data['search_event'][0].view(-1, *data['search_event'].shape[2:]) box_mask_z = None ce_keep_rate = None if self.cfg.MODEL.BACKBONE.CE_LOC: box_mask_z = generate_mask_cond(self.cfg, template_list[0].shape[0], template_list[0].device, data['template_anno'][0]) ce_start_epoch = self.cfg.TRAIN.CE_START_EPOCH ce_warm_epoch = self.cfg.TRAIN.CE_WARM_EPOCH ce_keep_rate = adjust_keep_rate(data['epoch'], warmup_epochs=ce_start_epoch, total_epochs=ce_start_epoch + ce_warm_epoch, ITERS_PER_EPOCH=1, base_keep_rate=self.cfg.MODEL.BACKBONE.CE_KEEP_RATIO[0]) if len(template_list) == 1: template_list = template_list[0] out_dict = self.net(template=template_list, search=search_img, event_template=template_event, event_search=search_event, ce_template_mask=box_mask_z, ce_keep_rate=ce_keep_rate, return_last_attn=False) return out_dict def compute_losses(self, pred_dict, gt_dict, return_status=True): # gt gaussian map gt_bbox = gt_dict['search_anno'][-1] # (Ns, batch, 4) (x1,y1,w,h) -> (batch, 4) gt_gaussian_maps = generate_heatmap(gt_dict['search_anno'], self.cfg.DATA.SEARCH.SIZE, self.cfg.MODEL.BACKBONE.STRIDE) gt_gaussian_maps = gt_gaussian_maps[-1].unsqueeze(1) # Get boxes pred_boxes = pred_dict['pred_boxes'] if torch.isnan(pred_boxes).any(): raise ValueError("Network outputs is NAN! Stop Training") num_queries = pred_boxes.size(1) pred_boxes_vec = box_cxcywh_to_xyxy(pred_boxes).view(-1, 4) # (B,N,4) --> (BN,4) (x1,y1,x2,y2) gt_boxes_vec = box_xywh_to_xyxy(gt_bbox)[:, None, :].repeat((1, num_queries, 1)).view(-1, 4).clamp(min=0.0, max=1.0) # (B,4) --> (B,1,4) --> (B,N,4) # compute giou and iou try: giou_loss, iou = self.objective['giou'](pred_boxes_vec, gt_boxes_vec) # (BN,4) (BN,4) except: giou_loss, iou = torch.tensor(0.0).cuda(), torch.tensor(0.0).cuda() # compute l1 loss l1_loss = self.objective['l1'](pred_boxes_vec, gt_boxes_vec) # (BN,4) (BN,4) # compute location loss if 'score_map' in pred_dict: location_loss = self.objective['focal'](pred_dict['score_map'], gt_gaussian_maps) else: location_loss = torch.tensor(0.0, device=l1_loss.device) rank_loss = self.loss_rank(pred_dict,gt_dict['search_anno'], gt_dict['template_anno']) # weighted sum loss = self.loss_weight['giou'] * giou_loss + self.loss_weight['l1'] * l1_loss + self.loss_weight['focal'] * location_loss + rank_loss*1.2 if return_status: # status for log mean_iou = iou.detach().mean() status = {"Loss/total": loss.item(), "Loss/giou": giou_loss.item(), "Loss/l1": l1_loss.item(), "Loss/location": location_loss.item(), "IoU": mean_iou.item()} return loss, status else: return loss def _random_permute(self,matrix): # matrix = random.choice(matrix) b, c, h, w = matrix.shape idx = [ torch.randperm(c).to(matrix.device) for i in range(b)] idx = torch.stack(idx, dim=0)[:, :, None, None].repeat([1,1,h,w]) # idx = torch.randperm(c)[None,:,None,None].repeat([b,1,h,w]).to(matrix.device) matrix01 = torch.gather(matrix, 1, idx) return matrix01 def crop_flag(self, flag, global_index_s, global_index_t,H1 = 64, H2 = 256): B,Ls = global_index_s.shape B, Lt = global_index_t.shape B,C,L1,L2 = flag.shape flag_t = flag[:,:,:H1,:] flag_s = flag[:,:,H1:,:] flag_t = torch.gather(flag_t,2,global_index_t[:,None,:,None].repeat([1,C,1,L2]).long()) flag_s = torch.gather(flag_s,2,global_index_s[:,None,:,None].repeat([1,C,1,L2]).long()) flag = torch.cat([flag_t, flag_s], dim = 2) flag_t = flag[:,:,:,:H1] flag_s = flag[:,:,:,H1:] flag_t = torch.gather(flag_t,3,global_index_t[:,None,None,:].repeat([1,C,int(Ls+Lt),1]).long()) flag_s = torch.gather(flag_s,3,global_index_s[:,None,None,:].repeat([1,C,int(Ls+Lt),1]).long()) flag = torch.cat([flag_t, flag_s], dim = 3) B, C, L11, L12 = flag.shape try: assert(L11 == int(Lt + Ls)) assert(L12 == int(Lt + Ls)) except: print('L11:{}, L12:{}, L1:{}, L2:{}'.format(L11, L12, L1, L2)) return flag def crop_fusion(self, flag, attn, global_index_s, global_index_t,H1 = 64, H2 = 256 ): flag = self.crop_flag(flag=flag, global_index_s=global_index_s, global_index_t=global_index_t) B,C,L1,L2 = flag.shape Ba, Ca, La, La2 = attn.shape _,idx1 = flag.mean(dim=3,keepdim=False).sort(dim=2,descending=True) # print('shape of flag:{}, idx1:{}'.format(flag.shape, idx1[:,:,:32,None].repeat([1,Ca,1,L2]).shape)) flag = torch.gather(flag,2,idx1[:,:,:32,None].repeat([1,C,1,L2]).long()) attn = torch.gather(attn,2,idx1[:,:,:32,None].repeat([1,Ca,1,L2]).long()) _,idx2 = flag.mean(dim=2,keepdim=False).sort(dim=2,descending=True) flag = torch.gather(flag,3,idx2[:,:,None,:32].repeat([1,C,32,1]).long()) attn = torch.gather(attn,3,idx2[:,:,None,:32].repeat([1,Ca,32,1]).long()) return attn * flag def loss_rank(self, outputs, targetsi, temp_annoi=None): """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] The target boxes are expected in format (center_x, center_y, h, w), normalized by the image size. """ attn = outputs['attn'] # print('attn shape:{}'.format(attn.shape)) attn1 = torch.cat([attn[:,:,114:344,57:114], attn[:,:,114:344,344:]],dim=3) attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) attn2 = torch.cat([attn[:,:,344:,:57], attn[:,:,344:,114:344]],dim=3) attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # print('attn1 shape:{},attn2 shape:{}, attn:{}'.format(attn1.shape,attn2.shape,attn.shape)) # attn = self._random_permute(attn) # attn = attn[:,:,:,:] # B1, C1, H1, W1 = attn.shape # global_index_s = outputs['out_global_s'] # global_index_t = outputs['out_global_t'] # try: # assert((global_index_s.shape[1] + global_index_t.shape[1])== int(H1/2)) # except: # print('Falut,shape of attn:{}, s:{}, t:{}'.format(attn.shape,global_index_s.shape, global_index_t.shape )) # H1 = int(64) # H2 = int(256) # l_t = int(math.sqrt(64)) # l_s = int(math.sqrt(256)) # temp_anno = temp_annoi[0,:,:] # targets = targetsi[0,:,:] # r_s = torch.arange(l_s).to(temp_anno.device) # r_t = torch.arange(l_t).to(temp_anno.device) # r_t = r_t[None,:].repeat([B1,1]) # cx, cy, w, h = temp_anno[:,0:1], temp_anno[:,1:2], temp_anno[:,2:3], temp_anno[:,3:4] # cx *= l_t # cy *= l_t # w *= l_t # h *= l_t # flagx_01 = r_t >= cx - w/2 # flagx_02 = r_t <= cx + w/2 # flagy_02 = r_t >= cy - h/2 # flagy_01 = r_t <= cy + h/2 # flagx = flagx_01.float()*flagx_02.float() # flagy = flagy_01.float()*flagy_02.float() # flagx = flagx[:,None,:].repeat([1,l_t,1]) # flagy = flagy[:,:,None].repeat([1,1,l_t]) # flag = flagx*flagy # flagt = flag.reshape([B1, H1]) # cx, cy, w, h = targets[:,0:1], targets[:,1:2], targets[:,2:3], targets[:,3:4] # cx *= l_s # cy *= l_s # w *= l_s # h *= l_s # flagx_01 = r_s >= cx - w/2 # flagx_02 = r_s <= cx + w/2 # flagy_02 = r_s >= cy - h/2 # flagy_01 = r_s <= cy + h/2 # flagx = flagx_01.float()*flagx_02.float() # flagy = flagy_01.float()*flagy_02.float() # flagx = flagx[:,None,:].repeat([1,l_s,1]) # flagy = flagy[:,:,None].repeat([1,1,l_s]) # flag = flagx*flagy # flags = flag.reshape([B1, H2]) # flag = torch.cat([flagt, flags], dim=1) # flag_total = flag[:,:,None].repeat([1,1,int(H1+H2)]) * flag[:,None,:].repeat([1,int(H1+H2),1]) # attn1 = self.crop_fusion(flag_total[:,None,:,:], attn, global_index_s, global_index_t) attn = torch.cat([attn1, attn2],dim=1) B, C, H, W = attn.shape # _,s1,_ = torch.svd(attn1.reshape([B*C, H, W])) _,s1,_ = torch.svd(attn.reshape([B*C, H, W])) s01 = torch.abs(s1 - 1) return torch.mean(s01) ,其中,loss = self.loss_weight['giou'] * giou_loss + self.loss_weight['l1'] * l1_loss + self.loss_weight['focal'] * location_loss + rank_loss*1.2的权重分别是:1,14,1,1.2。这样的权重设置表示什么?
- Assistant
在您给出的代码中,loss函数由几部分组成:giou_loss、l1_loss、focal_loss(location_loss)和rank_loss。这些损失都被赋予了不同的权重,以调整它们在总损失中的贡献程度。权重的设置反映了您在模型训练中对不同类型误差的重视程度。具体来说:
-
giou_loss: 1倍权重。GIoU损失是一种改进的IoU(交并比)损失,它解决了IoU在某些情况下(如无重叠时)不能提供关于目标位置改进方向的信息问题。对GIoU损失的权重设置为1,表明它对模型性能的优化是基本且重要的。 -
l1_loss: 14倍权重。L1损失是一种常见的回归损失,用于衡量预测框和真实框之间的绝对差异。较高的权重(14)意味着模型高度重视直接的回归精度,即预测框尽可能准确地匹配目标框。 -
focal_loss: 1倍权重。Focal损失主要用于解决类别不平衡问题,它在分类任务中常用于调整难易样本的学习重点,让模型更多关注难分样本。这里它被用于位置预测的学习,权重为1表示其贡献度在总损失中被视为标准且重要。 -
rank_loss: 1.2倍权重。Rank loss(或排序损失)可能是用于确保模型输出的排列或者排序质量。在给定的代码和上下文中,这种损失可能用于调整或优化模型对于目标排列的识别能力。权重1.2表明它相比于GIoU和Focal,略有加重,但比l1损失的重视度低得多。
综上,这样的权重设置显示出模型训练特别强调精确匹配目标框的重要性(L1损失的高权重),而对于IOU和位置类别不平衡的调整则保持了基本的关注。排名损失虽然增强了一些,但相对其他项的影响仍然不是最主要的。整体权重配置帮助模型在多方面达到平衡,优化目标检测的综合性能。
-
- User
那么当前的模型是:import math import logging from functools import partial from collections import OrderedDict from copy import deepcopy
import torch import torch.nn as nn import torch.nn.functional as F
from timm.models.layers import to_2tuple
from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise from .utils import combine_tokens, recover_tokens from .vit import VisionTransformer from ..layers.attn_blocks import CEBlock import random import numpy as np from .DCM import Counter_Guide_Enhanced
_logger = logging.getLogger(name)
class VisionTransformerCE(VisionTransformer): """ Vision Transformer with candidate elimination (CE) module
A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` - https://arxiv.org/abs/2010.11929 Includes distillation token & head support for `DeiT: Data-efficient Image Transformers` - https://arxiv.org/abs/2012.12877 """ def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None, act_layer=None, weight_init='', ce_loc=None, ce_keep_ratio=None): super().__init__() if isinstance(img_size, tuple): self.img_size = img_size else: self.img_size = to_2tuple(img_size) self.patch_size = patch_size self.in_chans = in_chans self.num_classes = num_classes self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models self.num_tokens = 2 if distilled else 1 norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) act_layer = act_layer or nn.GELU self.patch_embed = embed_layer( img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4) # self.pos_embed_event = PatchEmbed_event_new(in_chans=32, embed_dim=768, kernel_size=4, stride=4) dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule blocks = [] ce_index = 0 self.ce_loc = ce_loc for i in range(depth): ce_keep_ratio_i = 1.0 if ce_loc is not None and i in ce_loc: ce_keep_ratio_i = ce_keep_ratio[ce_index] ce_index += 1 blocks.append( CEBlock( dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer, keep_ratio_search=ce_keep_ratio_i) ) self.blocks = nn.Sequential(*blocks) self.norm = norm_layer(embed_dim) self.init_weights(weight_init) # 添加交互模块counter_guide-此处对模块已经做了更新 self.counter_guide = Counter_Guide_Enhanced(768, 768) # 新增随机遮蔽特征操作 def masking_fea(self,z, event_z, x, event_x, ratio=0.8 ): b,nz,c = z.shape b,nez,c = event_z.shape b,nx,c = x.shape b,nex,c = event_x.shape assert(nz == nez) assert(nx == nex) lenz_out = int(nz*ratio) lenx_out = int(nx*ratio) mask_nz = torch.rand(b,nz).float() mask_ez = torch.rand(b,nez).float() mask_nx = torch.rand(b,nx).float() mask_ex = torch.rand(b,nex).float() mask_nz = mask_nz>0.4 mask_ez = mask_ez>0.4 mask_ez = ~mask_nz + mask_ez mask_nz_idx = mask_nz.float().sort(1,descending=True)[-1].to(device = z.device) mask_ez_idx = mask_ez.float().sort(1,descending=True)[-1].to(device = z.device) mask_nx = mask_nx>0.4 mask_ex = mask_ex>0.4 mask_ex = ~mask_nx + mask_ex mask_nx_idx = mask_nx.float().sort(1,descending=True)[-1].to(device = z.device) mask_ex_idx = mask_ex.float().sort(1,descending=True)[-1].to(device = z.device) masked_z = torch.gather(z, 1, mask_nz_idx[:,:lenz_out,None].repeat([1,1,c])) masked_ez = torch.gather(event_z, 1, mask_ez_idx[:,:lenz_out,None].repeat([1,1,c])) masked_x = torch.gather(x, 1, mask_nx_idx[:,:lenx_out,None].repeat([1,1,c])) masked_ex = torch.gather(event_x, 1, mask_ex_idx[:,:lenx_out,None].repeat([1,1,c])) return masked_z, masked_ez, masked_x, masked_ex,{'x1':mask_nx_idx[:,:lenx_out],'x0':mask_nx_idx[:,lenx_out:], 'ex1':mask_ex_idx[:,:lenx_out],'ex0':mask_ex_idx[:,lenx_out:], } # 随机遮蔽特征操作完成 def forward_features(self, z, x, event_z, event_x, mask_z=None, mask_x=None, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False ): # 分支1 处理流程 B, H, W = x.shape[0], x.shape[2], x.shape[3] x = self.patch_embed(x) z = self.patch_embed(z) z += self.pos_embed_z x += self.pos_embed_x event_x = self.pos_embed_event(event_x) event_z = self.pos_embed_event(event_z) event_x += self.pos_embed_x event_z += self.pos_embed_z # masked_z, masked_event_z, masked_x, masked_event_x, token_idx = self.masking_fea(z, event_z, x, event_x, ratio=0.9) # z = masked_z # x = masked_x # event_x = masked_event_x # event_z = masked_event_z if mask_z is not None and mask_x is not None: mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_z = mask_z.flatten(1).unsqueeze(-1) mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_x = mask_x.flatten(1).unsqueeze(-1) mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode) mask_x = mask_x.squeeze(-1) if self.add_cls_token: cls_tokens = self.cls_token.expand(B, -1, -1) cls_tokens = cls_tokens + self.cls_pos_embed if self.add_sep_seg: x += self.search_segment_pos_embed z += self.template_segment_pos_embed x = combine_tokens(z, x, mode=self.cat_mode) if self.add_cls_token: x = torch.cat([cls_tokens, x], dim=1) x = self.pos_drop(x) lens_z = self.pos_embed_z.shape[1] lens_x = self.pos_embed_x.shape[1] global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device) global_index_t = global_index_t.repeat(B, 1) global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device) global_index_s = global_index_s.repeat(B, 1) removed_indexes_s = [] # # 分支2 处理流程 event_x = combine_tokens(event_z, event_x, mode=self.cat_mode) if self.add_cls_token: event_x = torch.cat([cls_tokens, event_x], dim=1) lens_z = self.pos_embed_z.shape[1] lens_x = self.pos_embed_x.shape[1] global_index_t1 = torch.linspace(0, lens_z - 1, lens_z).to(event_x.device) global_index_t1 = global_index_t1.repeat(B, 1) global_index_s1 = torch.linspace(0, lens_x - 1, lens_x).to(event_x.device) global_index_s1 = global_index_s1.repeat(B, 1) removed_indexes_s1 = [] out_attn = [] out_attn1 = [] for i, blk in enumerate(self.blocks): # 第一个分支处理 x, global_index_t, global_index_s, removed_index_s, attn = \ blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate) # 第二个分支处理 event_x, global_index_t1, global_index_s1, removed_index_s1, attn1 = \ blk(event_x, global_index_t1, global_index_s1, mask_x, ce_template_mask, ce_keep_rate) if self.ce_loc is not None and i in self.ce_loc: removed_indexes_s.append(removed_index_s) removed_indexes_s1.append(removed_index_s1) out_attn.append(attn) out_attn1.append(attn1) if i in [0,5,11] : # 新增原始特征(在counter_guide之前的特征,便于后续的loss计算) A_original_x = x.clone() # 保存原始x特征 (RGB) A_original_event_x = event_x.clone() # 保存原始event特征 (event) # 引入counter_guide进行特征增强 此处RGB模态特征作为主特征,event_x作为补充特征 enhanced_x, enhanced_event_x = self.counter_guide(x, event_x) # 保存经过counter_guide增强的特征 A_enhanced_x = enhanced_x # 保存增强后的特征 A_enhanced_event_x = enhanced_event_x # 保存增强后的特征 # 将增强后的特征与原特征相加,这里类似残差操作 x = x + enhanced_x event_x = event_x + enhanced_event_x out_attn_idx = random.choice(np.arange(len(out_attn))) out_attn = out_attn[out_attn_idx] out_attn_idx1 = random.choice(np.arange(len(out_attn1))) out_attn1 = out_attn1[out_attn_idx1] # 应用LayerNorm归一化处理 x = self.norm(x) event_x = self.norm(event_x) x_cat = torch.cat([event_x, x], dim=1) x = x_cat # cat仅仅是做 rgb 和 event voxel拼接,数据增强采用的随机遮蔽特征策略。 aux_dict = { "attn": out_attn, "attn1": out_attn1, "removed_indexes_s": removed_indexes_s, # used for visualization 'removed_indexes_s1': removed_indexes_s1, } return x, aux_dict def forward(self, z, x, event_z, event_x, ce_template_mask=None, ce_keep_rate=None, tnc_keep_rate=None, return_last_attn=False): x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,) return x, aux_dictdef _create_vision_transformer(pretrained=False, **kwargs): model = VisionTransformerCE(**kwargs)
if pretrained: if 'npz' in pretrained: model.load_pretrained(pretrained, prefix='') else: checkpoint = torch.load(pretrained, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False) print('Load pretrained model from: ' + pretrained) return modeldef vit_base_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model
def vit_large_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model
其中,import torch import torch.nn as nn import torch.nn.functional as F
class Cross_Attention(nn.Module): def init(self, input_channels, output_channels): super(Cross_Attention, self).init() self.query = nn.Linear(input_channels, output_channels) self.key = nn.Linear(input_channels, output_channels) self.value = nn.Linear(input_channels, output_channels) self.scale = output_channels ** -0.5
def forward(self, assistant, present): q = self.query(assistant) k = self.key(present) v = self.value(present) attention_scores = torch.bmm(q, k.transpose(1, 2)) * self.scale attention = F.softmax(attention_scores, dim=-1) attended_features = torch.bmm(attention, v) return attended_featuresclass Multi_Context(nn.Module): def init(self, input_channels, output_channels): super(Multi_Context, self).init() self.linear1 = nn.Linear(input_channels, output_channels) self.linear2 = nn.Linear(input_channels, output_channels) self.linear3 = nn.Linear(input_channels, output_channels) self.linear_final = nn.Linear(output_channels * 3, output_channels)
def forward(self, x): x1 = F.relu(self.linear1(x)) x2 = F.relu(self.linear2(x)) x3 = F.relu(self.linear3(x)) x = torch.cat([x1, x2, x3], dim=-1) x = self.linear_final(x) return xclass Adaptive_Weight(nn.Module): def init(self, input_channels):
super(Adaptive_Weight, self).init() self.fc1 = nn.Linear(input_channels, input_channels // 4) self.fc2 = nn.Linear(input_channels // 4, input_channels) self.sigmoid = nn.Sigmoid()def forward(self, x): x_avg = torch.mean(x, dim=1) weight = F.relu(self.fc1(x_avg)) weight = self.fc2(weight) weight = self.sigmoid(weight).unsqueeze(1) out = x * weight return outclass Counter_Guide_Enhanced(nn.Module): def init(self, input_channels, output_channels): super(Counter_Guide_Enhanced, self).init() self.mc1 = Multi_Context(input_channels, output_channels) self.mc2 = Multi_Context(input_channels, output_channels) self.sig = nn.Sigmoid() self.cross_attention = Cross_Attention(output_channels, output_channels) self.pre_adaptive_weight = Adaptive_Weight(output_channels) self.post_adaptive_weight = Adaptive_Weight(output_channels)
def forward(self, x, event_x): # 增强event模态特征 mc1 = self.mc1(event_x) # 利用sigmoid调整x的特征强度,作为一种特征筛选 x1 = x * self.sig(mc1) # 融合原始特征和经过特征筛选增强后的特征 out1 = x + x1 # 对称操作 # 增强x模态特征 mc2 = self.mc2(x) # 利用sigmoid调整event_x的特征强度,作为一种特征筛选 event_x1 = event_x * self.sig(mc2) # 融合原始事件特征和经过特征筛选增强后的事件特征 out2 = event_x + event_x1 # 优化交互前调整权重 (新增权重预调整) weighted_event_x = self.pre_adaptive_weight(out2) # 增强后的event_x weighted_x = self.pre_adaptive_weight(out1) # 增强后的x # 交互特征 enhanced_x = self.cross_attention(weighted_x, weighted_event_x) enhanced_event_x_via_x = self.cross_attention(weighted_event_x, enhanced_x) # 交互后调整权重 adaptive_enhanced_x = self.post_adaptive_weight(enhanced_x) adaptive_enhanced_event_x = self.post_adaptive_weight(enhanced_event_x_via_x) return adaptive_enhanced_x, adaptive_enhanced_event_x,总体模型的loss设置是:from . import BaseActorfrom lib.utils.misc import NestedTensor from lib.utils.box_ops import box_cxcywh_to_xyxy, box_xywh_to_xyxy import torch from lib.utils.merge import merge_template_search from ...utils.heapmap_utils import generate_heatmap from ...utils.ce_utils import generate_mask_cond, adjust_keep_rate
import torch.nn.functional as F
class CEUTrackActor(BaseActor): """ Actor for training CEUTrack models """
def __init__(self, net, objective, loss_weight, settings, cfg=None): super().__init__(net, objective) self.loss_weight = loss_weight self.settings = settings self.bs = self.settings.batchsize # batch size self.cfg = cfg def __call__(self, data): """ args: data - The input data, should contain the fields 'template', 'search', 'gt_bbox'. template_images: (N_t, batch, 3, H, W) search_images: (N_s, batch, 3, H, W) returns: loss - the training loss status - dict containing detailed losses """ # forward pass out_dict = self.forward_pass(data) # compute losses loss, status = self.compute_losses(out_dict, data) return loss, status def forward_pass(self, data): # currently only support 1 template and 1 search region assert len(data['template_images']) == 1 assert len(data['search_images']) == 1 assert len(data['template_event']) == 1 assert len(data['search_event']) == 1 template_list = [] for i in range(self.settings.num_template): template_img_i = data['template_images'][i].view(-1, *data['template_images'].shape[2:]) # (batch, 3, 128, 128) # template_att_i = data['template_att'][i].view(-1, *data['template_att'].shape[2:]) # (batch, 128, 128) template_list.append(template_img_i) search_img = data['search_images'][0].view(-1, *data['search_images'].shape[2:]) # (batch, 3, 320, 320) # search_att = data['search_att'][0].view(-1, *data['search_att'].shape[2:]) # (batch, 320, 320) template_event = data['template_event'][0].view(-1, *data['template_event'].shape[2:]) search_event = data['search_event'][0].view(-1, *data['search_event'].shape[2:]) box_mask_z = None ce_keep_rate = None if self.cfg.MODEL.BACKBONE.CE_LOC: box_mask_z = generate_mask_cond(self.cfg, template_list[0].shape[0], template_list[0].device, data['template_anno'][0]) ce_start_epoch = self.cfg.TRAIN.CE_START_EPOCH ce_warm_epoch = self.cfg.TRAIN.CE_WARM_EPOCH ce_keep_rate = adjust_keep_rate(data['epoch'], warmup_epochs=ce_start_epoch, total_epochs=ce_start_epoch + ce_warm_epoch, ITERS_PER_EPOCH=1, base_keep_rate=self.cfg.MODEL.BACKBONE.CE_KEEP_RATIO[0]) if len(template_list) == 1: template_list = template_list[0] out_dict = self.net(template=template_list, search=search_img, event_template=template_event, event_search=search_event, ce_template_mask=box_mask_z, ce_keep_rate=ce_keep_rate, return_last_attn=False) return out_dict def compute_losses(self, pred_dict, gt_dict, return_status=True): # gt gaussian map gt_bbox = gt_dict['search_anno'][-1] # (Ns, batch, 4) (x1,y1,w,h) -> (batch, 4) gt_gaussian_maps = generate_heatmap(gt_dict['search_anno'], self.cfg.DATA.SEARCH.SIZE, self.cfg.MODEL.BACKBONE.STRIDE) gt_gaussian_maps = gt_gaussian_maps[-1].unsqueeze(1) # Get boxes pred_boxes = pred_dict['pred_boxes'] if torch.isnan(pred_boxes).any(): raise ValueError("Network outputs is NAN! Stop Training") num_queries = pred_boxes.size(1) pred_boxes_vec = box_cxcywh_to_xyxy(pred_boxes).view(-1, 4) # (B,N,4) --> (BN,4) (x1,y1,x2,y2) gt_boxes_vec = box_xywh_to_xyxy(gt_bbox)[:, None, :].repeat((1, num_queries, 1)).view(-1, 4).clamp(min=0.0, max=1.0) # (B,4) --> (B,1,4) --> (B,N,4) # compute giou and iou try: giou_loss, iou = self.objective['giou'](pred_boxes_vec, gt_boxes_vec) # (BN,4) (BN,4) except: giou_loss, iou = torch.tensor(0.0).cuda(), torch.tensor(0.0).cuda() # compute l1 loss l1_loss = self.objective['l1'](pred_boxes_vec, gt_boxes_vec) # (BN,4) (BN,4) # compute location loss if 'score_map' in pred_dict: location_loss = self.objective['focal'](pred_dict['score_map'], gt_gaussian_maps) else: location_loss = torch.tensor(0.0, device=l1_loss.device) # 按照HRCEUTrack,新增loss约束 rank_rgb_loss = self.rank_rgb_loss(pred_dict,gt_dict['search_anno'], gt_dict['template_anno']) rank_event_loss= self.rank_event_loss(pred_dict,gt_dict['search_anno'], gt_dict['template_anno']) ####### # 计算特征一致性损失 # feature_consistency_loss = F.mse_loss(pred_dict['A_enhanced_x'], pred_dict['A_original_x']) + F.mse_loss(pred_dict['A_enhanced_event_x'], pred_dict['A_original_event_x']) # # 标量化特征一致性损失 # feature_consistency_loss = feature_consistency_loss.mean() # weighted sum loss = self.loss_weight['giou'] * giou_loss + self.loss_weight['l1'] * l1_loss + self.loss_weight['focal'] * location_loss + rank_rgb_loss * 1 + rank_event_loss * 1 # + 1 * feature_consistency_loss if return_status: # status for log mean_iou = iou.detach().mean() status = {"Loss/total": loss.item(), "Loss/giou": giou_loss.item(), "Loss/l1": l1_loss.item(), "Loss/location": location_loss.item(), #'Loss/feature_consistency': feature_consistency_loss.item(), "rank_rgb_loss": rank_rgb_loss.item(), "rank_event_loss": rank_event_loss.item(), "IoU": mean_iou.item()} return loss, status else: return loss#################################### 按照 HRCEUTrack新增 loss #########################################
def _random_permute(self,matrix): # matrix = random.choice(matrix) b, c, h, w = matrix.shape idx = [ torch.randperm(c).to(matrix.device) for i in range(b)] idx = torch.stack(idx, dim=0)[:, :, None, None].repeat([1,1,h,w]) # idx = torch.randperm(c)[None,:,None,None].repeat([b,1,h,w]).to(matrix.device) matrix01 = torch.gather(matrix, 1, idx) return matrix01 def crop_flag(self, flag, global_index_s, global_index_t,H1 = 64, H2 = 256): B,Ls = global_index_s.shape B, Lt = global_index_t.shape B,C,L1,L2 = flag.shape flag_t = flag[:,:,:H1,:] flag_s = flag[:,:,H1:,:] flag_t = torch.gather(flag_t,2,global_index_t[:,None,:,None].repeat([1,C,1,L2]).long()) flag_s = torch.gather(flag_s,2,global_index_s[:,None,:,None].repeat([1,C,1,L2]).long()) flag = torch.cat([flag_t, flag_s], dim = 2) flag_t = flag[:,:,:,:H1] flag_s = flag[:,:,:,H1:] flag_t = torch.gather(flag_t,3,global_index_t[:,None,None,:].repeat([1,C,int(Ls+Lt),1]).long()) flag_s = torch.gather(flag_s,3,global_index_s[:,None,None,:].repeat([1,C,int(Ls+Lt),1]).long()) flag = torch.cat([flag_t, flag_s], dim = 3) B, C, L11, L12 = flag.shape try: assert(L11 == int(Lt + Ls)) assert(L12 == int(Lt + Ls)) except: print('L11:{}, L12:{}, L1:{}, L2:{}'.format(L11, L12, L1, L2)) return flag def crop_fusion(self, flag, attn, global_index_s, global_index_t,H1 = 64, H2 = 256 ): flag = self.crop_flag(flag=flag, global_index_s=global_index_s, global_index_t=global_index_t) B,C,L1,L2 = flag.shape Ba, Ca, La, La2 = attn.shape _,idx1 = flag.mean(dim=3,keepdim=False).sort(dim=2,descending=True) # print('shape of flag:{}, idx1:{}'.format(flag.shape, idx1[:,:,:32,None].repeat([1,Ca,1,L2]).shape)) flag = torch.gather(flag,2,idx1[:,:,:32,None].repeat([1,C,1,L2]).long()) attn = torch.gather(attn,2,idx1[:,:,:32,None].repeat([1,Ca,1,L2]).long()) _,idx2 = flag.mean(dim=2,keepdim=False).sort(dim=2,descending=True) flag = torch.gather(flag,3,idx2[:,:,None,:32].repeat([1,C,32,1]).long()) attn = torch.gather(attn,3,idx2[:,:,None,:32].repeat([1,Ca,32,1]).long()) return attn * flag # def loss_rank_ori(self, outputs, targetsi, temp_annoi=None): # attn = outputs['attn'] # attn1 = torch.cat([attn[:,:,114:344,57:114], attn[:,:,114:344,344:]],dim=3) # attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # attn2 = torch.cat([attn[:,:,344:,:57], attn[:,:,344:,114:344]],dim=3) # attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # attn = torch.cat([attn1, attn2],dim=1) # B, C, H, W = attn.shape # _,s1,_ = torch.svd(attn.reshape([B*C, H, W])) # s01 = torch.abs(s1 - 1) # return torch.mean(s01) # def rank_rgb_loss(self, outputs, targetsi, temp_annoi=None): # attn = outputs['attn'] # # attn.shape torch.Size([2, 12, 320, 320]) # attn1 = attn[:, :, 57:, :57] # attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # print('attn1.shape:', attn1.shape) # attn1.shape: torch.Size([1, 1, 230, 57]) # attn2 = attn[:, :, :57, 57:] # attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # print('attn2.shape:', attn2.shape) # attn2.shape: torch.Size([1, 1, 57, 230]) # attn3 = attn[:, :, :57, :57] # attn3 = attn3.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # print('attn3.shape:', attn3.shape) # attn3.shape: torch.Size([1, 1, 57, 57]) # attn4 = attn[:, :, 57:, 57:] # attn4 = attn4.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # print('attn4.shape:', attn4.shape) # attn4.shape: torch.Size([1, 1, 230, 230]) # # 展平注意力矩阵 # attn1_flat = attn1.view(1, -1) # attn2_flat = attn2.view(1, -1) # attn = torch.cat([attn1_flat, attn2_flat], dim=0) # _, s1, _ = torch.svd(attn) # s01 = torch.abs(s1 - 1) # return torch.mean(s01) def rank_rgb_loss(self, outputs, targetsi, temp_annoi=None): attn = outputs['attn'] attn1 = attn[:, :, 57:, :57] attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) attn2 = attn[:, :, :57, 57:] attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # 计算 attn1 的 SVD 并计算奇异值 B1, C1, H1, W1 = attn1.shape _, s1, _ = torch.svd(attn1.reshape([B1*C1, H1, W1])) s01 = torch.abs(s1 - 1) loss1 = torch.mean(s01) # 计算 attn2 的 SVD 并计算奇异值 B2, C2, H2, W2 = attn2.shape _, s2, _ = torch.svd(attn2.reshape([B2*C2, H2, W2])) s02 = torch.abs(s2 - 1) loss2 = torch.mean(s02) loss = max(loss1, loss2) return loss def rank_event_loss(self, outputs, targetsi, temp_annoi=None): attn = outputs['attn1'] attn1 = attn[:, :, 57:, :57] attn1 = attn1.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) attn2 = attn[:, :, :57, 57:] attn2 = attn2.mean(dim=0, keepdim=True).mean(dim=1, keepdim=True) # 计算 attn1 的 SVD 并计算奇异值 B1, C1, H1, W1 = attn1.shape _, s1, _ = torch.svd(attn1.reshape([B1*C1, H1, W1])) s01 = torch.abs(s1 - 1) loss1 = torch.mean(s01) # 计算 attn2 的 SVD 并计算奇异值 B2, C2, H2, W2 = attn2.shape _, s2, _ = torch.svd(attn2.reshape([B2*C2, H2, W2])) s02 = torch.abs(s2 - 1) loss2 = torch.mean(s02) loss = max(loss1, loss2) return loss,权重依然是:1,14,1,1.2,1.2。那么现在需要考虑当前模型与上段代码中的模型的不同,进而需要判断loss权重是否需要改变,尤其是rank_loss的权重。 - Assistant
对于当前模型与先前描述的模型的不同,主要体现在以下几方面:
-
增强的交叉注意力模块(Counter_Guide_Enhanced):这是一个比较显著的增强,它通过引入交叉注意力和多上下文模块以及自适应权重处理,目标是更好地融合RGB和事件信息。这可能会对模型的性能产生明显影响,提高特征的区分能力。
-
随机遮蔽特徃操作(masking_fea方法):这一操作增加了模型在处理输入数据时的随机性,旨在通过部分遮蔽原始特征来提高模型对不完整或遮挡数据的鲁棒性。
-
双分支结构:引入了并行处理RGB和事件数据的结构,每一个分支都经过独立的变换网络,并着重于处理其对应的数据类型。
鉴于这些改动,尤其是在交叉注意力和双分支结构方面的增强,我们需要重新考虑
rank_loss的权重。从理论上讲,这些改动使得模型在整合多模态信息方面更为有效,增加的复杂性可能需要通过更重视这部分损失来平衡,确保这些复杂的互动被恰当地学习:- 增加
rank_loss权重:由于新增了两个分支(RGB和事件),并使用了交叉注意力和自适应权重模块,这意呅着这部分的错误预测可能对最终结果产生较大的负面影响。适当增加这部分损失的权重可以帮助模型更重视学习这些新引入的组件的有效融合。 - 监控改变后的效果:增加损失函数权重是一种试验性的改动,最好通过实验来验证它的实际影响。可以通过在验证集上监测性能(如准确度、召回率等指标)来评估权重改动的效果。
建议实验性地将
rank_loss的权重稍微调高,如将原先的权重1.2调整为1.5或更高,观察这一改变如何影响总体性能和过拟合情况。这需要在实际的训练过程中进行实验和调整。 -