在# 将 4输入分开,构建新的相同模态结合的2输入,2分支 import math import logging from functools import partial…
- User
在# 将 4输入分开,构建新的相同模态结合的2输入,2分支 import math import logging from functools import partial from collections import OrderedDict from copy import deepcopy
import torch import torch.nn as nn import torch.nn.functional as F
from timm.models.layers import to_2tuple
from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise from .utils import combine_tokens, recover_tokens from .vit import VisionTransformer from ..layers.attn_blocks import CEBlock from .ad_counter_guide import Counter_Guide_Enhanced from .UEP import UEPModule from .Attention import Attention _logger = logging.getLogger(name)
class VisionTransformerCE(VisionTransformer): """ Vision Transformer with candidate elimination (CE) module
A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` - https://arxiv.org/abs/2010.11929 Includes distillation token & head support for `DeiT: Data-efficient Image Transformers` - https://arxiv.org/abs/2012.12877 """ def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None, act_layer=None, weight_init='', ce_loc=None, ce_keep_ratio=None): super().__init__() if isinstance(img_size, tuple): self.img_size = img_size else: self.img_size = to_2tuple(img_size) self.patch_size = patch_size self.in_chans = in_chans self.num_classes = num_classes self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models self.num_tokens = 2 if distilled else 1 norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) act_layer = act_layer or nn.GELU self.patch_embed = embed_layer( img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4) dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule blocks = [] ce_index = 0 self.ce_loc = ce_loc for i in range(depth): ce_keep_ratio_i = 1.0 if ce_loc is not None and i in ce_loc: ce_keep_ratio_i = ce_keep_ratio[ce_index] ce_index += 1 blocks.append( CEBlock( dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer, keep_ratio_search=ce_keep_ratio_i) ) self.blocks = nn.Sequential(*blocks) self.norm = norm_layer(embed_dim) self.init_weights(weight_init) # 添加交互模块counter_guide self.counter_guide = Counter_Guide_Enhanced(768, 768) self.uep_module1 = UEPModule(256, embed_dim, embed_dim // 4, self.img_size) self.uep_module2 = UEPModule(64, embed_dim, embed_dim // 4, self.img_size) def forward_features(self, z, x, event_z, event_x, event_z2, event_x2, mask_z=None, mask_x=None, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False ): # 分支1 处理流程 B, H, W = x.shape[0], x.shape[2], x.shape[3] x = self.patch_embed(x) z = self.patch_embed(z) # x = self.uep_module1(x) z = self.uep_module2(z) # z += self.pos_embed_z x += self.pos_embed_x ################################ if mask_z is not None and mask_x is not None: mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_z = mask_z.flatten(1).unsqueeze(-1) mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_x = mask_x.flatten(1).unsqueeze(-1) mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode) mask_x = mask_x.squeeze(-1) if self.add_cls_token: cls_tokens = self.cls_token.expand(B, -1, -1) cls_tokens = cls_tokens + self.cls_pos_embed if self.add_sep_seg: x += self.search_segment_pos_embed z += self.template_segment_pos_embed x = combine_tokens(z, x, mode=self.cat_mode) if self.add_cls_token: x = torch.cat([cls_tokens, x], dim=1) x = self.pos_drop(x) lens_z = self.pos_embed_z.shape[1] lens_x = self.pos_embed_x.shape[1] global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device) global_index_t = global_index_t.repeat(B, 1) global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device) global_index_s = global_index_s.repeat(B, 1) removed_indexes_s = [] # # 分支2 处理流程 event_x = self.pos_embed_event(event_x) event_z = self.pos_embed_event(event_z) event_x += self.pos_embed_x event_z += self.pos_embed_z # event_x = self.uep_module1(event_x) event_z = self.uep_module2(event_z) #################################### event_x = combine_tokens(event_z, event_x, mode=self.cat_mode) if self.add_cls_token: event_x = torch.cat([cls_tokens, event_x], dim=1) lens_z = self.pos_embed_z.shape[1] lens_x = self.pos_embed_x.shape[1] global_index_t1 = torch.linspace(0, lens_z - 1, lens_z).to(event_x.device) global_index_t1 = global_index_t1.repeat(B, 1) global_index_s1 = torch.linspace(0, lens_x - 1, lens_x).to(event_x.device) global_index_s1 = global_index_s1.repeat(B, 1) removed_indexes_s1 = [] ## 分支3 处理流程 event_x2 = self.patch_embed(event_x2) event_z2 = self.patch_embed(event_z2) # event_x2 = self.uep_module1(event_x2) event_z2 = self.uep_module2(event_z2) # event_x2 += self.pos_embed_x event_z2 += self.pos_embed_z ############################################################ event_x2 = combine_tokens(event_z2, event_x2, mode=self.cat_mode) if self.add_cls_token: event_x2 = torch.cat([cls_tokens, event_x2], dim=1) lens_z2 = self.pos_embed_z.shape[1] lens_x2 = self.pos_embed_x.shape[1] global_index_t2 = torch.linspace(0, lens_z2 - 1, lens_z2).to(event_x2.device) global_index_t2 = global_index_t1.repeat(B, 1) global_index_s2 = torch.linspace(0, lens_x2 - 1, lens_x2).to(event_x2.device) global_index_s2 = global_index_s2.repeat(B, 1) removed_indexes_s2 = [] for i, blk in enumerate(self.blocks): # 第一个分支处理 x, global_index_t, global_index_s, removed_index_s, attn = \ blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate) # 第二个分支处理 event_x, global_index_t1, global_index_s1, removed_index_s1, attn1 = \ blk(event_x, global_index_t1, global_index_s1, mask_x, ce_template_mask, ce_keep_rate) event_x2, global_index_t2, global_index_s2, removed_index_s2, attn2 = \ blk(event_x2, global_index_t2, global_index_s2, mask_x, ce_template_mask, ce_keep_rate) ############### # 在第1层增加 counter_guide 模块,验证早期融合效果 # if i == 11: # A_original_x = x.clone() # A_original_event_x = event_x.clone() # A_original_event_x2 = event_x2.clone() # # 对 x 和 event_x 进行特征交互与增强 # enhanced_x_1, enhanced_event_x = self.counter_guide(x, event_x) # # 对 x 和 event_x2 进行特征交互与增强 # enhanced_x_2, enhanced_event_x2 = self.counter_guide(x, event_x2) # # 将增强后的特征与原特征相加 # x = x + enhanced_x_1 + enhanced_x_2 # event_x = event_x + enhanced_event_x # event_x2 = event_x2 + enhanced_event_x2 # # # 保存增强后的特征 # A_enhanced_x = enhanced_x_1 + enhanced_x_2 # A_enhanced_event_x = enhanced_event_x # A_enhanced_event_x2 = enhanced_event_x2 if i == 11: enhanced_x_1, enhanced_event_x = self.counter_guide(x, event_x) enhanced_x_2, enhanced_event_x2 = self.counter_guide(x, event_x2) x = x + enhanced_x_1 + enhanced_x_2 event_x = event_x + enhanced_event_x event_x2 = event_x2 + enhanced_event_x2 if self.ce_loc is not None and i in self.ce_loc: removed_indexes_s.append(removed_index_s) removed_indexes_s1.append(removed_index_s1) removed_indexes_s2.append(removed_index_s2) # 恢复特征tokens ############### # 应用LayerNorm归一化处理 x = self.norm(x) lens_x_new = global_index_s.shape[1] lens_z_new = global_index_t.shape[1] z = x[:, :lens_z_new] x = x[:, lens_z_new:] if removed_indexes_s and removed_indexes_s[0] is not None: removed_indexes_cat = torch.cat(removed_indexes_s, dim=1) pruned_lens_x = lens_x - lens_x_new pad_x = torch.zeros([B, pruned_lens_x, x.shape[2]], device=x.device) x = torch.cat([x, pad_x], dim=1) index_all = torch.cat([global_index_s, removed_indexes_cat], dim=1) C = x.shape[-1] x = torch.zeros_like(x).scatter(dim=1, index=index_all.unsqueeze(-1).expand(B, -1, C).to(torch.int64), src=x) x = recover_tokens(x, global_index_t.shape[1], lens_x, mode=self.cat_mode) x = x[:,:lens_x] ############################## event_x = self.norm(event_x) lens_x_new1 = global_index_s1.shape[1] lens_z_new1 = global_index_t1.shape[1] event_z = event_x[:, :lens_z_new1] event_x = event_x[:, lens_z_new1:] if removed_indexes_s1 and removed_indexes_s1[0] is not None: removed_indexes_cat1 = torch.cat(removed_indexes_s1, dim=1) pruned_lens_x1 = lens_x - lens_x_new1 pad_x1 = torch.zeros([B, pruned_lens_x1, event_x.shape[2]], device=event_x.device) event_x = torch.cat([event_x, pad_x1], dim=1) index_all1 = torch.cat([global_index_s1, removed_indexes_cat1], dim=1) C = event_x.shape[-1] event_x = torch.zeros_like(event_x).scatter_(dim=1, index=index_all1.unsqueeze(-1).expand(B, -1, C).to(torch.int64), src=event_x) event_x = recover_tokens(event_x, global_index_t1.shape[1], lens_x, mode=self.cat_mode) event_x = event_x[:,:lens_x] ######################## event_x2 = self.norm(event_x2) lens_x_new2 = global_index_s2.shape[1] lens_z_new2 = global_index_t2.shape[1] event_z2 = event_x2[:, :lens_z_new2] event_x2 = event_x2[:, lens_z_new2:] if removed_indexes_s2 and removed_indexes_s2[0] is not None: removed_indexes_cat2 = torch.cat(removed_indexes_s2, dim=1) pruned_lens_x2 = lens_x2 - lens_x_new2 pad_x2 = torch.zeros([B, pruned_lens_x2, event_x2.shape[2]], device=event_x2.device) event_x2 = torch.cat([event_x2, pad_x2], dim=1) index_all2 = torch.cat([global_index_s2, removed_indexes_cat2], dim=1) C = event_x2.shape[-1] event_x2 = torch.zeros_like(event_x2).scatter_(dim=1, index=index_all2.unsqueeze(-1).expand(B, -1, C).to(torch.int64), src=event_x2) event_x2 = recover_tokens(event_x2, global_index_t2.shape[1], lens_x2, mode=self.cat_mode) event_x2 = event_x2[:,:lens_x] ################################# # 直接拼接 # x1 = x + event_x + event_x2 x1 = x - event_x x2 = x - event_x2 x_cat = torch.cat([x1, x2], dim=1) x = x_cat aux_dict = { "attn": attn, 'attn1': attn1, 'attn2': attn2, "removed_indexes_s": removed_indexes_s, # used for visualization 'removed_indexes_s1': removed_indexes_s1, 'removed_indexes_s2': removed_indexes_s2, # 'A_original_x': A_original_x, # 'A_original_event_x': A_original_event_x, # 'A_original_event_x2': A_original_event_x2, # 'A_enhanced_x': A_enhanced_x, # 'A_enhanced_event_x': A_enhanced_event_x, # 'A_enhanced_event_x2': A_enhanced_event_x2 } return x, aux_dict def forward(self, z, x, event_z, event_x, event_z2, event_x2, ce_template_mask=None, ce_keep_rate=None, tnc_keep_rate=None, return_last_attn=False): x, aux_dict = self.forward_features(z, x, event_z, event_x, event_z2, event_x2,ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,) # print('x.shape:',x.shape) # x.shape: torch.Size([6, 960, 768]) # 320*3 = 960 (batch_size, num_tokens, embed_dim) return x, aux_dictdef _create_vision_transformer(pretrained=False, **kwargs): model = VisionTransformerCE(**kwargs)
if pretrained: if 'npz' in pretrained: model.load_pretrained(pretrained, prefix='') else: checkpoint = torch.load(pretrained, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False) print('Load pretrained model from: ' + pretrained) return modeldef vit_base_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model
def vit_large_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model中,import torch import torch.nn as nn import torch.nn.functional as F class Attention(nn.Module): def init(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0.): super().init() self.num_heads = num_heads head_dim = dim // num_heads self.scale = head_dim ** -0.5
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) self.proj = nn.Linear(dim, dim) self.proj_drop = nn.Dropout(proj_drop) def forward(self, x, return_attention=False): B, N, C = x.shape qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) attn = (q @ k.transpose(-2, -1)) * self.scale attn = attn.softmax(dim=-1) attn = self.attn_drop(attn) x = (attn @ v).transpose(1, 2).reshape(B, N, C) x = self.proj(x) x = self.proj_drop(x) if return_attention: return x, attn return x,那么将Attention代替CGE可以吗? - Assistant
是的,你可以尝试使用
Attention模块代替Counter_Guide_Enhanced(CGE)模块。不过,请注意,Counter_Guide_Enhanced这个名称可能是指一个用于特定任务或结构的定制化模块,而Attention模块通常是一个通用的机制。因此,你可能需要根据具体情况调整Attention模块的实现,以确保其行为与原来的布局相匹配。以下是如何将
Attention模块集成到你的代码中的一般步骤:-
导入和初始化
Attention模块:在
VisionTransformerCE类的初始化方法中,替换或增加对Counter_Guide_Enhanced的实例化为Attention:self.attention = Attention(embed_dim) -
在前向传播中使用
Attention:在
forward_features方法中,替换使用Counter_Guide_Enhanced的地方,用Attention来计算特征交互:if i == 11: # 假设x和event_x是两个需要交互的特征 enhanced_x, _ = self.attention(x) enhanced_event_x, _ = self.attention(event_x) x = x + enhanced_x event_x = event_x + enhanced_event_x -
调整
Attention参数:确保在初始化
Attention模块时选择适当的参数,如dim(嵌入维度)、num_heads(注意力头数)、以及任何其他可选的参数。
在实际操作中,你可能会需要根据你的具体任务需求对
Attention的实现进行调整,比如叠加额外的线性层或自定义的损失函数。这些调整都应该围绕如何最好地促进特征融合和任务优化而制定。 -
- User
vit中的交互注意力是如何定义的?和上文中的一样吗?
- Assistant
在Vision Transformer (ViT) 中,注意力机制是通过一个多头自注意力(Multi-Head Self-Attention, MHSA)机制来实现的。这通常是标准的Transformer架构的一部分,与BERT、GPT等使用的注意力机制类似。
ViT中的注意力定义和上文中的
Attention类有一些相似之处,但为了吃透两者有什么不同,下面是ViT中注意力的基本实现原理:-
输入拆分为查询(Q)、键(K)、值(V): 输入序列通过线性变换被映射为查询、键和值向量。这部分在你的
Attention实现中是通过self.qkv完成的。 -
计算注意力权重: 使用查询和键计算注意力权重。权重通常通过点积计算,然后应用缩放(通常除以
sqrt(头数的维度))来稳定梯度计算: [ \text{Attention}(Q, K, V) = \text{softmax}\left(\frac{QK^T}{\sqrt{d_k}}\right)V ] 其中d_k是单个头的维度。 -
多头机制: 多头自注意力意味着多个这种注意力机制在不同的子空间中并行执行。每个头都有自己的独立的
W_q、W_k和W_v矩阵。这有助于捕获不同类型的关系。 -
输出投影: 多头注意力的结果被串联起来,并经过线性层投影回原始的输入维度。这有点类似于你的
proj层的角色。
随着这些步骤的细化和搭建,ViT中的注意力通常会添加在块处理逻辑中,组合成更广泛的Encoder结构,而不仅仅是个别的模块。
在你的代码中你正试图替代的
Counter_Guide_Enhanced模块可能包含了额外的应用逻辑或Task-specific的优化,这可能在简单的Attention实现中有所缺失。因此,如果它不完全是替代标准自注意力机制的系统的一部分的话,那么要相应地作出修改。总结来说,ViT中的注意力机制和上述
Attention实现是类似的,但实际应用可能还需要考虑更多系统级别的优化和集成。 -
- User
但是原本的CGE是进行两个分支之间的交互的。这样设计的:import torch import torch.nn as nn import torch.nn.functional as F
为结合vit中的attention优势,设计了新的交叉注意力
class Cross_Attention(nn.Module): def init(self, input_channels, output_channels): super(Cross_Attention, self).init() self.query = nn.Linear(input_channels, output_channels) self.key = nn.Linear(input_channels, output_channels) self.value = nn.Linear(input_channels, output_channels) self.scale = output_channels ** -0.5 # 缩放因子,防止softmax的梯度消失/爆炸
def forward(self, assistant, present): # 生成查询、键和值 q = self.query(assistant) k = self.key(present) v = self.value(present) # 计算注意力分数 attention_scores = torch.bmm(q, k.transpose(1, 2)) * self.scale attention = F.softmax(attention_scores, dim=-1) # 应用注意力机制 attended_features = torch.bmm(attention, v) return attended_featuresclass Multi_Context(nn.Module): def init(self, input_channels, output_channels): # 修正了def init为def init super(Multi_Context, self).init() self.linear1 = nn.Linear(input_channels, output_channels) self.linear2 = nn.Linear(input_channels, output_channels) self.linear3 = nn.Linear(input_channels, output_channels) self.linear_final = nn.Linear(output_channels * 3, output_channels)
def forward(self, x): x1 = F.relu(self.linear1(x)) x2 = F.relu(self.linear2(x)) x3 = F.relu(self.linear3(x)) x = torch.cat([x1, x2, x3], dim=-1) # Concatenate along the feature dimension x = self.linear_final(x) return xclass EnhancedMultiContext(nn.Module): def init(self, input_channels, output_channels): super(EnhancedMultiContext, self).init() self.linear1 = nn.Linear(input_channels, output_channels) self.linear2 = nn.Linear(input_channels, output_channels) self.linear3 = nn.Linear(input_channels, output_channels) self.relu1 = nn.ReLU() self.leaky_relu2 = nn.LeakyReLU() self.swish3 = nn.SiLU() # Alternatively, you can use nn.SiLU() for Swish activation self.linear_final = nn.Linear(output_channels * 3, output_channels)
def forward(self, x): x1 = self.relu1(self.linear1(x)) x2 = self.leaky_relu2(self.linear2(x)) x3 = self.swish3(self.linear3(x)) x = torch.cat([x1, x2, x3], dim=-1) x = self.linear_final(x) return xclass Adaptive_Weight(nn.Module): def init(self, input_channels): # 修正了def init为def init super(Adaptive_Weight, self).init() self.fc1 = nn.Linear(input_channels, input_channels // 2) self.fc2 = nn.Linear(input_channels // 2, input_channels) self.sigmoid = nn.Sigmoid()
def forward(self, x): x_avg = torch.mean(x, dim=1) # Global Average Pooling along the ‘sequence length’ dimension weight = F.relu(self.fc1(x_avg)) weight = self.fc2(weight) weight = self.sigmoid(weight).unsqueeze(1) out = x * weight return outclass Counter_Guide_Enhanced(nn.Module): def init(self, input_channels, output_channels): super(Counter_Guide_Enhanced, self).init() self.mc = Multi_Context(input_channels, output_channels) self.cross_attention = Cross_Attention(output_channels, output_channels) self.adaptive_weight = Adaptive_Weight(output_channels)
def forward(self, x, event_x): enhanced_event_x = self.mc(event_x) enhanced_x = self.cross_attention(x, event_x) enhanced_event_x_via_x = self.cross_attention(event_x, enhanced_x) adaptive_enhanced_x = self.adaptive_weight(enhanced_x) adaptive_enhanced_event_x = self.adaptive_weight(enhanced_event_x_via_x) return adaptive_enhanced_x, adaptive_enhanced_event_x - Assistant
原来你使用的
Counter_Guide_Enhanced(CGE) 模块采用了交叉注意力机制,这是一个相对复杂的模块设计,结合了多种机制来实现两个分支之间的特征交互和增强。为了明确其工作原理,这里是你的代码结构的简要说明:
-
Multi-Context 处理:
- 使用了三个线性层,分别使用 ReLU、LeakyReLU 和 SiLU 作为激活函数来处理输入特征。处理后的特征被连接在一起并通过一个线性层得到输出。
-
Cross Attention:
- 实现了一个交叉注意力机制,其中一个分支中的特征(如
x)用于生成查询,另一个分支的特征(如event_x)用于生成键和值。这种连接允许网络捕捉不同特征间的语义关系。
- 实现了一个交叉注意力机制,其中一个分支中的特征(如
-
Adaptive Weighting:
- 使用全局平均池化和两个线性层生成自适应权重,以对已经通过注意力机制处理过的特征进行加权调整。
-
Counter Guide Enhanced for Interaction:
- 处理输入
x和event_x的特征交互。跨分支的增强由Cross_Attention模块促进,而Adaptive_Weight机制对这些增强特征进行加权和调整。
- 处理输入
对于两分支的交互这一用途,
Counter_Guide_Enhanced模块有效地融合了来自不同特征空间的语义信息。这种结构很可能致力于捕捉和利用场景和事件特征之间复杂的相互关系。如要用 ViT 中的一般注意力机制或你的
Attention类来实施这一点,要注意需要额外实现交叉注意力的逻辑,外加上任何专门的任务相关技术(例如多样化特征增强和适应性控制),这可能对简单替换来说过于繁琐。因此更推荐的方式是分析现有模块提供的特性,然后根据任务需要在Attention里进行必要的功能补充。 -
- User
现在对concat进行重写,重新设计新的常用的,最基础的Fusion方法,取代concat,x_cat = torch.cat([x1, x2], dim=1)
- Assistant
在深度学习中,简单地将两个特征向量/张量进行拼接(
concat)是实现特征融合的一种常见方法。然而,实际上有许多其他基本的特征融合方法可以使用。以下是一些常见且基础的特征融合方法,你可以根据你的需求来替换concat:-
加法融合 (Addition):
- 将两个特征张量逐元素相加。这样可以简单地把特征空间叠加起来,适合在特征维度和大小相同的情况下使用。
x_fuse = x1 + x2 -
平均融合 (Average):
- 将特征张量按元素平均化,这相当于给每一个特征张量一个相同的权重。
x_fuse = (x1 + x2) / 2 -
元素乘积 (Element-wise Multiplication):
- 通过逐元素相乘来融合不同特征。这可以被视为对那些在两个特征中都高度活跃的部分的强化。
x_fuse = x1 * x2 -
混合加权 (Weighted Sum):
- 为每个特征张量分配一个学习到的权重,然后进行加法。你可以使用一个线性层或可学习参数来实现加权。
alpha = 0.5 # 也可以是一个通过学习得到的参数 x_fuse = alpha * x1 + (1 - alpha) * x2 -
拼接后项目(Projection) (Projection after Concat):
- 首先对特征进行拼接,然后通过一个线性层进行投影。这有助于减少维度膨胀,并加入一些非线性变换以提高灵活性。
concatenated_features = torch.cat([x1, x2], dim=-1) projection_layer = nn.Linear(concatenated_features.size(-1), out_features) # 你可以定义out_features x_fuse = projection_layer(concatenated_features)
对于简单替换
concat,可以先尝试加法或平均,这两者在许多情况下工作良好且简单易用。如果需要更多的控制力(比如根据任务动态调整的能力),则可以考虑混合加权或其他高级方法。选择哪种方法要基于你希望在目标特征上得到的效果以及输入特征的具体情况。某些融合方法可能对于捕捉某些数据集上的模式或特性比其他方法更有效。
-
- User
有没有attention融合?
- Assistant
是的,确实存在使用注意力机制进行特征融合的方法,这种方法通常被称为注意力融合 (Attention Fusion)。利用注意力机制进行特征融合的核心思想是通过计算不同特征之间的相关性,引导网络动态地选择或加权组合这些特征。这种方法在多模态数据或者需要融合不同来源的信息时特别有用。
以下是一个简单的注意力融合的实现示例:
import torch import torch.nn as nn import torch.nn.functional as F class AttentionFusion(nn.Module): def __init__(self, input_dim, hidden_dim): """ :param input_dim: 输入特征的维度 :param hidden_dim: 用于计算注意力的隐藏层维度 """ super(AttentionFusion, self).__init__() self.query = nn.Linear(input_dim, hidden_dim) self.key = nn.Linear(input_dim, hidden_dim) self.value = nn.Linear(input_dim, input_dim) # 输出维度和输入相同 def forward(self, x1, x2): # 计算query, key, value q = self.query(x1) # 从x1生成查询 k = self.key(x2) # 从x2生成键 v = self.value(x2) # 从x2生成值 # 计算注意力权重 attn_weights = F.softmax(torch.bmm(q, k.transpose(1, 2)) / (k.size(-1) ** 0.5), dim=-1) # 应用注意力权重到值上 attn_output = torch.bmm(attn_weights, v) # 返回融合后的特征 return attn_output工作原理:
-
Query-Key-Value机制:从两个输入特征向量(或张量)
x1和x2中获取查询(Query)、键(Key)和值(Value)。在这个例子中,查询从x1中提取,而键和值从x2中提取。 -
注意力权重计算:通过计算
Query和Key之间的点积(然后通过除法和softmax调节)获取注意力权重。 -
应用注意力权重:将注意力权重应用到值
Value上,从而得到融合后的输出。
这种注意力机制的方案可以灵活地将两个输入特征融合到一起,用于强调不同特征的重要性和关联性。在多模态或者复杂特征信息融合场景中尤其有用。你可以根据实际情况进一步调整注意力机制中的参数或添加更复杂的层来捕捉更细腻的特性关系。
-
- User
那么在# 将 4输入分开,构建新的相同模态结合的2输入,2分支 import math import logging from functools import partial from collections import OrderedDict from copy import deepcopy
import torch import torch.nn as nn import torch.nn.functional as F
from timm.models.layers import to_2tuple
from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise from .utils import combine_tokens, recover_tokens from .vit import VisionTransformer from ..layers.attn_blocks import CEBlock from .ad_counter_guide import Counter_Guide_Enhanced from .UEP import UEPModule from .Attention import AttentionFusion _logger = logging.getLogger(name)
class VisionTransformerCE(VisionTransformer): """ Vision Transformer with candidate elimination (CE) module
A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` - https://arxiv.org/abs/2010.11929 Includes distillation token & head support for `DeiT: Data-efficient Image Transformers` - https://arxiv.org/abs/2012.12877 """ def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None, act_layer=None, weight_init='', ce_loc=None, ce_keep_ratio=None): super().__init__() if isinstance(img_size, tuple): self.img_size = img_size else: self.img_size = to_2tuple(img_size) self.patch_size = patch_size self.in_chans = in_chans self.num_classes = num_classes self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models self.num_tokens = 2 if distilled else 1 norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) act_layer = act_layer or nn.GELU self.patch_embed = embed_layer( img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4) dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule blocks = [] ce_index = 0 self.ce_loc = ce_loc for i in range(depth): ce_keep_ratio_i = 1.0 if ce_loc is not None and i in ce_loc: ce_keep_ratio_i = ce_keep_ratio[ce_index] ce_index += 1 blocks.append( CEBlock( dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer, keep_ratio_search=ce_keep_ratio_i) ) self.blocks = nn.Sequential(*blocks) self.norm = norm_layer(embed_dim) self.init_weights(weight_init) # 添加交互模块counter_guide self.counter_guide = Counter_Guide_Enhanced(768, 768) self.uep_module1 = UEPModule(256, embed_dim, embed_dim // 4, self.img_size) self.uep_module2 = UEPModule(64, embed_dim, embed_dim // 4, self.img_size) def forward_features(self, z, x, event_z, event_x, event_z2, event_x2, mask_z=None, mask_x=None, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False ): # 分支1 处理流程 B, H, W = x.shape[0], x.shape[2], x.shape[3] x = self.patch_embed(x) z = self.patch_embed(z) # x = self.uep_module1(x) z = self.uep_module2(z) # z += self.pos_embed_z x += self.pos_embed_x ################################ if mask_z is not None and mask_x is not None: mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_z = mask_z.flatten(1).unsqueeze(-1) mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_x = mask_x.flatten(1).unsqueeze(-1) mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode) mask_x = mask_x.squeeze(-1) if self.add_cls_token: cls_tokens = self.cls_token.expand(B, -1, -1) cls_tokens = cls_tokens + self.cls_pos_embed if self.add_sep_seg: x += self.search_segment_pos_embed z += self.template_segment_pos_embed x = combine_tokens(z, x, mode=self.cat_mode) if self.add_cls_token: x = torch.cat([cls_tokens, x], dim=1) x = self.pos_drop(x) lens_z = self.pos_embed_z.shape[1] lens_x = self.pos_embed_x.shape[1] global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device) global_index_t = global_index_t.repeat(B, 1) global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device) global_index_s = global_index_s.repeat(B, 1) removed_indexes_s = [] # # 分支2 处理流程 event_x = self.pos_embed_event(event_x) event_z = self.pos_embed_event(event_z) event_x += self.pos_embed_x event_z += self.pos_embed_z # event_x = self.uep_module1(event_x) event_z = self.uep_module2(event_z) #################################### event_x = combine_tokens(event_z, event_x, mode=self.cat_mode) if self.add_cls_token: event_x = torch.cat([cls_tokens, event_x], dim=1) lens_z = self.pos_embed_z.shape[1] lens_x = self.pos_embed_x.shape[1] global_index_t1 = torch.linspace(0, lens_z - 1, lens_z).to(event_x.device) global_index_t1 = global_index_t1.repeat(B, 1) global_index_s1 = torch.linspace(0, lens_x - 1, lens_x).to(event_x.device) global_index_s1 = global_index_s1.repeat(B, 1) removed_indexes_s1 = [] ## 分支3 处理流程 event_x2 = self.patch_embed(event_x2) event_z2 = self.patch_embed(event_z2) # event_x2 = self.uep_module1(event_x2) event_z2 = self.uep_module2(event_z2) # event_x2 += self.pos_embed_x event_z2 += self.pos_embed_z ############################################################ event_x2 = combine_tokens(event_z2, event_x2, mode=self.cat_mode) if self.add_cls_token: event_x2 = torch.cat([cls_tokens, event_x2], dim=1) lens_z2 = self.pos_embed_z.shape[1] lens_x2 = self.pos_embed_x.shape[1] global_index_t2 = torch.linspace(0, lens_z2 - 1, lens_z2).to(event_x2.device) global_index_t2 = global_index_t1.repeat(B, 1) global_index_s2 = torch.linspace(0, lens_x2 - 1, lens_x2).to(event_x2.device) global_index_s2 = global_index_s2.repeat(B, 1) removed_indexes_s2 = [] for i, blk in enumerate(self.blocks): # 第一个分支处理 x, global_index_t, global_index_s, removed_index_s, attn = \ blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate) # 第二个分支处理 event_x, global_index_t1, global_index_s1, removed_index_s1, attn1 = \ blk(event_x, global_index_t1, global_index_s1, mask_x, ce_template_mask, ce_keep_rate) event_x2, global_index_t2, global_index_s2, removed_index_s2, attn2 = \ blk(event_x2, global_index_t2, global_index_s2, mask_x, ce_template_mask, ce_keep_rate) ############### # 在第1层增加 counter_guide 模块,验证早期融合效果 # if i == 11: # A_original_x = x.clone() # A_original_event_x = event_x.clone() # A_original_event_x2 = event_x2.clone() # # 对 x 和 event_x 进行特征交互与增强 # enhanced_x_1, enhanced_event_x = self.counter_guide(x, event_x) # # 对 x 和 event_x2 进行特征交互与增强 # enhanced_x_2, enhanced_event_x2 = self.counter_guide(x, event_x2) # # 将增强后的特征与原特征相加 # x = x + enhanced_x_1 + enhanced_x_2 # event_x = event_x + enhanced_event_x # event_x2 = event_x2 + enhanced_event_x2 # # # 保存增强后的特征 # A_enhanced_x = enhanced_x_1 + enhanced_x_2 # A_enhanced_event_x = enhanced_event_x # A_enhanced_event_x2 = enhanced_event_x2 if i == 11: enhanced_x_1, enhanced_event_x = self.counter_guide(x, event_x) enhanced_x_2, enhanced_event_x2 = self.counter_guide(x, event_x2) x = x + enhanced_x_1 + enhanced_x_2 event_x = event_x + enhanced_event_x event_x2 = event_x2 + enhanced_event_x2 if self.ce_loc is not None and i in self.ce_loc: removed_indexes_s.append(removed_index_s) removed_indexes_s1.append(removed_index_s1) removed_indexes_s2.append(removed_index_s2) # 恢复特征tokens ############### # 应用LayerNorm归一化处理 x = self.norm(x) lens_x_new = global_index_s.shape[1] lens_z_new = global_index_t.shape[1] z = x[:, :lens_z_new] x = x[:, lens_z_new:] if removed_indexes_s and removed_indexes_s[0] is not None: removed_indexes_cat = torch.cat(removed_indexes_s, dim=1) pruned_lens_x = lens_x - lens_x_new pad_x = torch.zeros([B, pruned_lens_x, x.shape[2]], device=x.device) x = torch.cat([x, pad_x], dim=1) index_all = torch.cat([global_index_s, removed_indexes_cat], dim=1) C = x.shape[-1] x = torch.zeros_like(x).scatter(dim=1, index=index_all.unsqueeze(-1).expand(B, -1, C).to(torch.int64), src=x) x = recover_tokens(x, global_index_t.shape[1], lens_x, mode=self.cat_mode) x = x[:,:lens_x] ############################## event_x = self.norm(event_x) lens_x_new1 = global_index_s1.shape[1] lens_z_new1 = global_index_t1.shape[1] event_z = event_x[:, :lens_z_new1] event_x = event_x[:, lens_z_new1:] if removed_indexes_s1 and removed_indexes_s1[0] is not None: removed_indexes_cat1 = torch.cat(removed_indexes_s1, dim=1) pruned_lens_x1 = lens_x - lens_x_new1 pad_x1 = torch.zeros([B, pruned_lens_x1, event_x.shape[2]], device=event_x.device) event_x = torch.cat([event_x, pad_x1], dim=1) index_all1 = torch.cat([global_index_s1, removed_indexes_cat1], dim=1) C = event_x.shape[-1] event_x = torch.zeros_like(event_x).scatter_(dim=1, index=index_all1.unsqueeze(-1).expand(B, -1, C).to(torch.int64), src=event_x) event_x = recover_tokens(event_x, global_index_t1.shape[1], lens_x, mode=self.cat_mode) event_x = event_x[:,:lens_x] ######################## event_x2 = self.norm(event_x2) lens_x_new2 = global_index_s2.shape[1] lens_z_new2 = global_index_t2.shape[1] event_z2 = event_x2[:, :lens_z_new2] event_x2 = event_x2[:, lens_z_new2:] if removed_indexes_s2 and removed_indexes_s2[0] is not None: removed_indexes_cat2 = torch.cat(removed_indexes_s2, dim=1) pruned_lens_x2 = lens_x2 - lens_x_new2 pad_x2 = torch.zeros([B, pruned_lens_x2, event_x2.shape[2]], device=event_x2.device) event_x2 = torch.cat([event_x2, pad_x2], dim=1) index_all2 = torch.cat([global_index_s2, removed_indexes_cat2], dim=1) C = event_x2.shape[-1] event_x2 = torch.zeros_like(event_x2).scatter_(dim=1, index=index_all2.unsqueeze(-1).expand(B, -1, C).to(torch.int64), src=event_x2) event_x2 = recover_tokens(event_x2, global_index_t2.shape[1], lens_x2, mode=self.cat_mode) event_x2 = event_x2[:,:lens_x] ################################# # 直接拼接 # x1 = x + event_x + event_x2 x1 = x - event_x x2 = x - event_x2 x_cat = torch.cat([x1, x2], dim=1) x = x_cat aux_dict = { "attn": attn, 'attn1': attn1, 'attn2': attn2, "removed_indexes_s": removed_indexes_s, # used for visualization 'removed_indexes_s1': removed_indexes_s1, 'removed_indexes_s2': removed_indexes_s2, # 'A_original_x': A_original_x, # 'A_original_event_x': A_original_event_x, # 'A_original_event_x2': A_original_event_x2, # 'A_enhanced_x': A_enhanced_x, # 'A_enhanced_event_x': A_enhanced_event_x, # 'A_enhanced_event_x2': A_enhanced_event_x2 } return x, aux_dict def forward(self, z, x, event_z, event_x, event_z2, event_x2, ce_template_mask=None, ce_keep_rate=None, tnc_keep_rate=None, return_last_attn=False): x, aux_dict = self.forward_features(z, x, event_z, event_x, event_z2, event_x2,ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,) # print('x.shape:',x.shape) # x.shape: torch.Size([6, 960, 768]) # 320*3 = 960 (batch_size, num_tokens, embed_dim) return x, aux_dictdef _create_vision_transformer(pretrained=False, **kwargs): model = VisionTransformerCE(**kwargs)
if pretrained: if 'npz' in pretrained: model.load_pretrained(pretrained, prefix='') else: checkpoint = torch.load(pretrained, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False) print('Load pretrained model from: ' + pretrained) return modeldef vit_base_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model
def vit_large_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model中取代concat方法
- Assistant
要在这个代码中用注意力机制取代简单的
concat操作,可以使用一个自定义的注意力融合模块。这种实现方式会动态地计算每个特征分量的相关性,从而有选择地融合特征。以下是一个如何用AttentionFusion来替代concat的例子:-
确保你的
AttentionFusion模块是类似于我们之前讨论的,它有查询(query)、键(key)和值(value)的生成。同时也有注意力机制的正向传播实现。 -
在你的
VisionTransformerCE类中,取代concat的地方使用这个注意力融合机制。
下面是如何更改
forward_features()方法以使用AttentionFusion模块进行特征融合的示例:import torch import torch.nn as nn import torch.nn.functional as F class AttentionFusion(nn.Module): def __init__(self, input_dim, hidden_dim): super(AttentionFusion, self).__init__() self.query = nn.Linear(input_dim, hidden_dim) self.key = nn.Linear(input_dim, hidden_dim) self.value = nn.Linear(input_dim, input_dim) # 输出维度和输入相同 def forward(self, x1, x2): q = self.query(x1) k = self.key(x2) v = self.value(x2) attn_weights = F.softmax(torch.bmm(q, k.transpose(1, 2)) / (k.size(-1) ** 0.5), dim=-1) attn_output = torch.bmm(attn_weights, v) return attn_output class VisionTransformerCE(VisionTransformer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.attention_fusion = AttentionFusion(input_dim=768, hidden_dim=256) def forward_features(self, z, x, event_z, event_x, event_z2, event_x2, mask_z=None, mask_x=None, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False): # Existing code for processing branches... ## Following the previous logic... # Use attention fusion to combine x1 and x2: x_fused = self.attention_fusion(x1, x2) x = x_fused # Proceed with the rest of your logic... aux_dict = { "attn": attn, 'attn1': attn1, 'attn2': attn2, "removed_indexes_s": removed_indexes_s, 'removed_indexes_s1': removed_indexes_s1, 'removed_indexes_s2': removed_indexes_s2, } return x, aux_dict如何工作:
-
AttentionFusion模块中,你定义了如何计算查询、键和值,并用它们构造注意力机制。这使得系统能够为特定输入使用自适应融合,而不是简单的
concat。 -
在
VisionTransformerCE类中,你在初始化中添加了AttentionFusion,并在forward_features()方法中使用attention_fusion来处理并融合x1和x2。
在这种设计下,你能够依靠注意力机制来捕获不同输入特征之间的相关性,这可能会在某些任务上表现得更好。确保满足这些特性也能够根据特定需求对
AttentionFusion的设置(如维度、非线性激活等)进行设计。 -