在import math import logging from functools import partial from collections import…
- User
在import math import logging from functools import partial from collections import OrderedDict from copy import deepcopy
import torch import torch.nn as nn import torch.nn.functional as F
from timm.models.layers import to_2tuple
from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise from .utils import combine_tokens, recover_tokens from .vit import VisionTransformer from ..layers.attn_blocks import CEBlock import random import numpy as np
_logger = logging.getLogger(name)
class VisionTransformerCE(VisionTransformer): """ Vision Transformer with candidate elimination (CE) module
A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` - https://arxiv.org/abs/2010.11929 Includes distillation token & head support for `DeiT: Data-efficient Image Transformers` - https://arxiv.org/abs/2012.12877 """ def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None, act_layer=None, weight_init='', ce_loc=None, ce_keep_ratio=None): """ Args: img_size (int, tuple): input image size patch_size (int, tuple): patch size in_chans (int): number of input channels num_classes (int): number of classes for classification head embed_dim (int): embedding dimension depth (int): depth of transformer num_heads (int): number of attention heads mlp_ratio (int): ratio of mlp hidden dim to embedding dim qkv_bias (bool): enable bias for qkv if True representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set distilled (bool): model includes a distillation token and head as in DeiT models drop_rate (float): dropout rate attn_drop_rate (float): attention dropout rate drop_path_rate (float): stochastic depth rate embed_layer (nn.Module): patch embedding layer norm_layer: (nn.Module): normalization layer weight_init: (str): weight init scheme """ # super().__init__() super().__init__() if isinstance(img_size, tuple): self.img_size = img_size else: self.img_size = to_2tuple(img_size) self.patch_size = patch_size self.in_chans = in_chans self.num_classes = num_classes self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models self.num_tokens = 2 if distilled else 1 norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) act_layer = act_layer or nn.GELU self.patch_embed = embed_layer( img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4) # self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4) # self.pos_embed_event_z = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=3, stride=1) # attn = CrossAttn(768, 4, 3072, 0.1, 'relu') # self.cross_attn = Iter_attn(attn, 2) dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule blocks = [] ce_index = 0 self.ce_loc = ce_loc for i in range(depth): ce_keep_ratio_i = 1.0 if ce_loc is not None and i in ce_loc: ce_keep_ratio_i = ce_keep_ratio[ce_index] ce_index += 1 blocks.append( CEBlock( dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer, keep_ratio_search=ce_keep_ratio_i) ) self.blocks = nn.Sequential(*blocks) self.norm = norm_layer(embed_dim) self.init_weights(weight_init) def masking_fea(self,z, event_z, x, event_x, ratio=0.8 ): b,nz,c = z.shape b,nez,c = event_z.shape b,nx,c = x.shape b,nex,c = event_x.shape assert(nz == nez) assert(nx == nex) lenz_out = int(nz*ratio) lenx_out = int(nx*ratio) mask_nz = torch.rand(b,nz).float() mask_ez = torch.rand(b,nez).float() mask_nx = torch.rand(b,nx).float() mask_ex = torch.rand(b,nex).float() mask_nz = mask_nz>0.4 mask_ez = mask_ez>0.4 mask_ez = ~mask_nz + mask_ez mask_nz_idx = mask_nz.float().sort(1,descending=True)[-1].to(device = z.device) mask_ez_idx = mask_ez.float().sort(1,descending=True)[-1].to(device = z.device) mask_nx = mask_nx>0.4 mask_ex = mask_ex>0.4 mask_ex = ~mask_nx + mask_ex mask_nx_idx = mask_nx.float().sort(1,descending=True)[-1].to(device = z.device) mask_ex_idx = mask_ex.float().sort(1,descending=True)[-1].to(device = z.device) masked_z = torch.gather(z, 1, mask_nz_idx[:,:lenz_out,None].repeat([1,1,c])) masked_ez = torch.gather(event_z, 1, mask_ez_idx[:,:lenz_out,None].repeat([1,1,c])) masked_x = torch.gather(x, 1, mask_nx_idx[:,:lenx_out,None].repeat([1,1,c])) masked_ex = torch.gather(event_x, 1, mask_ex_idx[:,:lenx_out,None].repeat([1,1,c])) return masked_z, masked_ez, masked_x, masked_ex,{'x1':mask_nx_idx[:,:lenx_out],'x0':mask_nx_idx[:,lenx_out:], 'ex1':mask_ex_idx[:,:lenx_out],'ex0':mask_ex_idx[:,lenx_out:], } def forward_features(self, z, x, event_z, event_x, mask_z=None, mask_x=None, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False,Track=False ): B, H, W = x.shape[0], x.shape[2], x.shape[3] # print('shape of event_z before projection:{}, event_x:{}'.format(event_z.shape, event_x.shape)) event_z = self.pos_embed_event(event_z) # [:,:,:,:1000] event_x = self.pos_embed_event(event_x) # B 768 1024 x = self.patch_embed(x) z = self.patch_embed(z) # print('shape of event_z:{}, event_x:{}, x:{}, z:{}'.format(event_z.shape,event_x.shape,x.shape,z.shape )) event_z += self.pos_embed_z event_x += self.pos_embed_x z += self.pos_embed_z x += self.pos_embed_x # attention mask handling # B, H, W if mask_z is not None and mask_x is not None: mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_z = mask_z.flatten(1).unsqueeze(-1) mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_x = mask_x.flatten(1).unsqueeze(-1) mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode) mask_x = mask_x.squeeze(-1) if self.add_cls_token: cls_tokens = self.cls_token.expand(B, -1, -1) cls_tokens = cls_tokens + self.cls_pos_embed if self.add_sep_seg: x += self.search_segment_pos_embed z += self.template_segment_pos_embed if Track == False: z, event_z, x, event_x, token_idx = self.masking_fea(z, event_z, x, event_x, ratio=0.9) x = combine_tokens(z, event_z, x, event_x, mode=self.cat_mode) # 64+64+256+256=640 # x = combine_tokens(z, x, event_z, event_x, mode=self.cat_mode) # 64+64+256+256=640 if self.add_cls_token: x = torch.cat([cls_tokens, x], dim=1) x = self.pos_drop(x) # lens_z = self.pos_embed_z.shape[1] # lens_x = self.pos_embed_x.shape[1] lens_z = z.shape[1] lens_x = x.shape[1] global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device) global_index_t = global_index_t.repeat(B, 1) global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device) global_index_s = global_index_s.repeat(B, 1) removed_indexes_s = [] out_attn = [] out_attn5 = [] fifth_layer_attn = None for i, blk in enumerate(self.blocks): # out_global_s.append(global_index_s) # out_global_t.append(global_index_t) x, global_index_t, global_index_s, removed_index_s, attn = \ blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate) # if i == 4: # fifth_layer_attn = attn if self.ce_loc is not None and i in self.ce_loc: removed_indexes_s.append(removed_index_s) out_attn.append(attn) out_attn5.append(fifth_layer_attn) out_attn_idx = random.choice(np.arange(len(out_attn))) out_attn = out_attn[out_attn_idx] x = self.norm(x) lens_x_new = global_index_s.shape[1] lens_z_new = global_index_t.shape[1] z = x[:, :lens_z_new*2] x = x[:, lens_z_new*2:] if removed_indexes_s and removed_indexes_s[0] is not None: removed_indexes_cat = torch.cat(removed_indexes_s, dim=1) pruned_lens_x = lens_x - lens_x_new pad_x = torch.zeros([B, pruned_lens_x, x.shape[2]], device=x.device) x = torch.cat([x, pad_x], dim=1) index_all = torch.cat([global_index_s, removed_indexes_cat], dim=1) # recover original token order C = x.shape[-1] x = torch.zeros_like(x).scatter_(dim=1, index=index_all.unsqueeze(-1).expand(B, -1, C).to(torch.int64), src=x) x = recover_tokens(x, lens_z_new, lens_x, mode=self.cat_mode) # event_x = x[:, lens_x:] # RGB head x = x[:, :lens_x] # RGB head x = torch.cat([event_x, x], dim=1) aux_dict = { # "attn": attn, "attn": out_attn, # "fifth_layer_attn": out_attn5, "removed_indexes_s": removed_indexes_s, # used for visualization } return x, aux_dict def forward(self, z, x, event_z, event_x, ce_template_mask=None, ce_keep_rate=None, tnc_keep_rate=None, return_last_attn=False,Track=False): x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,Track=Track) return x, aux_dictdef _create_vision_transformer(pretrained=False, **kwargs): model = VisionTransformerCE(**kwargs)
if pretrained: if 'npz' in pretrained: model.load_pretrained(pretrained, prefix='') else: checkpoint = torch.load(pretrained, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False) print('Load pretrained model from: ' + pretrained) return modeldef vit_base_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model
def vit_large_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model中,rgb的search和template元素和event的search和template元素分别是如何排列的,以及分别是多少个?
- Assistant
在给定的代码中,对于排列和数量的处理包括以下详细步骤:
-
模型初始化:
- 初始化阶段,模型设置了
self.patch_embed(针对 RGB 输入)和self.pos_embed_event(针对 Event 输入)作为特征提取组件。 self.patch_size被设置为 16。
- 初始化阶段,模型设置了
-
特征提取:
- 模型分别通过
self.patch_embed和self.pos_embed_event来处理 RGB 图像 (x和z) 与 Event 数据 (event_x和event_z)。 - 对于 RGB 图像 (
x和z),假设输入图像尺寸为(224, 224), 使用patch_size16,每个图像被分割为(224/16) * (224/16) = 14 * 14 = 196个patches。
- 模型分别通过
-
模型前向传播 (
forward_features方法):- RGB 和 Event 的特征通过相应的
PatchEmbed和PatchEmbed_event进行转换后,通过加入位置嵌入 (pos_embed和pos_embed_event) 进行增强。 - 组合特征: 如
combine_tokens方法将 RGB 和 Event 的特征合并。根据代码逻辑概述,combine_tokens管理 RGB 和 Event 数据的顺序,细节依赖于self.cat_mode,但一般来说,可能是顺序连接或其他形式的组合。 - 特征遮掩与选择:
masking_fea方法实现对特征的随机掩膜和选择,影响了特征的最终组成。 - 特征提取完成后,模型沿用 Transformer 所有层对动态调整和选择后的合并特征进行处理。
- RGB 和 Event 的特征通过相应的
-
特征维度和数量:
z,event_z,x,event_x初始化后,均为[batch_size, num_patches, embed_dim]。- 给定 RGB 图像每个为196个patches,每个 Event 数据同样按照分块逻辑处理(
self.pos_embed_event按照内部设置处理,假设与 RGB 相同也是196个patches),则z,x分别对应 RGB 数据的模板(template)和搜索(search)部分,数量为196。 event_z,event_x对应 Event 数据的模板和搜索部分,数量同样为196。
在您的代码框架中,深入的逻辑(如具体的组合方式及其细节)需要在具体的
self.cat_mode设置及combine_tokens实现细节里查看。如需明确细节,建议进一步检查这些方法的实现或提供更多相关信息。 -