-------------------------------------------------------- Swin Transformer V2 Copyright…
- User
--------------------------------------------------------
Swin Transformer V2
Copyright (c) 2022 Microsoft
Licensed under The MIT License [see LICENSE for details]
Written by Ze Liu
--------------------------------------------------------
import os import torch import torch.nn as nn import torch.nn.functional as F import torch.utils.checkpoint as checkpoint from timm.models.layers import DropPath, to_2tuple, trunc_normal_ import numpy as np from collections import OrderedDict
class Mlp(nn.Module): """ Multilayer perceptron."""
def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): super().__init__() out_features = out_features or in_features hidden_features = hidden_features or in_features self.fc1 = nn.Linear(in_features, hidden_features) self.act = act_layer() self.fc2 = nn.Linear(hidden_features, out_features) self.drop = nn.Dropout(drop) def forward(self, x): x = self.fc1(x) x = self.act(x) x = self.drop(x) x = self.fc2(x) x = self.drop(x) return xdef window_partition(x, window_size): """ Args: x: (B, H, W, C) window_size (int): window size
Returns: windows: (num_windows*B, window_size, window_size, C) """ B, H, W, C = x.shape x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) return windowsdef window_reverse(windows, window_size, H, W): """ Args: windows: (num_windows*B, window_size, window_size, C) window_size (int): Window size H (int): Height of image W (int): Width of image
Returns: x: (B, H, W, C) """ B = int(windows.shape[0] / (H * W / window_size / window_size)) x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) return xclass WindowAttention(nn.Module): """ Window based multi-head self attention (W-MSA) module with relative position bias. It supports both of shifted and non-shifted window.
Args: dim (int): Number of input channels. window_size (tuple[int]): The height and width of the window. num_heads (int): Number of attention heads. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 proj_drop (float, optional): Dropout ratio of output. Default: 0.0 pretrained_window_size (tuple[int]): The height and width of the window in pre-training. """ def __init__(self, dim, window_size, num_heads, qkv_bias=True, attn_drop=0., proj_drop=0., pretrained_window_size=[0, 0]): super().__init__() self.dim = dim self.window_size = window_size # Wh, Ww self.pretrained_window_size = pretrained_window_size self.num_heads = num_heads self.logit_scale = nn.Parameter(torch.log(10 * torch.ones((num_heads, 1, 1))), requires_grad=True) # mlp to generate continuous relative position bias self.cpb_mlp = nn.Sequential(nn.Linear(2, 512, bias=True), nn.ReLU(inplace=True), nn.Linear(512, num_heads, bias=False)) # get relative_coords_table relative_coords_h = torch.arange(-(self.window_size[0] - 1), self.window_size[0], dtype=torch.float32) relative_coords_w = torch.arange(-(self.window_size[1] - 1), self.window_size[1], dtype=torch.float32) relative_coords_table = torch.stack( torch.meshgrid([relative_coords_h, relative_coords_w])).permute(1, 2, 0).contiguous().unsqueeze(0) # 1, 2*Wh-1, 2*Ww-1, 2 if pretrained_window_size[0] > 0: relative_coords_table[:, :, :, 0] /= (pretrained_window_size[0] - 1) relative_coords_table[:, :, :, 1] /= (pretrained_window_size[1] - 1) else: relative_coords_table[:, :, :, 0] /= (self.window_size[0] - 1) relative_coords_table[:, :, :, 1] /= (self.window_size[1] - 1) relative_coords_table *= 8 # normalize to -8, 8 relative_coords_table = torch.sign(relative_coords_table) * torch.log2( torch.abs(relative_coords_table) + 1.0) / np.log2(8) self.register_buffer("relative_coords_table", relative_coords_table) # get pair-wise relative position index for each token inside the window coords_h = torch.arange(self.window_size[0]) coords_w = torch.arange(self.window_size[1]) coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 relative_coords[:, :, 1] += self.window_size[1] - 1 relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww self.register_buffer("relative_position_index", relative_position_index) self.qkv = nn.Linear(dim, dim * 3, bias=False) if qkv_bias: self.q_bias = nn.Parameter(torch.zeros(dim)) self.v_bias = nn.Parameter(torch.zeros(dim)) else: self.q_bias = None self.v_bias = None self.attn_drop = nn.Dropout(attn_drop) self.proj = nn.Linear(dim, dim) self.proj_drop = nn.Dropout(proj_drop) self.softmax = nn.Softmax(dim=-1) def forward(self, x, mask=None): """ Args: x: input features with shape of (num_windows*B, N, C) mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None """ B_, N, C = x.shape qkv_bias = None if self.q_bias is not None: qkv_bias = torch.cat((self.q_bias, torch.zeros_like(self.v_bias, requires_grad=False), self.v_bias)) qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias) qkv = qkv.reshape(B_, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) # cosine attention attn = (F.normalize(q, dim=-1) @ F.normalize(k, dim=-1).transpose(-2, -1)) logit_scale = torch.clamp(self.logit_scale, max=torch.log(torch.tensor(1. / 0.01))).exp() attn = attn * logit_scale relative_position_bias_table = self.cpb_mlp(self.relative_coords_table).view(-1, self.num_heads) relative_position_bias = relative_position_bias_table[self.relative_position_index.view(-1)].view( self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1) # Wh*Ww,Wh*Ww,nH relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww relative_position_bias = 16 * torch.sigmoid(relative_position_bias) attn = attn + relative_position_bias.unsqueeze(0) if mask is not None: nW = mask.shape[0] attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) attn = attn.view(-1, self.num_heads, N, N) attn = self.softmax(attn) else: attn = self.softmax(attn) attn = self.attn_drop(attn) x = (attn @ v).transpose(1, 2).reshape(B_, N, C) x = self.proj(x) x = self.proj_drop(x) return x def extra_repr(self) -> str: return f'dim={self.dim}, window_size={self.window_size}, ' \ f'pretrained_window_size={self.pretrained_window_size}, num_heads={self.num_heads}' def flops(self, N): # calculate flops for 1 window with token length of N flops = 0 # qkv = self.qkv(x) flops += N * self.dim * 3 * self.dim # attn = (q @ k.transpose(-2, -1)) flops += self.num_heads * N * (self.dim // self.num_heads) * N # x = (attn @ v) flops += self.num_heads * N * N * (self.dim // self.num_heads) # x = self.proj(x) flops += N * self.dim * self.dim return flopsclass SwinTransformerBlock(nn.Module): r""" Swin Transformer Block.
Args: dim (int): Number of input channels. num_heads (int): Number of attention heads. window_size (int): Window size. shift_size (int): Shift size for SW-MSA. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True drop (float, optional): Dropout rate. Default: 0.0 attn_drop (float, optional): Attention dropout rate. Default: 0.0 drop_path (float, optional): Stochastic depth rate. Default: 0.0 act_layer (nn.Module, optional): Activation layer. Default: nn.GELU norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm pretrained_window_size (int): Window size in pre-training. """ def __init__(self, dim, num_heads, window_size=7, shift_size=0, mlp_ratio=4., qkv_bias=True, drop=0., attn_drop=0., drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, pretrained_window_size=0): super().__init__() self.dim = dim self.num_heads = num_heads self.window_size = window_size self.shift_size = shift_size self.mlp_ratio = mlp_ratio assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" self.norm1 = norm_layer(dim) self.attn = WindowAttention( dim, window_size=to_2tuple(self.window_size), num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop, pretrained_window_size=to_2tuple(pretrained_window_size)) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() self.norm2 = norm_layer(dim) mlp_hidden_dim = int(dim * mlp_ratio) self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) self.H = None self.W = None def forward(self, x, mask_matrix): """ Forward function. Args: x: Input feature, tensor size (B, H*W, C). H, W: Spatial resolution of the input feature. mask_matrix: Attention mask for cyclic shift. """ B, L, C = x.shape H, W = self.H, self.W assert L == H * W, "input feature has wrong size" shortcut = x x = x.view(B, H, W, C) # pad feature maps to multiples of window size pad_l = pad_t = 0 pad_r = (self.window_size - W % self.window_size) % self.window_size pad_b = (self.window_size - H % self.window_size) % self.window_size x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) _, Hp, Wp, _ = x.shape # cyclic shift if self.shift_size > 0: shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) attn_mask = mask_matrix else: shifted_x = x attn_mask = None # partition windows x_windows = window_partition(shifted_x, self.window_size) # nW*B, window_size, window_size, C x_windows = x_windows.view(-1, self.window_size * self.window_size, C) # nW*B, window_size*window_size, C # W-MSA/SW-MSA attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C # merge windows attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp) # B H' W' C # reverse cyclic shift if self.shift_size > 0: x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) else: x = shifted_x if pad_r > 0 or pad_b > 0: x = x[:, :H, :W, :].contiguous() x = x.view(B, H * W, C) x = shortcut + self.drop_path(self.norm1(x)) # FFN x = x + self.drop_path(self.norm2(self.mlp(x))) return xclass PatchMerging(nn.Module): r""" Patch Merging Layer.
Args: dim (int): Number of input channels. norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm """ def __init__(self, dim, norm_layer=nn.LayerNorm): super().__init__() self.dim = dim self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) self.norm = norm_layer(2 * dim) def forward(self, x, H, W): B, L, C = x.shape assert L == H * W, "input feature has wrong size" assert H % 2 == 0 and W % 2 == 0, f"x size ({H}*{W}) are not even." x = x.view(B, H, W, C) # padding pad_input = (H % 2 == 1) or (W % 2 == 1) if pad_input: x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2)) x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C x = self.reduction(x) x = self.norm(x) return xclass BasicLayer(nn.Module): """ A basic Swin Transformer layer for one stage.
Args: dim (int): Number of input channels. depth (int): Number of blocks. num_heads (int): Number of attention heads. window_size (int): Local window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True drop (float, optional): Dropout rate. Default: 0.0 attn_drop (float, optional): Attention dropout rate. Default: 0.0 drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. pretrained_window_size (int): Local window size in pre-training. """ def __init__(self, dim, depth, num_heads, window_size, mlp_ratio=4., qkv_bias=True, drop=0., attn_drop=0., drop_path=0., norm_layer=nn.LayerNorm, downsample=None, use_checkpoint=False, pretrained_window_size=0): super().__init__() self.window_size = window_size self.shift_size = window_size // 2 self.depth = depth self.use_checkpoint = use_checkpoint # build blocks self.blocks = nn.ModuleList([ SwinTransformerBlock(dim=dim, num_heads=num_heads, window_size=window_size, shift_size=0 if (i % 2 == 0) else window_size // 2, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop, attn_drop=attn_drop, drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, norm_layer=norm_layer, pretrained_window_size=pretrained_window_size) for i in range(depth)]) # patch merging layer if downsample is not None: self.downsample = downsample(dim=dim, norm_layer=norm_layer) else: self.downsample = None def forward(self, x, H, W): """ Forward function. Args: x: Input feature, tensor size (B, H*W, C). H, W: Spatial resolution of the input feature. """ # calculate attention mask for SW-MSA Hp = int(np.ceil(H / self.window_size)) * self.window_size Wp = int(np.ceil(W / self.window_size)) * self.window_size img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device) # 1 Hp Wp 1 h_slices = (slice(0, -self.window_size), slice(-self.window_size, -self.shift_size), slice(-self.shift_size, None)) w_slices = (slice(0, -self.window_size), slice(-self.window_size, -self.shift_size), slice(-self.shift_size, None)) cnt = 0 for h in h_slices: for w in w_slices: img_mask[:, h, w, :] = cnt cnt += 1 mask_windows = window_partition(img_mask, self.window_size) # nW, window_size, window_size, 1 mask_windows = mask_windows.view(-1, self.window_size * self.window_size) attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0)) for blk in self.blocks: blk.H, blk.W = H, W if self.use_checkpoint: x = checkpoint.checkpoint(blk, x, attn_mask) else: x = blk(x, attn_mask) if self.downsample is not None: x_down = self.downsample(x, H, W) Wh, Ww = (H + 1) // 2, (W + 1) // 2 return x, H, W, x_down, Wh, Ww else: return x, H, W, x, H, Wclass PatchEmbed(nn.Module): r""" Image to Patch Embedding
Args: patch_size (int): Patch token size. Default: 4. in_chans (int): Number of input image channels. Default: 3. embed_dim (int): Number of linear projection output channels. Default: 96. norm_layer (nn.Module, optional): Normalization layer. Default: None """ def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None): super().__init__() patch_size = to_2tuple(patch_size) self.patch_size = patch_size self.in_chans = in_chans self.embed_dim = embed_dim self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) if norm_layer is not None: self.norm = norm_layer(embed_dim) else: self.norm = None def forward(self, x): """Forward function.""" # padding _, _, H, W = x.size() if W % self.patch_size[1] != 0: x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1])) if H % self.patch_size[0] != 0: x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0])) x = self.proj(x) # B C Wh Ww if self.norm is not None: Wh, Ww = x.size(2), x.size(3) x = x.flatten(2).transpose(1, 2) x = self.norm(x) x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww) return xclass SwinTransformer(nn.Module): """ Swin Transformer backbone. A PyTorch impl of :
Swin Transformer: Hierarchical Vision Transformer using Shifted Windows- https://arxiv.org/pdf/2103.14030Args: pretrain_img_size (int): Input image size for training the pretrained model, used in absolute postion embedding. Default 224. patch_size (int | tuple(int)): Patch size. Default: 4. in_chans (int): Number of input image channels. Default: 3. embed_dim (int): Number of linear projection output channels. Default: 96. depths (tuple[int]): Depths of each Swin Transformer stage. num_heads (tuple[int]): Number of attention head of each stage. window_size (int): Window size. Default: 7. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True drop_rate (float): Dropout rate. attn_drop_rate (float): Attention dropout rate. Default: 0. drop_path_rate (float): Stochastic depth rate. Default: 0.2. norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. ape (bool): If True, add absolute position embedding to the patch embedding. Default: False. patch_norm (bool): If True, add normalization after patch embedding. Default: True. out_indices (Sequence[int]): Output from which stages. frozen_stages (int): Stages to be frozen (stop grad and set eval mode). -1 means not freezing any parameters. use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. """ def __init__(self, pretrain_img_size=224, patch_size=4, in_chans=3, embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24], window_size=7, mlp_ratio=4., qkv_bias=True, drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1, norm_layer=nn.LayerNorm, ape=False, patch_norm=True, out_indices=(0, 1, 2, 3), frozen_stages=-1, use_checkpoint=False, pretrained_window_size=[0, 0, 0, 0]): super().__init__() self.pretrain_img_size = pretrain_img_size self.num_layers = len(depths) self.embed_dim = embed_dim self.ape = ape self.patch_norm = patch_norm self.out_indices = out_indices self.frozen_stages = frozen_stages # split image into non-overlapping patches self.patch_embed = PatchEmbed( patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim, norm_layer=norm_layer if self.patch_norm else None) # absolute position embedding if self.ape: pretrain_img_size = to_2tuple(pretrain_img_size) patch_size = to_2tuple(patch_size) patches_resolution = [pretrain_img_size[0] // patch_size[0], pretrain_img_size[1] // patch_size[1]] self.absolute_pos_embed = nn.Parameter( torch.zeros(1, embed_dim, patches_resolution[0], patches_resolution[1])) trunc_normal_(self.absolute_pos_embed, std=.02) self.pos_drop = nn.Dropout(p=drop_rate) # stochastic depth dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] # stochastic depth decay rule # build layers self.layers = nn.ModuleList() for i_layer in range(self.num_layers-1): layer = BasicLayer( dim=int(embed_dim * 2 ** i_layer), depth=depths[i_layer], num_heads=num_heads[i_layer], window_size=window_size, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])], norm_layer=norm_layer, downsample=PatchMerging if (i_layer < self.num_layers - 2) else None, use_checkpoint=use_checkpoint, pretrained_window_size=pretrained_window_size[i_layer]) self.layers.append(layer) num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)] self.num_features = num_features self.apply(self._init_weights) # add a norm layer for each output for i_layer in out_indices: layer = norm_layer(num_features[i_layer]) layer_name = f'norm{i_layer}' self.add_module(layer_name, layer) self._freeze_stages() def _freeze_stages(self): if self.frozen_stages >= 0: self.patch_embed.eval() for param in self.patch_embed.parameters(): param.requires_grad = False if self.frozen_stages >= 1 and self.ape: self.absolute_pos_embed.requires_grad = False if self.frozen_stages >= 2: self.pos_drop.eval() for i in range(0, self.frozen_stages - 1): m = self.layers[i] m.eval() for param in m.parameters(): param.requires_grad = False def _init_weights(self, m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.LayerNorm): nn.init.constant_(m.bias, 0) nn.init.constant_(m.weight, 1.0) def forward(self, x): x = self.patch_embed(x) Wh, Ww = x.size(2), x.size(3) if self.ape: # interpolate the position embedding to the corresponding size absolute_pos_embed = F.interpolate(self.absolute_pos_embed, size=(Wh, Ww), mode='bicubic') x = (x + absolute_pos_embed).flatten(2).transpose(1, 2) # B Wh*Ww C else: x = x.flatten(2).transpose(1, 2) x = self.pos_drop(x) outs = [] for i in range(3): layer = self.layers[i] x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) if i in self.out_indices: norm_layer = getattr(self, f'norm{i}') x_out = norm_layer(x_out) out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous() outs.append(out) outputs = OrderedDict() outputs['layer1'] = outs[0] outputs['layer2'] = outs[1] outputs['layer3'] = outs[2] return outputs def flops(self): flops = 0 flops += self.patch_embed.flops() for i, layer in enumerate(self.layers): flops += layer.flops() flops += self.num_features * self.patches_resolution[0] * self.patches_resolution[1] // (2 ** self.num_layers) flops += self.num_features * self.num_classes return flopsdef get_model_config(model_name): default_config = { 'pretrain_img_size': 256, 'patch_size': 4, 'in_chans': 3, 'embed_dim': 96, 'depths': [2, 2, 6, 2], 'num_heads': [3, 6, 12, 24], 'window_size': 7, 'mlp_ratio': 4., 'qkv_bias': True, 'drop_rate': 0.0, 'drop_path_rate': 0.1, 'ape': False, 'patch_norm': True, 'use_checkpoint': False, 'out_indices': (0, 1, 2, 3), } if model_name == 'swin_t8': default_config['drop_path_rate'] = 0.2 default_config['embed_dim'] = 96 default_config['depths'] = [2, 2, 6, 2] default_config['num_heads'] = [3, 6, 12, 24] default_config['window_size'] = 8 default_config['out_indices'] = (0, 1, 2)
elif model_name == 'swin_t16': default_config['drop_path_rate'] = 0.2 default_config['embed_dim'] = 96 default_config['depths'] = [2, 2, 6, 2] default_config['num_heads'] = [3, 6, 12, 24] default_config['window_size'] = 16 default_config['out_indices'] = (0, 1, 2) elif model_name == 'swin_s': default_config['drop_path_rate'] = 0.3 default_config['embed_dim'] = 96 default_config['depths'] = [2, 2, 18, 2] default_config['num_heads'] = [3, 6, 12, 24] default_config['window_size'] = 8 default_config['out_indices'] = (0, 1, 2) else: print('error') exit() return default_configdef get_model(config): model = SwinTransformer(pretrain_img_size=config['pretrain_img_size'], patch_size=config['patch_size'], in_chans=config['in_chans'], embed_dim=config['embed_dim'], depths=config['depths'], num_heads=config['num_heads'], window_size=config['window_size'], mlp_ratio=config['mlp_ratio'], qkv_bias=config['qkv_bias'], drop_rate=config['drop_rate'], drop_path_rate=config['drop_path_rate'], ape=config['ape'], patch_norm=config['patch_norm'], out_indices=config['out_indices'], use_checkpoint=config['use_checkpoint']) return model
def swint_v2(): """swin_t8 window size=8 """ model_config = get_model_config(model_name='swin_t8') pth_path = os.path.dirname(file) + '/swinv2_tiny_patch4_window8_256.pth'
"""swin_t16 window size=16""" # model_config = get_model_config(model_name='swin_t16') # pth_path = os.path.dirname(__file__) + '/swinv2_tiny_patch4_window16_256.pth' """swin_s""" # model_config = get_model_config(model_name='swin_s') # pth_path = os.path.dirname(__file__) + '/swinv2_small_patch4_window8_256.pth' model = get_model(model_config) pth_dict = torch.load(pth_path) pth_dict = pth_dict['model'] model_dict = model.state_dict() jieduan_pth_dict = {} for k, v in pth_dict.items(): if 'attn_mask' in str(k): continue if str(k).startswith('layers.2.downsample'): break jieduan_pth_dict[k] = v model_dict.update(jieduan_pth_dict) model.load_state_dict(model_dict) del jieduan_pth_dict del model_dict torch.cuda.empty_cache() return modelif name == 'main': os.environ['CUDA_VISIBLE_DEVICES'] = '7' model = swint_v2().cuda() print(model) img = torch.ones(2, 3, 256, 256).cuda() img2 = torch.ones(2, 3, 128, 128).cuda() res = model(img) res2 = model(img2) print(res['layer1'].shape) print(res['layer2'].shape) print(res['layer3'].shape) print(res2['layer1'].shape) print(res2['layer2'].shape) print(res2['layer3'].shape)
n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) print('n_parameters:', n_parameters) 中文分析 - Assistant
该代码实现了Swin Transformer V2,这是一个视觉变换模型,主要用于图像处理任务。以下是对代码中关键部分的中文分析:
1. Mlp 类
- 作用:多层感知器(MLP)模块,处理馈送前向网络。
- 构成:包括两个全连接层(
nn.Linear),一个激活函数(如GELU),和一个Dropout。
2. window_partition 和 window_reverse 函数
- window_partition:将图像的特征图划分为多个窗口,用于局部注意力计算。
- window_reverse:将窗口级别的特征图恢复成原来的空间布局。
3. WindowAttention 类
- 作用:通过窗口方式应用自注意力机制。支持相对位置偏差。
- 构成:包括线性层用于投影查询、键、值(qkv),用于注意力计算的softmax层及projections。
4. SwinTransformerBlock 类
- 作用:实现Swin Transformer的基础块,结合窗口(局部)自注意力机制和多层感知器。
- 构成:包括窗口注意力层、MLP层、正则化层(如LayerNorm)和drop路径(DropPath)。
5. PatchMerging 类
- 作用:用于逐层缩小特征图的分辨率。
- 构成:通过减少特征图的空间维度,并投影到更高的特征维度。
6. BasicLayer 类
- 作用:在一个阶段中应用多个Swin Transformer Block。
- 构成:包括多个Swin Transformer Block和一个到下一个阶段的下采样层。
7. PatchEmbed 类
- 作用:将输入图像划分为不重叠的补丁,并进行线性嵌入。
- 构成:使用卷积层实现图像到patch的转换。
8. SwinTransformer 类
- 主架构:实现Swin Transformer的完整模型。
- 特性:
- 支持绝对位置嵌入。
- 分阶段的特征处理,每个阶段分别由BasicLayer构成。
- 输出指定阶段的特征。
9. get_model_config 和 get_model 函数
- get_model_config:根据模型名称返回相应的默认配置。
- get_model:根据配置实例化Swin Transformer模型。
10. swint_v2 函数和脚本末尾部分
- swint_v2:加载预训练模型参数,并构建和返回带有配置的SwinTransformer模型。
- 脚本测试:创建模型实例并输入示例数据进行测试,同时计算模型的参数数量。
总结
该代码实现了Swin Transformer V2的PyTorch版本,通过定义多种模块实现了图像的分阶段嵌入、特征提取、窗口级注意力机制,以及特征的下采样与归一化,是用作图像分类或其他视觉任务的强有力模型。
语言处理
本文档代码使用了Python和PyTorch,并且非常重视模块化设计,同时在使用GPU加速运算。
即便不熟悉视觉变换模型的细节,只要具备一定的PyTorch框架使用经验,也可以在此基础上进一步修改或实现更多定制功能。
- User
再分析一下:import torch import torch.nn as nn from torch.nn.functional import linear, pad from torch import Tensor from torch.nn import MultiheadAttention
from typing import Optional, Tuple, List import warnings
def multi_head_attention_forward( query: Tensor, key: Tensor, value: Tensor, embed_dim_to_check: int, num_heads: int, in_proj_weight: Tensor, in_proj_bias: Tensor, bias_k: Optional[Tensor], bias_v: Optional[Tensor], add_zero_attn: bool, dropout_p: float, out_proj_weight: Tensor, out_proj_bias: Tensor, training: bool = True, key_padding_mask: Optional[Tensor] = None, need_weights: bool = True, attn_mask: Optional[Tensor] = None, use_separate_proj_weight: bool = False, q_proj_weight: Optional[Tensor] = None, k_proj_weight: Optional[Tensor] = None, v_proj_weight: Optional[Tensor] = None, static_k: Optional[Tensor] = None, static_v: Optional[Tensor] = None, gaussian: Optional[Tensor] = None, lamb = None, ) -> Tuple[Tensor, Optional[Tensor]]: r""" Args: query, key, value: map a query and a set of key-value pairs to an output. See "Attention Is All You Need" for more details. embed_dim_to_check: total dimension of the model. num_heads: parallel attention heads. in_proj_weight, in_proj_bias: input projection weight and bias. bias_k, bias_v: bias of the key and value sequences to be added at dim=0. add_zero_attn: add a new batch of zeros to the key and value sequences at dim=1. dropout_p: probability of an element to be zeroed. out_proj_weight, out_proj_bias: the output projection weight and bias. training: apply dropout if is
True. key_padding_mask: if provided, specified padding elements in the key will be ignored by the attention. This is an binary mask. When the value is True, the corresponding value on the attention layer will be filled with -inf. need_weights: output attn_output_weights. attn_mask: 2D or 3D mask that prevents attention to certain positions. This is an additive mask (i.e. the values will be added to the attention layer). A 2D mask will be broadcasted for all the batches while a 3D mask allows to specify a different mask for the entries of each batch. use_separate_proj_weight: the function accept the proj. weights for query, key, and value in different forms. If false, in_proj_weight will be used, which is a combination of q_proj_weight, k_proj_weight, v_proj_weight. q_proj_weight, k_proj_weight, v_proj_weight, in_proj_bias: input projection weight and bias. static_k, static_v: static key and value used for attention operators. gaussian: the generated Gaussian-like weight mapShape: Inputs: - query: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is the embedding dimension. - key: :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is the embedding dimension. - value: :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is the embedding dimension. - key_padding_mask: :math:`(N, S)` where N is the batch size, S is the source sequence length. If a ByteTensor is provided, the non-zero positions will be ignored while the zero positions will be unchanged. If a BoolTensor is provided, the positions with the value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged. - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length. 3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length, S is the source sequence length. attn_mask ensures that position i is allowed to attend the unmasked positions. If a ByteTensor is provided, the non-zero positions are not allowed to attend while the zero positions will be unchanged. If a BoolTensor is provided, positions with ``True`` are not allowed to attend while ``False`` values will be unchanged. If a FloatTensor is provided, it will be added to the attention weight. - static_k: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length, N is the batch size, E is the embedding dimension. E/num_heads is the head dimension. - static_v: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length, N is the batch size, E is the embedding dimension. E/num_heads is the head dimension. Outputs: - attn_output: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is the embedding dimension. - attn_output_weights: :math:`(N, L, S)` where N is the batch size, L is the target sequence length, S is the source sequence length. """ tgt_len, bsz, embed_dim = query.size() assert embed_dim == embed_dim_to_check # allow MHA to have different sizes for the feature dimension assert key.size(0) == value.size(0) and key.size(1) == value.size(1) head_dim = embed_dim // num_heads assert head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads" scaling = float(head_dim) ** -0.5 if not use_separate_proj_weight: if torch.equal(query, key) and torch.equal(key, value): # self-attention q, k, v = linear(query, in_proj_weight, in_proj_bias).chunk(3, dim=-1) elif torch.equal(key, value): # encoder-decoder attention # This is inline in_proj function with in_proj_weight and in_proj_bias _b = in_proj_bias _start = 0 _end = embed_dim _w = in_proj_weight[_start:_end, :] if _b is not None: _b = _b[_start:_end] q = linear(query, _w, _b) if key is None: assert value is None k = None v = None else: # This is inline in_proj function with in_proj_weight and in_proj_bias _b = in_proj_bias _start = embed_dim _end = None _w = in_proj_weight[_start:, :] if _b is not None: _b = _b[_start:] k, v = linear(key, _w, _b).chunk(2, dim=-1) else: # This is inline in_proj function with in_proj_weight and in_proj_bias _b = in_proj_bias _start = 0 _end = embed_dim _w = in_proj_weight[_start:_end, :] if _b is not None: _b = _b[_start:_end] q = linear(query, _w, _b) # This is inline in_proj function with in_proj_weight and in_proj_bias _b = in_proj_bias _start = embed_dim _end = embed_dim * 2 _w = in_proj_weight[_start:_end, :] if _b is not None: _b = _b[_start:_end] k = linear(key, _w, _b) # This is inline in_proj function with in_proj_weight and in_proj_bias _b = in_proj_bias _start = embed_dim * 2 _end = None _w = in_proj_weight[_start:, :] if _b is not None: _b = _b[_start:] v = linear(value, _w, _b) else: q_proj_weight_non_opt = torch.jit._unwrap_optional(q_proj_weight) len1, len2 = q_proj_weight_non_opt.size() assert len1 == embed_dim and len2 == query.size(-1) k_proj_weight_non_opt = torch.jit._unwrap_optional(k_proj_weight) len1, len2 = k_proj_weight_non_opt.size() assert len1 == embed_dim and len2 == key.size(-1) v_proj_weight_non_opt = torch.jit._unwrap_optional(v_proj_weight) len1, len2 = v_proj_weight_non_opt.size() assert len1 == embed_dim and len2 == value.size(-1) if in_proj_bias is not None: q = linear(query, q_proj_weight_non_opt, in_proj_bias[0:embed_dim]) k = linear(key, k_proj_weight_non_opt, in_proj_bias[embed_dim:(embed_dim * 2)]) v = linear(value, v_proj_weight_non_opt, in_proj_bias[(embed_dim * 2):]) else: q = linear(query, q_proj_weight_non_opt, in_proj_bias) k = linear(key, k_proj_weight_non_opt, in_proj_bias) v = linear(value, v_proj_weight_non_opt, in_proj_bias) q = q * scaling if attn_mask is not None: if attn_mask.dim() == 2: attn_mask = attn_mask.unsqueeze(0) if list(attn_mask.size()) != [1, query.size(0), key.size(0)]: raise RuntimeError('The size of the 2D attn_mask is not correct.') elif attn_mask.dim() == 3: if list(attn_mask.size()) != [bsz * num_heads, query.size(0), key.size(0)]: raise RuntimeError('The size of the 3D attn_mask is not correct.') else: raise RuntimeError("attn_mask's dimension {} is not supported".format(attn_mask.dim())) # attn_mask's dim is 3 now. if bias_k is not None and bias_v is not None: if static_k is None and static_v is None: k = torch.cat([k, bias_k.repeat(1, bsz, 1)]) v = torch.cat([v, bias_v.repeat(1, bsz, 1)]) if attn_mask is not None: attn_mask = pad(attn_mask, (0, 1)) if key_padding_mask is not None: key_padding_mask = pad(key_padding_mask, (0, 1)) else: assert static_k is None, "bias cannot be added to static key." assert static_v is None, "bias cannot be added to static value." else: assert bias_k is None assert bias_v is None q = q.contiguous().view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1) if k is not None: k = k.contiguous().view(-1, bsz * num_heads, head_dim).transpose(0, 1) if v is not None: v = v.contiguous().view(-1, bsz * num_heads, head_dim).transpose(0, 1) if static_k is not None: assert static_k.size(0) == bsz * num_heads assert static_k.size(2) == head_dim k = static_k if static_v is not None: assert static_v.size(0) == bsz * num_heads assert static_v.size(2) == head_dim v = static_v src_len = k.size(1) if key_padding_mask is not None: assert key_padding_mask.size(0) == bsz assert key_padding_mask.size(1) == src_len if add_zero_attn: src_len += 1 k = torch.cat([k, torch.zeros((k.size(0), 1) + k.size()[2:], dtype=k.dtype, device=k.device)], dim=1) v = torch.cat([v, torch.zeros((v.size(0), 1) + v.size()[2:], dtype=v.dtype, device=v.device)], dim=1) if attn_mask is not None: attn_mask = pad(attn_mask, (0, 1)) if key_padding_mask is not None: key_padding_mask = pad(key_padding_mask, (0, 1)) attn_output_weights = torch.bmm(q, k.transpose(1, 2)) assert list(attn_output_weights.size()) == [bsz * num_heads, tgt_len, src_len] if attn_mask is not None: attn_output_weights += attn_mask if key_padding_mask is not None: attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len) attn_output_weights = attn_output_weights.masked_fill( key_padding_mask.unsqueeze(1).unsqueeze(2), float('-inf'),) attn_output_weights = attn_output_weights.view(bsz * num_heads, tgt_len, src_len) attn_output_weights = attn_output_weights + gaussian[0].permute(2, 0, 1) attn_output_weights = torch.nn.functional.softmax(attn_output_weights, dim=-1) # High-Frequency Emphasis HFE attn_d = torch.ones(attn_output_weights.shape[-2:], device=attn_output_weights.device)/tgt_len attn_d = attn_d[None, None, ...] attn_h = attn_output_weights.view(bsz, num_heads, tgt_len, src_len) - attn_d attn_h = attn_h * (1. + lamb[None, :, None, None]) attn_output_weights = (attn_d + attn_h).view(bsz*num_heads, tgt_len, src_len) attn_output_weights = torch.nn.functional.dropout(attn_output_weights, p=dropout_p, training=training) attn_output = torch.bmm(attn_output_weights, v) assert list(attn_output.size()) == [bsz * num_heads, tgt_len, head_dim] attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim) attn_output = linear(attn_output, out_proj_weight, out_proj_bias) return attn_output, attn_output_weights.view(bsz, num_heads, tgt_len, src_len)class GaussianMultiheadAttention(MultiheadAttention): def init(self, embed_dim, num_heads, **kwargs): super(GaussianMultiheadAttention, self).init(embed_dim, num_heads, **kwargs) self.gaussian = True self.lamb = nn.Parameter(torch.zeros(num_heads), requires_grad=True)
def forward(self, query, key, value, key_padding_mask=None, need_weights=False, attn_mask=None, gaussian=None): # type: (Tensor, Tensor, Tensor, Optional[Tensor], bool, Optional[Tensor], Optional[Tensor]) -> Tuple[Tensor, Optional[Tensor]] r""" Args: query, key, value: map a query and a set of key-value pairs to an output. See "Attention Is All You Need" for more details. key_padding_mask: if provided, specified padding elements in the key will be ignored by the attention. When given a binary mask and a value is True, the corresponding value on the attention layer will be ignored. When given a byte mask and a value is non-zero, the corresponding value on the attention layer will be ignored need_weights: output attn_output_weights. attn_mask: 2D or 3D mask that prevents attention to certain positions. A 2D mask will be broadcasted for all the batches while a 3D mask allows to specify a different mask for the entries of each batch. gaussian: 2D gaussian attention map that focus attention to certain object queries' initial estimations with handcrafted query spatial priors. Shape: - Inputs: - query: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is the embedding dimension. - key: :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is the embedding dimension. - value: :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is the embedding dimension. - key_padding_mask: :math:`(N, S)` where N is the batch size, S is the source sequence length. If a ByteTensor is provided, the non-zero positions will be ignored while the position with the zero positions will be unchanged. If a BoolTensor is provided, the positions with the value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged. - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length. 3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length, S is the source sequence length. attn_mask ensure that position i is allowed to attend the unmasked positions. If a ByteTensor is provided, the non-zero positions are not allowed to attend while the zero positions will be unchanged. If a BoolTensor is provided, positions with ``True`` is not allowed to attend while ``False`` values will be unchanged. If a FloatTensor is provided, it will be added to the attention weight. - gaussian: :math:`(L, S, nhead * batch_size)`, where nhead is the number of head in multi-head attention module, L is the target sequence length, S is the source sequence length. - Outputs: - attn_output: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is the embedding dimension. - attn_output_weights: :math:`(N, L, S)` where N is the batch size, L is the target sequence length, S is the source sequence length. """ if not self._qkv_same_embed_dim: return multi_head_attention_forward( query, key, value, self.embed_dim, self.num_heads, self.in_proj_weight, self.in_proj_bias, self.bias_k, self.bias_v, self.add_zero_attn, self.dropout, self.out_proj.weight, self.out_proj.bias, training=self.training, key_padding_mask=key_padding_mask, need_weights=need_weights, attn_mask=attn_mask, use_separate_proj_weight=True, q_proj_weight=self.q_proj_weight, k_proj_weight=self.k_proj_weight, v_proj_weight=self.v_proj_weight, gaussian=gaussian, lamb=self.lamb) else: return multi_head_attention_forward( query, key, value, self.embed_dim, self.num_heads, self.in_proj_weight, self.in_proj_bias, self.bias_k, self.bias_v, self.add_zero_attn, self.dropout, self.out_proj.weight, self.out_proj.bias, training=self.training, key_padding_mask=key_padding_mask, need_weights=need_weights, attn_mask=attn_mask, gaussian=gaussian, lamb=self.lamb) - Assistant
代码分析:多头注意力机制
这段代码实现了多头注意力机制中带高斯权重的模块。代码主要分为两个部分:
multi_head_attention_forward函数和GaussianMultiheadAttention类。下面将对这两个部分进行详细分析。1.
multi_head_attention_forward函数该函数是实现多头注意力机制的核心部分。其作用是计算查询、键和值之间的注意力权重,并生成输出。以下是对函数重要部分的分析:
-
输入参数:
query,key,value:这是多头注意力操作中输入的三组张量,分别表示查询、键和值。embed_dim_to_check:用于检查模型的嵌入维度。num_heads:并行注意力头的数量。in_proj_weight,in_proj_bias:输入投影的权重和偏置。- 其他参数用于调整注意力机制的具体细节,如
bias_k,bias_v,attn_mask等。
-
核心步骤:
- QKV 投影:根据输入张量
query,key,value和投影权重参数,通过线性变换生成q,k,v。 - 调整形状:对得到的
q,k,v进行形状调整,使其适合后续的多头操作。 - 注意力计算:通过对
q和k的点积计算得到注意力权重,结合softmax激活函数以获得概率分布。 - 应用掩码和加权平均:如果提供了掩码,对非关注区域进行屏蔽,根据注意力权重加权求和获得注意力输出。
- 引入高斯权重:该部分引入了高斯权重
gaussian,并对注意力权重进行了调整,这是此实现的一个特点。 - 生成输出:经过投影后的注意力输出作为最终输出,返回输出结果和注意力权重。
- QKV 投影:根据输入张量
2.
GaussianMultiheadAttention类这是一个继承自
torch.nn.MultiheadAttention的类,主要添加了对高斯权重的支持。-
构造方法:
- 定义了基础属性如
embed_dim和num_heads,并增加了一个可学习参数lamb。
- 定义了基础属性如
-
前向方法:
- 重写了
forward方法,通过调用multi_head_attention_forward函数,使其支持使用高斯权重。 - 当
_qkv_same_embed_dim为False时,使用不同的是投影权重;否则直接调用函数。
- 重写了
总结
以上代码对标准多头注意力机制进行了扩展,增加了对高斯权重的支持。这可以在注意力权重的计算中更强调对某些特定位置的关注,尤其在需要关注具体空间位置的场景下可能提升模型表现。多头注意力机制广泛用于各类深度学习模型,尤其是变换器类模型中,这一实现为其提供了更灵活的适应能力。
-
- User
分析一下:import math import torch import torch.nn as nn from timm.models.layers import Mlp, DropPath, trunc_normal_, lecun_normal_
from lib.models.layers.attn import Attention
def candidate_elimination(attn: torch.Tensor, tokens: torch.Tensor, lens_t: int, keep_ratio: float, global_index: torch.Tensor, box_mask_z: torch.Tensor): """ Eliminate potential background candidates for computation reduction and noise cancellation. Args: attn (torch.Tensor): [B, num_heads, L_t + L_s, L_t + L_s], attention weights tokens (torch.Tensor): [B, L_t + L_s, C], template and search region tokens lens_t (int): length of template keep_ratio (float): keep ratio of search region tokens (candidates) global_index (torch.Tensor): global index of search region tokens box_mask_z (torch.Tensor): template mask used to accumulate attention weights
Returns: tokens_new (torch.Tensor): tokens after candidate elimination keep_index (torch.Tensor): indices of kept search region tokens removed_index (torch.Tensor): indices of removed search region tokens """ lens_s = attn.shape[-1] - lens_t bs, hn, _, _ = attn.shape lens_keep = math.ceil(keep_ratio * lens_s) if lens_keep == lens_s: return tokens, global_index, None attn_t = attn[:, :, :lens_t, lens_t:] if box_mask_z is not None: box_mask_z = box_mask_z.unsqueeze(1).unsqueeze(-1).expand(-1, attn_t.shape[1], -1, attn_t.shape[-1]) box_mask_z = torch.cat([box_mask_z, box_mask_z], dim=-2) # attn_t = attn_t[:, :, box_mask_z, :] attn_t = attn_t[box_mask_z] attn_t = attn_t.view(bs, hn, -1, lens_s) attn_t = attn_t.mean(dim=2).mean(dim=1) # B, H, L-T, L_s --> B, L_s # attn_t = [attn_t[i, :, box_mask_z[i, :], :] for i in range(attn_t.size(0))] # attn_t = [attn_t[i].mean(dim=1).mean(dim=0) for i in range(len(attn_t))] # attn_t = torch.stack(attn_t, dim=0) else: attn_t = attn_t.mean(dim=2).mean(dim=1) # B, H, L-T, L_s --> B, L_s # use sort instead of topk, due to the speed issue # https://github.com/pytorch/pytorch/issues/22812 sorted_attn, indices = torch.sort(attn_t, dim=1, descending=True) topk_attn, topk_idx = sorted_attn[:, :lens_keep], indices[:, :lens_keep] non_topk_attn, non_topk_idx = sorted_attn[:, lens_keep:], indices[:, lens_keep:] keep_index = global_index.gather(dim=1, index=topk_idx) removed_index = global_index.gather(dim=1, index=non_topk_idx) # separate template and search tokens tokens_t = tokens[:, :lens_t] tokens_s = tokens[:, lens_t:] # obtain the attentive and inattentive tokens B, L, C = tokens_s.shape # topk_idx_ = topk_idx.unsqueeze(-1).expand(B, lens_keep, C) attentive_tokens = tokens_s.gather(dim=1, index=topk_idx.unsqueeze(-1).expand(B, -1, C)) # inattentive_tokens = tokens_s.gather(dim=1, index=non_topk_idx.unsqueeze(-1).expand(B, -1, C)) # compute the weighted combination of inattentive tokens # fused_token = non_topk_attn @ inattentive_tokens # concatenate these tokens # tokens_new = torch.cat([tokens_t, attentive_tokens, fused_token], dim=0) tokens_new = torch.cat([tokens_t, attentive_tokens], dim=1) return tokens_new, keep_index, removed_indexclass CEBlock(nn.Module):
def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, keep_ratio_search=1.0,): super().__init__() self.norm1 = norm_layer(dim) self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() self.norm2 = norm_layer(dim) mlp_hidden_dim = int(dim * mlp_ratio) self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) self.keep_ratio_search = keep_ratio_search def forward(self, x, global_index_template, global_index_search, mask=None, ce_template_mask=None, keep_ratio_search=None): x_attn, attn = self.attn(self.norm1(x), mask, True) x = x + self.drop_path(x_attn) lens_t = global_index_template.shape[1] removed_index_search = None if self.keep_ratio_search < 1 and (keep_ratio_search is None or keep_ratio_search < 1): keep_ratio_search = self.keep_ratio_search if keep_ratio_search is None else keep_ratio_search x, global_index_search, removed_index_search = candidate_elimination(attn, x, lens_t, keep_ratio_search, global_index_search, ce_template_mask) x = x + self.drop_path(self.mlp(self.norm2(x))) return x, global_index_template, global_index_search, removed_index_search, attnclass Block(nn.Module):
def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): super().__init__() self.norm1 = norm_layer(dim) self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() self.norm2 = norm_layer(dim) mlp_hidden_dim = int(dim * mlp_ratio) self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) def forward(self, x, mask=None): x = x + self.drop_path(self.attn(self.norm1(x), mask)) x = x + self.drop_path(self.mlp(self.norm2(x))) return x - Assistant
代码解析:候选消除与注意力块
这段代码主要实现了两类神经网络模块:一个是候选消除块(
CEBlock),另一个是常规的块(Block)。此外,candidate_elimination函数用于消除潜在的背景候选,从而减少计算和噪音。以下是对代码的详细分析:1.
candidate_elimination函数该函数的目的在于通过消除不重要的候选特征来简化计算并降低噪音。
-
输入参数:
attn:注意力权重张量,表示模板和搜索区域之间的注意力关系。tokens:表示模板和搜索区域的特征。lens_t:模板的长度。keep_ratio:保留的搜索区域特征的比例。global_index:搜索区域特征的全局索引。box_mask_z:模板的掩码。
-
核心步骤:
- 设置模板和搜索区域的分割:通过
lens_t分割模板和搜索特征。 - 消去潜在候选:
- 计算模板部分对于搜索区域的注意力。
- 基于指定的保持比例
keep_ratio,决定保留多少搜索特征。 - 使用排序对注意力权重进行排名,选择最重要的特征。
- 返回结果:返回保留后的特征、保留索引和被消除的索引。
- 设置模板和搜索区域的分割:通过
2.
CEBlock类-
作用:这是一个将注意力和MLP结合在一起的神经网络模块。特别之处在于整合了候选消除的逻辑,以减少搜索区域上的计算。
-
构造方法:
- 使用
Attention类作为注意力机制。 - 包含正则化、drop path和MLP来增强模型鲁棒性。
- 使用
-
前向传递:
- 计算注意力输出并更新特征。
- 在条件满足时,进行候选消除,从而仅保留最重要的特征。
- 更新后特征。
3.
Block类-
作用:这是一个标准的块,不如
CEBlock那么复杂,没有候选消除逻辑。 -
构造方法:
- 类似于
CEBlock,也使用了Attention、MLP、正则化和drop path。
- 类似于
-
前向传递:
- 依次应用注意力和MLP更新特征,通过drop path增强训练稳定性。
总结
这段代码主要在于通过注意力机制对输入特征进行变换和处理,特别在
CEBlock中引入了候选消除机制,以减少多余的计算和噪音。利用注意力权重,智能化选择较为重要的搜索特征,从而优化计算效率,适用于需要对输入信息进行简化和降噪的视觉任务中。 -
- User
那么在Block和CEBlock中是否也可以扩展?增加了对高斯权重?
- Assistant
在
Block和CEBlock中增加对高斯权重的支持是可行的。高斯权重可以用来调整注意力权重的分布,从而更好地关注特定的空间位置或特征。这种扩展可以通过以下步骤实现:步骤
-
修改Attention模块: 首先需要确保
Attention类或者模块能够接受和处理高斯权重。这可能涉及到在注意力计算中引入高斯权重的影响。 -
添加高斯权重参数: 在
CEBlock和Block的初始化方法中,可以增加与高斯权重相关的参数。例如,可以引入一个可学习的参数或外部传入的高斯地图。 -
更新前向方法:
- 在前向传递中,将高斯权重作为额外参数传递给
self.attn调用。 - 在注意力计算过程中,将高斯权重应用到注意力权重矩阵上,例如通过元素级相乘的方式。
- 在前向传递中,将高斯权重作为额外参数传递给
下面是一个示例,展示如何在
CEBlock中扩展高斯权重:class CEBlock(nn.Module): def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, keep_ratio_search=1.0, use_gaussian=False): super().__init__() self.use_gaussian = use_gaussian self.norm1 = norm_layer(dim) self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() self.norm2 = norm_layer(dim) mlp_hidden_dim = int(dim * mlp_ratio) self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) self.keep_ratio_search = keep_ratio_search def forward(self, x, global_index_template, global_index_search, mask=None, ce_template_mask=None, keep_ratio_search=None, gaussian=None): if self.use_gaussian and gaussian is not None: # Pass the Gaussian map to the attention mechanism x_attn, attn = self.attn(self.norm1(x), mask, True, gaussian) else: x_attn, attn = self.attn(self.norm1(x), mask, True) x = x + self.drop_path(x_attn) lens_t = global_index_template.shape[1] removed_index_search = None if self.keep_ratio_search < 1 and (keep_ratio_search is None or keep_ratio_search < 1): keep_ratio_search = self.keep_ratio_search if keep_ratio_search is None else keep_ratio_search x, global_index_search, removed_index_search = candidate_elimination(attn, x, lens_t, keep_ratio_search, global_index_search, ce_template_mask) x = x + self.drop_path(self.mlp(self.norm2(x))) return x, global_index_template, global_index_search, removed_index_search, attn注意事项
- 在Attention类中确保对高斯权重允许:
Attention类里,需要确保在计算注意力权重时能接受和利用高斯映射。 - 确保高斯权重的输入和处理: 适当地处理高斯权重的形状和广播,以适配注意力计算。
- 模型兼容性: 在扩展功能时,保持对原有功能的兼容性,以便能够在没有高斯权重输入时正常工作。
结论
通过引入和结合高斯权重,可以进一步增强模型对特定区域或特征的关注能力。这一机制对于需要强调空间信息或具有先验知识的应用场景可能颇具价值。
-
- User
此处的attention是:import torch import torch.nn as nn import torch.nn.functional as F from timm.models.layers import trunc_normal_
from lib.models.layers.rpe import generate_2d_concatenated_self_attention_relative_positional_encoding_index
class Attention(nn.Module): def init(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0., rpe=False, z_size=7, x_size=14): super().init() self.num_heads = num_heads head_dim = dim // num_heads self.scale = head_dim ** -0.5
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) self.proj = nn.Linear(dim, dim) self.proj_drop = nn.Dropout(proj_drop) self.rpe =rpe if self.rpe: relative_position_index = \ generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size], [x_size, x_size]) self.register_buffer("relative_position_index", relative_position_index) # define a parameter table of relative position bias self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads, relative_position_index.max() + 1))) trunc_normal_(self.relative_position_bias_table, std=0.02) def forward(self, x, mask=None, return_attention=False): # x: B, N, C # mask: [B, N, ] torch.bool B, N, C = x.shape qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple) attn = (q @ k.transpose(-2, -1)) * self.scale if self.rpe: relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0) attn += relative_position_bias if mask is not None: attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf'),) attn = attn.softmax(dim=-1) attn = self.attn_drop(attn) x = (attn @ v).transpose(1, 2).reshape(B, N, C) x = self.proj(x) x = self.proj_drop(x) if return_attention: return x, attn else: return xclass Attention_talking_head(nn.Module): # taken from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py # with slight modifications to add Talking Heads Attention (https://arxiv.org/pdf/2003.02436v1.pdf) def init(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., rpe=True, z_size=7, x_size=14): super().init()
self.num_heads = num_heads head_dim = dim // num_heads self.scale = qk_scale or head_dim ** -0.5 self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) self.proj = nn.Linear(dim, dim) self.proj_l = nn.Linear(num_heads, num_heads) self.proj_w = nn.Linear(num_heads, num_heads) self.proj_drop = nn.Dropout(proj_drop) self.rpe = rpe if self.rpe: relative_position_index = \ generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size], [x_size, x_size]) self.register_buffer("relative_position_index", relative_position_index) # define a parameter table of relative position bias self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads, relative_position_index.max() + 1))) trunc_normal_(self.relative_position_bias_table, std=0.02) def forward(self, x, mask=None): B, N, C = x.shape qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) q, k, v = qkv[0] * self.scale, qkv[1], qkv[2] attn = (q @ k.transpose(-2, -1)) if self.rpe: relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0) attn += relative_position_bias if mask is not None: attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf'),) attn = self.proj_l(attn.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) attn = attn.softmax(dim=-1) attn = self.proj_w(attn.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) attn = self.attn_drop(attn) x = (attn @ v).transpose(1, 2).reshape(B, N, C) x = self.proj(x) x = self.proj_drop(x) return x,现在按照multi_head_attention_forward中添加高斯的方法,将高斯集成到现在的attention中。 - Assistant
为了将高斯权重集成到现有的
Attention模块中,我们可以参考multi_head_attention_forward函数中的实现,将高斯权重调整添加到注意力权重计算过程中。这可以通过一个新的参数来传递高斯矩阵,并将其应用于注意力权重计算。下面是如何在现有Attention和Attention_talking_head类中添加该特性的一个示例:改进后的
Attention类class Attention(nn.Module): def __init__(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0., rpe=False, z_size=7, x_size=14): super().__init__() self.num_heads = num_heads head_dim = dim // num_heads self.scale = head_dim ** -0.5 self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) self.proj = nn.Linear(dim, dim) self.proj_drop = nn.Dropout(proj_drop) self.rpe =rpe if self.rpe: relative_position_index = \ generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size], [x_size, x_size]) self.register_buffer("relative_position_index", relative_position_index) # define a parameter table of relative position bias self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads, relative_position_index.max() + 1))) trunc_normal_(self.relative_position_bias_table, std=0.02) def forward(self, x, mask=None, return_attention=False, gaussian=None): # x: B, N, C B, N, C = x.shape qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple) attn = (q @ k.transpose(-2, -1)) * self.scale if self.rpe: relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0) attn += relative_position_bias if mask is not None: attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf'),) if gaussian is not None: attn += gaussian.unsqueeze(0).unsqueeze(1) # Assuming gaussian has shape (B, N, N) attn = attn.softmax(dim=-1) attn = self.attn_drop(attn) x = (attn @ v).transpose(1, 2).reshape(B, N, C) x = self.proj(x) x = self.proj_drop(x) if return_attention: return x, attn else: return x改进后的
Attention_talking_head类同样的调整可以应用于
Attention_talking_head类:class Attention_talking_head(nn.Module): def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., rpe=True, z_size=7, x_size=14): super().__init__() self.num_heads = num_heads head_dim = dim // num_heads self.scale = qk_scale or head_dim ** -0.5 self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) self.proj = nn.Linear(dim, dim) self.proj_l = nn.Linear(num_heads, num_heads) self.proj_w = nn.Linear(num_heads, num_heads) self.proj_drop = nn.Dropout(proj_drop) self.rpe = rpe if self.rpe: relative_position_index = \ generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size], [x_size, x_size]) self.register_buffer("relative_position_index", relative_position_index) # define a parameter table of relative position bias self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads, relative_position_index.max() + 1))) trunc_normal_(self.relative_position_bias_table, std=0.02) def forward(self, x, mask=None, gaussian=None): B, N, C = x.shape qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) q, k, v = qkv[0] * self.scale, qkv[1], qkv[2] attn = (q @ k.transpose(-2, -1)) if self.rpe: relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0) attn += relative_position_bias if mask is not None: attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf'),) if gaussian is not None: attn += gaussian.unsqueeze(0).unsqueeze(1) # Assuming gaussian has shape (B, N, N) attn = self.proj_l(attn.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) attn = attn.softmax(dim=-1) attn = self.proj_w(attn.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) attn = self.attn_drop(attn) x = (attn @ v).transpose(1, 2).reshape(B, N, C) x = self.proj(x) x = self.proj_drop(x) return x改动说明
- 增加
gaussian参数: 在forward方法中多了一个gaussian参数,用于传递高斯矩阵。 - 应用高斯调整: 如果提供了高斯矩阵,则将其添加到注意力权重上。这一处理与相对位置偏置类似。
- 假设高斯的形状: 假定高斯矩阵具有与注意力权重匹配的形状,或适当的广播形状。
通过这个扩展,模型可以在权重计算中进一步考虑空间信息或事先设定的注意力偏好。
- 增加
- User
我们这样修改:import torch import torch.nn as nn import torch.nn.functional as F from timm.models.layers import trunc_normal_
from lib.models.layers.rpe import generate_2d_concatenated_self_attention_relative_positional_encoding_index
class Attention(nn.Module): def init(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0., rpe=False, z_size=7, x_size=14): super().init() self.num_heads = num_heads head_dim = dim // num_heads self.scale = head_dim ** -0.5
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) self.proj = nn.Linear(dim, dim) self.proj_drop = nn.Dropout(proj_drop) self.rpe =rpe if self.rpe: relative_position_index = \ generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size], [x_size, x_size]) self.register_buffer("relative_position_index", relative_position_index) # define a parameter table of relative position bias self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads, relative_position_index.max() + 1))) trunc_normal_(self.relative_position_bias_table, std=0.02) # 引入 高斯 self.lamb = nn.Parameter(torch.zeros(num_heads), requires_grad=True) # 添加高斯参数 def forward(self, x, mask=None, return_attention=False): # x: B, N, C # mask: [B, N, ] torch.bool B, N, C = x.shape qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple) attn = (q @ k.transpose(-2, -1)) * self.scale if self.rpe: relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0) attn += relative_position_bias if mask is not None: attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf'),) attn = attn.softmax(dim=-1) attn = self.attn_drop(attn) # 引入高斯权重 gaussian = torch.exp(-((attn.unsqueeze(-1) - 0.5) ** 2) / (2 * 0.1 ** 2)) # 调整高斯函数参数 attn = attn + gaussian * self.lamb.unsqueeze(0).unsqueeze(2).unsqueeze(3) ############ x = (attn @ v).transpose(1, 2).reshape(B, N, C) x = self.proj(x) x = self.proj_drop(x) if return_attention: return x, attn else: return x - Assistant
在这个修改后的
Attention类中,我们引入了高斯权重,并对注意力权重进行了调整。具体的高斯改动如下:修改说明:
-
高斯参数
lamb:- 引入了一个新的可学习参数
lamb,它的形状为(num_heads,)。 - 该参数将作用于每个注意力头的高斯调整中。
- 引入了一个新的可学习参数
-
生成高斯权重:
- 使用了高斯函数来生成一个高斯矩阵,并将其与原有的注意力权重相结合。
- 高斯权重基于一个均值为0.5和标准差上为0.1的正态分布。这是一个经验设定,你可以根据具体需求进行调整。
- 使用
torch.exp计算高斯权重,之后乘以lamb以进行强度调整。
-
更新注意力权重:
- 将得到的高斯矩阵乘以系数
lamb并加到注意力权重上。这样可以在注意力权重的基础上,进一步引入空间上的先验偏置。
- 将得到的高斯矩阵乘以系数
使用方法:
- 输入与输出:
- 输入为
(B, N, C)的特征向量,同样采取了对mask和return_attention两个参数的支持。 - 输出将是调整后的特征向量,原则上维度不变。
- 输入为
代码中的注意事项:
- 广播机制:确保所有不同形状的张量在相加或乘之前,应适当调整形状(例如使用
unsqueeze)。 - 高斯函数参数调整:函数中采用的默认值(如均值和方差)是一个选择的起始点。你可以根据具体数据特点来调节这些值。
- 学习参数:你可以将高斯方差设为可学习参数,以便模型在训练过程中自行调整到最合适的状态。
使用场景:
这样整合了高斯权重的注意力机制可以在需要进一步突出某些特定空间位置的信息时使用,适用于如目标检测、语义分割等任务。在这类任务中,各个特征的空间关系尤为重要。
-
- User
那么在当前修改中,与multi_head_attention_forward中使用高斯的方法有何不同?
- Assistant
在当前修改的
Attention类中,我们引入了一个高斯公式来调整注意力权重,而在multi_head_attention_forward函数中,对高斯的使用可能是直接与注意力权重相结合的。以下是两者之间不同点的详细分析:1. 高斯处理方式
-
在
Attention类中的高斯处理:- 使用高斯函数生成一个高斯分布的权重矩阵:
gaussian = torch.exp(-((attn.unsqueeze(-1) - 0.5) ** 2) / (2 * 0.1 ** 2)) - 使用这个权重来调整注意力矩阵,以此来进一步强化某一特定分布的权重。
- 使用高斯函数生成一个高斯分布的权重矩阵:
-
在
multi_head_attention_forward中的高斯处理:- 该函数中高斯权重可能是直接从外部传入的一个张量,然后与注意力权重直接相加或点乘。
- 这种方法依赖对高斯权重的直接使用,可能更多依赖先验知识。
2. 高斯权重的来源
-
自生成高斯权重:
- 在
Attention类中,高斯权重的生成是基于一个固定公式,且通过模型学习参数lamb进行调整。通过训练得到的参数,可以适应特定数据集。
- 在
-
外部提供的高斯权重:
- 在
multi_head_attention_forward的实现中,高斯权重从外部传入,意味着这些权重可能是基于另一种方法或先验知识计算出的。
- 在
3. 调整注意力权重的方式
- 方法不同:
- 在
Attention类中,生成的高斯权重是通过一个预定公式,目的是创建某种中心化的注意力分布。 - 在
multi_head_attention_forward中,高斯权重的使用灵活,可能直接影响注意力的计算,其方式可能基于特定场景和需求。
- 在
4. 目的和扩展性
-
目的:
- 两种方法的设计目的可能不同。
Attention类中的方法,更多是希望约束或引导注意力权重到一个高斯分布上。 multi_head_attention_forward的方法,更强调灵活应用现有先验知识来处理注意力权重。
- 两种方法的设计目的可能不同。
-
扩展性:
Attention采用固定公式扩展性更强,可通过调整均值方差等参数定制化。multi_head_attention_forward可以接收不同形态的高斯信息,更灵活。
通过这种分析,你可以看到,在实现与设计中选择合适的方式,往往需要考虑到具体任务的需求、模型的灵活性和数据特征。
-
- User
那么现在根据multi_head_attention_forward中高斯的计算和使用,修改Attention中的高斯计算
- Assistant
为了将
Attention类中的高斯计算方式与multi_head_attention_forward中的高斯使用方式对齐,我们需要更改高斯权重的计算,并在处理注意力权重时更紧密地结合高斯调整。以下是可能的步骤:1. 高斯权重的传入
首先,我们需要接收外部的高斯权重信息。这意味着我们可能需要在
forward函数中增加一个参数,专门用于传入高斯权重。2. 高斯权重的处理方式
根据
multi_head_attention_forward中的方法,外部的高斯权重通常用于直接调整注意力权重。在Attention类的实现中,我们可以考虑高斯权重如何适应和调整注意力权重。代码示例
class Attention(nn.Module): def __init__(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0., rpe=False, z_size=7, x_size=14): super().__init__() self.num_heads = num_heads head_dim = dim // num_heads self.scale = head_dim ** -0.5 self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) self.proj = nn.Linear(dim, dim) self.proj_drop = nn.Dropout(proj_drop) self.rpe = rpe if self.rpe: relative_position_index = \ generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size], [x_size, x_size]) self.register_buffer("relative_position_index", relative_position_index) self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads, relative_position_index.max() + 1))) trunc_normal_(self.relative_position_bias_table, std=0.02) # 引入高斯比例参数 self.lamb = nn.Parameter(torch.zeros(num_heads), requires_grad=True) def forward(self, x, mask=None, return_attention=False, gaussian=None): B, N, C = x.shape qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) q, k, v = qkv.unbind(0) attn = (q @ k.transpose(-2, -1)) * self.scale if self.rpe: relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0) attn += relative_position_bias # 根据 mask 设置注意力分数 if mask is not None: attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf')) # 应用高斯权重 if gaussian is not None: attn += gaussian * self.lamb.unsqueeze(0).unsqueeze(2).unsqueeze(3) attn = attn.softmax(dim=-1) attn = self.attn_drop(attn) x = (attn @ v).transpose(1, 2).reshape(B, N, C) x = self.proj(x) x = self.proj_drop(x) if return_attention: return x, attn else: return x修改说明
-
传入高斯权重: 添加一个
gaussian的参数至forward方法,以接收外部计算好的高斯权重。 -
高斯权重应用: 在计算好的注意力矩阵上应用传入的高斯权重,乘以系数
lamb后与注意力矩阵相加。这与multi_head_attention_forward耦合高斯权重的方式一致。 -
灵活性: 外部传入的高斯权重可以根据实际需要灵活调整,
Attention类本身不亲自生成高斯权重,却能对其进行有效整合和应用。
通过这样的调整,高斯权重的适应性得以提高,应用场景更为广泛,适用于在有先验高斯信息的注意力机制中增强对特定空间或特征的关注。
-
- User
将import torch import torch.nn as nn from torch.nn.functional import linear, pad from torch import Tensor from torch.nn import MultiheadAttention
from typing import Optional, Tuple, List import warnings
def multi_head_attention_forward( query: Tensor, key: Tensor, value: Tensor, embed_dim_to_check: int, num_heads: int, in_proj_weight: Tensor, in_proj_bias: Tensor, bias_k: Optional[Tensor], bias_v: Optional[Tensor], add_zero_attn: bool, dropout_p: float, out_proj_weight: Tensor, out_proj_bias: Tensor, training: bool = True, key_padding_mask: Optional[Tensor] = None, need_weights: bool = True, attn_mask: Optional[Tensor] = None, use_separate_proj_weight: bool = False, q_proj_weight: Optional[Tensor] = None, k_proj_weight: Optional[Tensor] = None, v_proj_weight: Optional[Tensor] = None, static_k: Optional[Tensor] = None, static_v: Optional[Tensor] = None, gaussian: Optional[Tensor] = None, lamb = None, ) -> Tuple[Tensor, Optional[Tensor]]: r""" Args: query, key, value: map a query and a set of key-value pairs to an output. See "Attention Is All You Need" for more details. embed_dim_to_check: total dimension of the model. num_heads: parallel attention heads. in_proj_weight, in_proj_bias: input projection weight and bias. bias_k, bias_v: bias of the key and value sequences to be added at dim=0. add_zero_attn: add a new batch of zeros to the key and value sequences at dim=1. dropout_p: probability of an element to be zeroed. out_proj_weight, out_proj_bias: the output projection weight and bias. training: apply dropout if is
True. key_padding_mask: if provided, specified padding elements in the key will be ignored by the attention. This is an binary mask. When the value is True, the corresponding value on the attention layer will be filled with -inf. need_weights: output attn_output_weights. attn_mask: 2D or 3D mask that prevents attention to certain positions. This is an additive mask (i.e. the values will be added to the attention layer). A 2D mask will be broadcasted for all the batches while a 3D mask allows to specify a different mask for the entries of each batch. use_separate_proj_weight: the function accept the proj. weights for query, key, and value in different forms. If false, in_proj_weight will be used, which is a combination of q_proj_weight, k_proj_weight, v_proj_weight. q_proj_weight, k_proj_weight, v_proj_weight, in_proj_bias: input projection weight and bias. static_k, static_v: static key and value used for attention operators. gaussian: the generated Gaussian-like weight mapShape: Inputs: - query: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is the embedding dimension. - key: :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is the embedding dimension. - value: :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is the embedding dimension. - key_padding_mask: :math:`(N, S)` where N is the batch size, S is the source sequence length. If a ByteTensor is provided, the non-zero positions will be ignored while the zero positions will be unchanged. If a BoolTensor is provided, the positions with the value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged. - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length. 3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length, S is the source sequence length. attn_mask ensures that position i is allowed to attend the unmasked positions. If a ByteTensor is provided, the non-zero positions are not allowed to attend while the zero positions will be unchanged. If a BoolTensor is provided, positions with ``True`` are not allowed to attend while ``False`` values will be unchanged. If a FloatTensor is provided, it will be added to the attention weight. - static_k: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length, N is the batch size, E is the embedding dimension. E/num_heads is the head dimension. - static_v: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length, N is the batch size, E is the embedding dimension. E/num_heads is the head dimension. Outputs: - attn_output: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is the embedding dimension. - attn_output_weights: :math:`(N, L, S)` where N is the batch size, L is the target sequence length, S is the source sequence length. """ tgt_len, bsz, embed_dim = query.size() assert embed_dim == embed_dim_to_check # allow MHA to have different sizes for the feature dimension assert key.size(0) == value.size(0) and key.size(1) == value.size(1) head_dim = embed_dim // num_heads assert head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads" scaling = float(head_dim) ** -0.5 if not use_separate_proj_weight: if torch.equal(query, key) and torch.equal(key, value): # self-attention q, k, v = linear(query, in_proj_weight, in_proj_bias).chunk(3, dim=-1) elif torch.equal(key, value): # encoder-decoder attention # This is inline in_proj function with in_proj_weight and in_proj_bias _b = in_proj_bias _start = 0 _end = embed_dim _w = in_proj_weight[_start:_end, :] if _b is not None: _b = _b[_start:_end] q = linear(query, _w, _b) if key is None: assert value is None k = None v = None else: # This is inline in_proj function with in_proj_weight and in_proj_bias _b = in_proj_bias _start = embed_dim _end = None _w = in_proj_weight[_start:, :] if _b is not None: _b = _b[_start:] k, v = linear(key, _w, _b).chunk(2, dim=-1) else: # This is inline in_proj function with in_proj_weight and in_proj_bias _b = in_proj_bias _start = 0 _end = embed_dim _w = in_proj_weight[_start:_end, :] if _b is not None: _b = _b[_start:_end] q = linear(query, _w, _b) # This is inline in_proj function with in_proj_weight and in_proj_bias _b = in_proj_bias _start = embed_dim _end = embed_dim * 2 _w = in_proj_weight[_start:_end, :] if _b is not None: _b = _b[_start:_end] k = linear(key, _w, _b) # This is inline in_proj function with in_proj_weight and in_proj_bias _b = in_proj_bias _start = embed_dim * 2 _end = None _w = in_proj_weight[_start:, :] if _b is not None: _b = _b[_start:] v = linear(value, _w, _b) else: q_proj_weight_non_opt = torch.jit._unwrap_optional(q_proj_weight) len1, len2 = q_proj_weight_non_opt.size() assert len1 == embed_dim and len2 == query.size(-1) k_proj_weight_non_opt = torch.jit._unwrap_optional(k_proj_weight) len1, len2 = k_proj_weight_non_opt.size() assert len1 == embed_dim and len2 == key.size(-1) v_proj_weight_non_opt = torch.jit._unwrap_optional(v_proj_weight) len1, len2 = v_proj_weight_non_opt.size() assert len1 == embed_dim and len2 == value.size(-1) if in_proj_bias is not None: q = linear(query, q_proj_weight_non_opt, in_proj_bias[0:embed_dim]) k = linear(key, k_proj_weight_non_opt, in_proj_bias[embed_dim:(embed_dim * 2)]) v = linear(value, v_proj_weight_non_opt, in_proj_bias[(embed_dim * 2):]) else: q = linear(query, q_proj_weight_non_opt, in_proj_bias) k = linear(key, k_proj_weight_non_opt, in_proj_bias) v = linear(value, v_proj_weight_non_opt, in_proj_bias) q = q * scaling if attn_mask is not None: if attn_mask.dim() == 2: attn_mask = attn_mask.unsqueeze(0) if list(attn_mask.size()) != [1, query.size(0), key.size(0)]: raise RuntimeError('The size of the 2D attn_mask is not correct.') elif attn_mask.dim() == 3: if list(attn_mask.size()) != [bsz * num_heads, query.size(0), key.size(0)]: raise RuntimeError('The size of the 3D attn_mask is not correct.') else: raise RuntimeError("attn_mask's dimension {} is not supported".format(attn_mask.dim())) # attn_mask's dim is 3 now. if bias_k is not None and bias_v is not None: if static_k is None and static_v is None: k = torch.cat([k, bias_k.repeat(1, bsz, 1)]) v = torch.cat([v, bias_v.repeat(1, bsz, 1)]) if attn_mask is not None: attn_mask = pad(attn_mask, (0, 1)) if key_padding_mask is not None: key_padding_mask = pad(key_padding_mask, (0, 1)) else: assert static_k is None, "bias cannot be added to static key." assert static_v is None, "bias cannot be added to static value." else: assert bias_k is None assert bias_v is None q = q.contiguous().view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1) if k is not None: k = k.contiguous().view(-1, bsz * num_heads, head_dim).transpose(0, 1) if v is not None: v = v.contiguous().view(-1, bsz * num_heads, head_dim).transpose(0, 1) if static_k is not None: assert static_k.size(0) == bsz * num_heads assert static_k.size(2) == head_dim k = static_k if static_v is not None: assert static_v.size(0) == bsz * num_heads assert static_v.size(2) == head_dim v = static_v src_len = k.size(1) if key_padding_mask is not None: assert key_padding_mask.size(0) == bsz assert key_padding_mask.size(1) == src_len if add_zero_attn: src_len += 1 k = torch.cat([k, torch.zeros((k.size(0), 1) + k.size()[2:], dtype=k.dtype, device=k.device)], dim=1) v = torch.cat([v, torch.zeros((v.size(0), 1) + v.size()[2:], dtype=v.dtype, device=v.device)], dim=1) if attn_mask is not None: attn_mask = pad(attn_mask, (0, 1)) if key_padding_mask is not None: key_padding_mask = pad(key_padding_mask, (0, 1)) attn_output_weights = torch.bmm(q, k.transpose(1, 2)) assert list(attn_output_weights.size()) == [bsz * num_heads, tgt_len, src_len] if attn_mask is not None: attn_output_weights += attn_mask if key_padding_mask is not None: attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len) attn_output_weights = attn_output_weights.masked_fill( key_padding_mask.unsqueeze(1).unsqueeze(2), float('-inf'),) attn_output_weights = attn_output_weights.view(bsz * num_heads, tgt_len, src_len) attn_output_weights = attn_output_weights + gaussian[0].permute(2, 0, 1) attn_output_weights = torch.nn.functional.softmax(attn_output_weights, dim=-1) # High-Frequency Emphasis HFE attn_d = torch.ones(attn_output_weights.shape[-2:], device=attn_output_weights.device)/tgt_len attn_d = attn_d[None, None, ...] attn_h = attn_output_weights.view(bsz, num_heads, tgt_len, src_len) - attn_d attn_h = attn_h * (1. + lamb[None, :, None, None]) attn_output_weights = (attn_d + attn_h).view(bsz*num_heads, tgt_len, src_len) attn_output_weights = torch.nn.functional.dropout(attn_output_weights, p=dropout_p, training=training) attn_output = torch.bmm(attn_output_weights, v) assert list(attn_output.size()) == [bsz * num_heads, tgt_len, head_dim] attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim) attn_output = linear(attn_output, out_proj_weight, out_proj_bias) return attn_output, attn_output_weights.view(bsz, num_heads, tgt_len, src_len)class GaussianMultiheadAttention(MultiheadAttention): def init(self, embed_dim, num_heads, **kwargs): super(GaussianMultiheadAttention, self).init(embed_dim, num_heads, **kwargs) self.gaussian = True self.lamb = nn.Parameter(torch.zeros(num_heads), requires_grad=True)
def forward(self, query, key, value, key_padding_mask=None, need_weights=False, attn_mask=None, gaussian=None): # type: (Tensor, Tensor, Tensor, Optional[Tensor], bool, Optional[Tensor], Optional[Tensor]) -> Tuple[Tensor, Optional[Tensor]] r""" Args: query, key, value: map a query and a set of key-value pairs to an output. See "Attention Is All You Need" for more details. key_padding_mask: if provided, specified padding elements in the key will be ignored by the attention. When given a binary mask and a value is True, the corresponding value on the attention layer will be ignored. When given a byte mask and a value is non-zero, the corresponding value on the attention layer will be ignored need_weights: output attn_output_weights. attn_mask: 2D or 3D mask that prevents attention to certain positions. A 2D mask will be broadcasted for all the batches while a 3D mask allows to specify a different mask for the entries of each batch. gaussian: 2D gaussian attention map that focus attention to certain object queries' initial estimations with handcrafted query spatial priors. Shape: - Inputs: - query: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is the embedding dimension. - key: :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is the embedding dimension. - value: :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is the embedding dimension. - key_padding_mask: :math:`(N, S)` where N is the batch size, S is the source sequence length. If a ByteTensor is provided, the non-zero positions will be ignored while the position with the zero positions will be unchanged. If a BoolTensor is provided, the positions with the value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged. - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length. 3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length, S is the source sequence length. attn_mask ensure that position i is allowed to attend the unmasked positions. If a ByteTensor is provided, the non-zero positions are not allowed to attend while the zero positions will be unchanged. If a BoolTensor is provided, positions with ``True`` is not allowed to attend while ``False`` values will be unchanged. If a FloatTensor is provided, it will be added to the attention weight. - gaussian: :math:`(L, S, nhead * batch_size)`, where nhead is the number of head in multi-head attention module, L is the target sequence length, S is the source sequence length. - Outputs: - attn_output: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is the embedding dimension. - attn_output_weights: :math:`(N, L, S)` where N is the batch size, L is the target sequence length, S is the source sequence length. """ if not self._qkv_same_embed_dim: return multi_head_attention_forward( query, key, value, self.embed_dim, self.num_heads, self.in_proj_weight, self.in_proj_bias, self.bias_k, self.bias_v, self.add_zero_attn, self.dropout, self.out_proj.weight, self.out_proj.bias, training=self.training, key_padding_mask=key_padding_mask, need_weights=need_weights, attn_mask=attn_mask, use_separate_proj_weight=True, q_proj_weight=self.q_proj_weight, k_proj_weight=self.k_proj_weight, v_proj_weight=self.v_proj_weight, gaussian=gaussian, lamb=self.lamb) else: return multi_head_attention_forward( query, key, value, self.embed_dim, self.num_heads, self.in_proj_weight, self.in_proj_bias, self.bias_k, self.bias_v, self.add_zero_attn, self.dropout, self.out_proj.weight, self.out_proj.bias, training=self.training, key_padding_mask=key_padding_mask, need_weights=need_weights, attn_mask=attn_mask, gaussian=gaussian, lamb=self.lamb)中涉及到高斯的部分单独列出来 - Assistant
在给定的
multi_head_attention_forward函数中,高斯权重部分主要涉及到如何利用一个高斯映射来影响注意力权重的计算。这些高斯权重的整合部分可以单独拉出来分析和归纳。以下是代码中与高斯相关的部分:高斯权重整合步骤
-
高斯权重参数:
- 函数接收一个名为
gaussian的参数,用于传递一个外部计算好的高斯权重矩阵。 - 定义了
lamb,这是一个用于调整高斯影响的参数。
- 函数接收一个名为
-
高斯权重应用到注意力权重:
- 在计算完成原始注意力权重
attn_output_weights后,将高斯权重直接与注意力权重相加。 - 注意,这里假设
gaussian张量的形状经过变换后可以与attn_output_weights对应。
- 在计算完成原始注意力权重
-
高斯调整与高频强调 (High-Frequency Emphasis, HFE):
- 增加了一个高频强调的步骤,即:
- 计算平均注意力分布
attn_d。 - 对原始加权和高斯调整后的注意力做微调,提升某些高频特征。
- 计算平均注意力分布
- 增加了一个高频强调的步骤,即:
高斯相关的主要代码段
# Apply Gaussian-related adjustments attn_output_weights = attn_output_weights + gaussian[0].permute(2, 0, 1) attn_output_weights = torch.nn.functional.softmax(attn_output_weights, dim=-1) # High-Frequency Emphasis (HFE) attn_d = torch.ones(attn_output_weights.shape[-2:], device=attn_output_weights.device) / tgt_len attn_d = attn_d[None, None, ...] attn_h = attn_output_weights.view(bsz, num_heads, tgt_len, src_len) - attn_d attn_h = attn_h * (1. + lamb[None, :, None, None]) attn_output_weights = (attn_d + attn_h).view(bsz * num_heads, tgt_len, src_len)解释
-
高斯整合:
attn_output_weights = attn_output_weights + gaussian[0].permute(2, 0, 1)是整合高斯权重的核心步骤,假设gaussian张量可以通过调整形状与注意力权重相加。
-
HFE 步骤:
- 先取得平均注意力分布
attn_d。 - 通过计算注意力差异
attn_h并结合lamb进行调整。 - 将调整后的结果再平滑整合到最终权重中,达到提升某些特征的目的。
- 先取得平均注意力分布
此设计使用了高斯分布和高频强调来增强对特定特征的区分能力,使得模型可以在某些位置上更敏感、更准确。结合了高斯权重的设计主要目的是为了解决注意力普遍性和局部强关注之间的关系。
-
- Assistant
如果您还有什么问题或者需要进一步的帮助,请随时告诉我!
- User
那么attn_output_weights是如何得到?在import torch import torch.nn as nn import torch.nn.functional as F from timm.models.layers import trunc_normal_
from lib.models.layers.rpe import generate_2d_concatenated_self_attention_relative_positional_encoding_index
class Attention(nn.Module): def init(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0., rpe=False, z_size=7, x_size=14): super().init() self.num_heads = num_heads head_dim = dim // num_heads self.scale = head_dim ** -0.5
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) self.proj = nn.Linear(dim, dim) self.proj_drop = nn.Dropout(proj_drop) self.rpe =rpe if self.rpe: relative_position_index = \ generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size], [x_size, x_size]) self.register_buffer("relative_position_index", relative_position_index) # define a parameter table of relative position bias self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads, relative_position_index.max() + 1))) trunc_normal_(self.relative_position_bias_table, std=0.02) # 引入 高斯比例参数 self.lamb = nn.Parameter(torch.zeros(num_heads), requires_grad=True) # 添加高斯参数 def forward(self, x, mask=None, return_attention=False, gaussian=None): # x: B, N, C # mask: [B, N, ] torch.bool B, N, C = x.shape qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple) attn = (q @ k.transpose(-2, -1)) * self.scale if self.rpe: relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0) attn += relative_position_bias if mask is not None: attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf'),) # 应用高斯权重 if gaussian is not None: attn += gaussian*self.lamb.unsqueeze(0).unsqueeze(2).unsqueeze(3) attn = attn.softmax(dim=-1) attn = self.attn_drop(attn) x = (attn @ v).transpose(1, 2).reshape(B, N, C) x = self.proj(x) x = self.proj_drop(x) if return_attention: return x, attn else: return x中如何计算的高斯?不一样吗? - Assistant
在这段
Attention类代码中,attn_output_weights可以理解为计算注意力权重的矩阵,即变量attn。该矩阵是在前向传递forward函数中计算的:计算
attn_output_weights-
线性变换和重整形:
- 使用线性变换
self.qkv从输入x中提取查询(query)、键(key)、值(value):qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads) q, k, v = qkv.unbind(0) # 将 qkv 拆分成单独的 q, k, v
- 使用线性变换
-
计算注意力权重:
- 计算查询
q和键k的内积,并缩放结果:attn = (q @ k.transpose(-2, -1)) * self.scale - 如果有位置偏差(
rpe),则加入到注意力值中:if self.rpe: relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0) attn += relative_position_bias - 如果有mask时,将指定位置的注意力权重设为负无穷:
if mask is not None: attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf'))
- 计算查询
-
应用高斯权重:
- 加入高斯权重选项,调整注意力值,根据外部提供的
gaussian和可学习参数lamb:if gaussian is not None: attn += gaussian * self.lamb.unsqueeze(0).unsqueeze(2).unsqueeze(3)
- 加入高斯权重选项,调整注意力值,根据外部提供的
-
得到归一化权重:
- 对注意力权重进行softmax归一化,得到
attn_output_weights:attn = attn.softmax(dim=-1) attn = self.attn_drop(attn)
- 对注意力权重进行softmax归一化,得到
比较
multi_head_attention_forward在
multi_head_attention_forward中的高斯使用方式,两者之间的差别主要在计算逻辑及目的上:-
生成方式:
Attention类中的高斯权重来自外部输入变量gaussian,并在内部用lamb进行增益调整,而multi_head_attention_forward直接将高斯矩阵与注意力矩阵进行相加,可能是一个基于先验或特定偏好设计的高斯分布。 -
应用步骤:
Attention类中通过高斯执行真正的微调,而在multi_head_attention_forward函数中,给出了高斯整合算法段落并利用特定手段(如 HFE)进一步加强某些数据的权值,注重更细致的特征调整。
两者在处理流程上,这使得
Attention可以灵活地处理、调节外部提供的高斯信息,而multi_head_attention_forward则侧重于更通用的实现方式,直接利用高斯权重进行调制。这意味着在实践中,使用哪一种方法,将取决于数据特征以及特定任务需求。 -
- User
但是Attention中并未对高斯定义
- Assistant
您说得对,在给定的
Attention类实现中,高斯权重gaussian只是作为一个可能的输入参数被引入到forward方法中,但没有在方法内对其进行具体的定义或生成。这意味着在使用该Attention类时,需要从外部提供高斯权重。这种设计使得Attention层能够通过参数灵活地利用高斯信息,而不需要在类内部生成高斯权重。处理高斯权重的方式
-
外部生成:
- 高斯权重(即参数
gaussian)应在调用Attention类的forward方法时,从外部计算和提供。这可以通过预处理的步骤或在模型其他层中生成。
- 高斯权重(即参数
-
注意力权重的调节:
- 如果提供了
gaussian,模型会将其与注意力矩阵相加,并乘以可学习参数lamb,这可以看作是对注意力分布的一种偏向性调整。
- 如果提供了
拓展与实现
如果希望让
Attention类自身生成某种形式的高斯权重,需要在类内部添加额外的逻辑。例如:# 伪代码示例:Attention类内部生成简单高斯权重 import torch class Attention(nn.Module): def __init__(self, ...): ... self.gaussian_sigma = 0.5 # 自定义的标准差 def forward(self, x, mask=None, return_attention=False): ... # 如果没有提供高斯权重,从内部生成 if gaussian is None: B, N, _ = x.shape gaussian = torch.exp(-((torch.arange(N, dtype=torch.float32) - N/2) ** 2) / (2 * self.gaussian_sigma ** 2)) gaussian = gaussian.unsqueeze(0).unsqueeze(1).to(x.device) # (1, 1, N) # 应用生成或外部传入的高斯权重 attn += gaussian * self.lamb.unsqueeze(0).unsqueeze(2).unsqueeze(3) ...选择性地使用
通过这种方法,您可以在需要时从外部输入精确的高斯矩阵,而在其他情况下,允许
Attention类自适应创建基本高斯分布。这赋予了模型设计高度的灵活性,以适应不同的任务和场景需求。这样就可以利用预定义的或动态生成的高斯信息来微调注意力机制造成的注意力模式。 -