Skip to content
The Chat Archive
gpt-4o-2024-08-06· OpenAI WildChat-4.8M (AI2)

-------------------------------------------------------- Swin Transformer V2 Copyright…

  1. User

    --------------------------------------------------------

    Swin Transformer V2

    Copyright (c) 2022 Microsoft

    Licensed under The MIT License [see LICENSE for details]

    Written by Ze Liu

    --------------------------------------------------------

    import os import torch import torch.nn as nn import torch.nn.functional as F import torch.utils.checkpoint as checkpoint from timm.models.layers import DropPath, to_2tuple, trunc_normal_ import numpy as np from collections import OrderedDict

    class Mlp(nn.Module): """ Multilayer perceptron."""

    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):
        super().__init__()
        out_features = out_features or in_features
        hidden_features = hidden_features or in_features
        self.fc1 = nn.Linear(in_features, hidden_features)
        self.act = act_layer()
        self.fc2 = nn.Linear(hidden_features, out_features)
        self.drop = nn.Dropout(drop)
    
    def forward(self, x):
        x = self.fc1(x)
        x = self.act(x)
        x = self.drop(x)
        x = self.fc2(x)
        x = self.drop(x)
        return x
    

    def window_partition(x, window_size): """ Args: x: (B, H, W, C) window_size (int): window size

    Returns:
        windows: (num_windows*B, window_size, window_size, C)
    """
    B, H, W, C = x.shape
    x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)
    windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)
    return windows
    

    def window_reverse(windows, window_size, H, W): """ Args: windows: (num_windows*B, window_size, window_size, C) window_size (int): Window size H (int): Height of image W (int): Width of image

    Returns:
        x: (B, H, W, C)
    """
    B = int(windows.shape[0] / (H * W / window_size / window_size))
    x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)
    x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)
    return x
    

    class WindowAttention(nn.Module): """ Window based multi-head self attention (W-MSA) module with relative position bias. It supports both of shifted and non-shifted window.

    Args:
        dim (int): Number of input channels.
        window_size (tuple[int]): The height and width of the window.
        num_heads (int): Number of attention heads.
        qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True
        attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0
        proj_drop (float, optional): Dropout ratio of output. Default: 0.0
        pretrained_window_size (tuple[int]): The height and width of the window in pre-training.
    """
    
    def __init__(self, dim, window_size, num_heads, qkv_bias=True, attn_drop=0., proj_drop=0.,
                 pretrained_window_size=[0, 0]):
    
        super().__init__()
        self.dim = dim
        self.window_size = window_size  # Wh, Ww
        self.pretrained_window_size = pretrained_window_size
        self.num_heads = num_heads
    
        self.logit_scale = nn.Parameter(torch.log(10 * torch.ones((num_heads, 1, 1))), requires_grad=True)
    
        # mlp to generate continuous relative position bias
        self.cpb_mlp = nn.Sequential(nn.Linear(2, 512, bias=True),
                                     nn.ReLU(inplace=True),
                                     nn.Linear(512, num_heads, bias=False))
    
        # get relative_coords_table
        relative_coords_h = torch.arange(-(self.window_size[0] - 1), self.window_size[0], dtype=torch.float32)
        relative_coords_w = torch.arange(-(self.window_size[1] - 1), self.window_size[1], dtype=torch.float32)
        relative_coords_table = torch.stack(
            torch.meshgrid([relative_coords_h,
                            relative_coords_w])).permute(1, 2, 0).contiguous().unsqueeze(0)  # 1, 2*Wh-1, 2*Ww-1, 2
        if pretrained_window_size[0] > 0:
            relative_coords_table[:, :, :, 0] /= (pretrained_window_size[0] - 1)
            relative_coords_table[:, :, :, 1] /= (pretrained_window_size[1] - 1)
        else:
            relative_coords_table[:, :, :, 0] /= (self.window_size[0] - 1)
            relative_coords_table[:, :, :, 1] /= (self.window_size[1] - 1)
        relative_coords_table *= 8  # normalize to -8, 8
        relative_coords_table = torch.sign(relative_coords_table) * torch.log2(
            torch.abs(relative_coords_table) + 1.0) / np.log2(8)
    
        self.register_buffer("relative_coords_table", relative_coords_table)
    
        # get pair-wise relative position index for each token inside the window
        coords_h = torch.arange(self.window_size[0])
        coords_w = torch.arange(self.window_size[1])
        coords = torch.stack(torch.meshgrid([coords_h, coords_w]))  # 2, Wh, Ww
        coords_flatten = torch.flatten(coords, 1)  # 2, Wh*Ww
        relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]  # 2, Wh*Ww, Wh*Ww
        relative_coords = relative_coords.permute(1, 2, 0).contiguous()  # Wh*Ww, Wh*Ww, 2
        relative_coords[:, :, 0] += self.window_size[0] - 1  # shift to start from 0
        relative_coords[:, :, 1] += self.window_size[1] - 1
        relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1
        relative_position_index = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww
        self.register_buffer("relative_position_index", relative_position_index)
    
        self.qkv = nn.Linear(dim, dim * 3, bias=False)
        if qkv_bias:
            self.q_bias = nn.Parameter(torch.zeros(dim))
            self.v_bias = nn.Parameter(torch.zeros(dim))
        else:
            self.q_bias = None
            self.v_bias = None
        self.attn_drop = nn.Dropout(attn_drop)
        self.proj = nn.Linear(dim, dim)
        self.proj_drop = nn.Dropout(proj_drop)
        self.softmax = nn.Softmax(dim=-1)
    
    def forward(self, x, mask=None):
        """
        Args:
            x: input features with shape of (num_windows*B, N, C)
            mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None
        """
        B_, N, C = x.shape
        qkv_bias = None
        if self.q_bias is not None:
            qkv_bias = torch.cat((self.q_bias, torch.zeros_like(self.v_bias, requires_grad=False), self.v_bias))
        qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias)
        qkv = qkv.reshape(B_, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)
        q, k, v = qkv[0], qkv[1], qkv[2]  # make torchscript happy (cannot use tensor as tuple)
    
        # cosine attention
        attn = (F.normalize(q, dim=-1) @ F.normalize(k, dim=-1).transpose(-2, -1))
        logit_scale = torch.clamp(self.logit_scale, max=torch.log(torch.tensor(1. / 0.01))).exp()
        attn = attn * logit_scale
    
        relative_position_bias_table = self.cpb_mlp(self.relative_coords_table).view(-1, self.num_heads)
        relative_position_bias = relative_position_bias_table[self.relative_position_index.view(-1)].view(
            self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1)  # Wh*Ww,Wh*Ww,nH
        relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous()  # nH, Wh*Ww, Wh*Ww
        relative_position_bias = 16 * torch.sigmoid(relative_position_bias)
        attn = attn + relative_position_bias.unsqueeze(0)
    
        if mask is not None:
            nW = mask.shape[0]
            attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)
            attn = attn.view(-1, self.num_heads, N, N)
            attn = self.softmax(attn)
        else:
            attn = self.softmax(attn)
    
        attn = self.attn_drop(attn)
    
        x = (attn @ v).transpose(1, 2).reshape(B_, N, C)
        x = self.proj(x)
        x = self.proj_drop(x)
        return x
    
    def extra_repr(self) -> str:
        return f'dim={self.dim}, window_size={self.window_size}, ' \
               f'pretrained_window_size={self.pretrained_window_size}, num_heads={self.num_heads}'
    
    def flops(self, N):
        # calculate flops for 1 window with token length of N
        flops = 0
        # qkv = self.qkv(x)
        flops += N * self.dim * 3 * self.dim
        # attn = (q @ k.transpose(-2, -1))
        flops += self.num_heads * N * (self.dim // self.num_heads) * N
        #  x = (attn @ v)
        flops += self.num_heads * N * N * (self.dim // self.num_heads)
        # x = self.proj(x)
        flops += N * self.dim * self.dim
        return flops
    

    class SwinTransformerBlock(nn.Module): r""" Swin Transformer Block.

    Args:
        dim (int): Number of input channels.
        num_heads (int): Number of attention heads.
        window_size (int): Window size.
        shift_size (int): Shift size for SW-MSA.
        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
        drop (float, optional): Dropout rate. Default: 0.0
        attn_drop (float, optional): Attention dropout rate. Default: 0.0
        drop_path (float, optional): Stochastic depth rate. Default: 0.0
        act_layer (nn.Module, optional): Activation layer. Default: nn.GELU
        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
        pretrained_window_size (int): Window size in pre-training.
    """
    
    def __init__(self, dim, num_heads, window_size=7, shift_size=0,
                 mlp_ratio=4., qkv_bias=True, drop=0., attn_drop=0., drop_path=0.,
                 act_layer=nn.GELU, norm_layer=nn.LayerNorm, pretrained_window_size=0):
        super().__init__()
        self.dim = dim
        self.num_heads = num_heads
        self.window_size = window_size
        self.shift_size = shift_size
        self.mlp_ratio = mlp_ratio
        assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size"
    
        self.norm1 = norm_layer(dim)
        self.attn = WindowAttention(
            dim, window_size=to_2tuple(self.window_size), num_heads=num_heads,
            qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop,
            pretrained_window_size=to_2tuple(pretrained_window_size))
    
        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
        self.norm2 = norm_layer(dim)
        mlp_hidden_dim = int(dim * mlp_ratio)
        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
    
        self.H = None
        self.W = None
    
    
    def forward(self, x, mask_matrix):
        """ Forward function.
        Args:
            x: Input feature, tensor size (B, H*W, C).
            H, W: Spatial resolution of the input feature.
            mask_matrix: Attention mask for cyclic shift.
        """
        B, L, C = x.shape
        H, W = self.H, self.W
        assert L == H * W, "input feature has wrong size"
    
        shortcut = x
        x = x.view(B, H, W, C)
    
        # pad feature maps to multiples of window size
        pad_l = pad_t = 0
        pad_r = (self.window_size - W % self.window_size) % self.window_size
        pad_b = (self.window_size - H % self.window_size) % self.window_size
        x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b))
        _, Hp, Wp, _ = x.shape
    
        # cyclic shift
        if self.shift_size > 0:
            shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))
            attn_mask = mask_matrix
        else:
            shifted_x = x
            attn_mask = None
    
        # partition windows
        x_windows = window_partition(shifted_x, self.window_size)  # nW*B, window_size, window_size, C
        x_windows = x_windows.view(-1, self.window_size * self.window_size, C)  # nW*B, window_size*window_size, C
    
        # W-MSA/SW-MSA
        attn_windows = self.attn(x_windows, mask=attn_mask)  # nW*B, window_size*window_size, C
    
        # merge windows
        attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)
        shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp)  # B H' W' C
    
        # reverse cyclic shift
        if self.shift_size > 0:
            x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))
        else:
            x = shifted_x
    
        if pad_r > 0 or pad_b > 0:
            x = x[:, :H, :W, :].contiguous()
    
        x = x.view(B, H * W, C)
        x = shortcut + self.drop_path(self.norm1(x))
    
        # FFN
        x = x + self.drop_path(self.norm2(self.mlp(x)))
    
        return x
    

    class PatchMerging(nn.Module): r""" Patch Merging Layer.

    Args:
        dim (int): Number of input channels.
        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
    """
    
    def __init__(self, dim, norm_layer=nn.LayerNorm):
        super().__init__()
        self.dim = dim
        self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False)
        self.norm = norm_layer(2 * dim)
    
    def forward(self, x, H, W):
        B, L, C = x.shape
        assert L == H * W, "input feature has wrong size"
        assert H % 2 == 0 and W % 2 == 0, f"x size ({H}*{W}) are not even."
    
        x = x.view(B, H, W, C)
    
        # padding
        pad_input = (H % 2 == 1) or (W % 2 == 1)
        if pad_input:
            x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2))
    
        x0 = x[:, 0::2, 0::2, :]  # B H/2 W/2 C
        x1 = x[:, 1::2, 0::2, :]  # B H/2 W/2 C
        x2 = x[:, 0::2, 1::2, :]  # B H/2 W/2 C
        x3 = x[:, 1::2, 1::2, :]  # B H/2 W/2 C
        x = torch.cat([x0, x1, x2, x3], -1)  # B H/2 W/2 4*C
        x = x.view(B, -1, 4 * C)  # B H/2*W/2 4*C
    
        x = self.reduction(x)
        x = self.norm(x)
    
        return x
    

    class BasicLayer(nn.Module): """ A basic Swin Transformer layer for one stage.

    Args:
        dim (int): Number of input channels.
        depth (int): Number of blocks.
        num_heads (int): Number of attention heads.
        window_size (int): Local window size.
        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
        drop (float, optional): Dropout rate. Default: 0.0
        attn_drop (float, optional): Attention dropout rate. Default: 0.0
        drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0
        norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
        downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
        pretrained_window_size (int): Local window size in pre-training.
    """
    
    def __init__(self, dim, depth, num_heads, window_size,
                 mlp_ratio=4., qkv_bias=True, drop=0., attn_drop=0.,
                 drop_path=0., norm_layer=nn.LayerNorm, downsample=None, use_checkpoint=False,
                 pretrained_window_size=0):
    
        super().__init__()
        self.window_size = window_size
        self.shift_size = window_size // 2
        self.depth = depth
        self.use_checkpoint = use_checkpoint
    
        # build blocks
        self.blocks = nn.ModuleList([
            SwinTransformerBlock(dim=dim,
                                 num_heads=num_heads, window_size=window_size,
                                 shift_size=0 if (i % 2 == 0) else window_size // 2,
                                 mlp_ratio=mlp_ratio,
                                 qkv_bias=qkv_bias,
                                 drop=drop, attn_drop=attn_drop,
                                 drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path,
                                 norm_layer=norm_layer,
                                 pretrained_window_size=pretrained_window_size)
            for i in range(depth)])
    
        # patch merging layer
        if downsample is not None:
            self.downsample = downsample(dim=dim, norm_layer=norm_layer)
        else:
            self.downsample = None
    
    def forward(self, x, H, W):
        """ Forward function.
        Args:
            x: Input feature, tensor size (B, H*W, C).
            H, W: Spatial resolution of the input feature.
        """
        # calculate attention mask for SW-MSA
        Hp = int(np.ceil(H / self.window_size)) * self.window_size
        Wp = int(np.ceil(W / self.window_size)) * self.window_size
        img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device)  # 1 Hp Wp 1
        h_slices = (slice(0, -self.window_size),
                    slice(-self.window_size, -self.shift_size),
                    slice(-self.shift_size, None))
        w_slices = (slice(0, -self.window_size),
                    slice(-self.window_size, -self.shift_size),
                    slice(-self.shift_size, None))
        cnt = 0
        for h in h_slices:
            for w in w_slices:
                img_mask[:, h, w, :] = cnt
                cnt += 1
    
        mask_windows = window_partition(img_mask, self.window_size)  # nW, window_size, window_size, 1
        mask_windows = mask_windows.view(-1, self.window_size * self.window_size)
        attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)
        attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))
    
        for blk in self.blocks:
            blk.H, blk.W = H, W
            if self.use_checkpoint:
                x = checkpoint.checkpoint(blk, x, attn_mask)
    
            else:
                x = blk(x, attn_mask)
        if self.downsample is not None:
            x_down = self.downsample(x, H, W)
            Wh, Ww = (H + 1) // 2, (W + 1) // 2
            return x, H, W, x_down, Wh, Ww
        else:
            return x, H, W, x, H, W
    

    class PatchEmbed(nn.Module): r""" Image to Patch Embedding

    Args:
        patch_size (int): Patch token size. Default: 4.
        in_chans (int): Number of input image channels. Default: 3.
        embed_dim (int): Number of linear projection output channels. Default: 96.
        norm_layer (nn.Module, optional): Normalization layer. Default: None
    """
    
    def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None):
        super().__init__()
        patch_size = to_2tuple(patch_size)
        self.patch_size = patch_size
    
        self.in_chans = in_chans
        self.embed_dim = embed_dim
    
        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
        if norm_layer is not None:
            self.norm = norm_layer(embed_dim)
        else:
            self.norm = None
    
    def forward(self, x):
        """Forward function."""
        # padding
        _, _, H, W = x.size()
        if W % self.patch_size[1] != 0:
            x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1]))
        if H % self.patch_size[0] != 0:
            x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0]))
    
        x = self.proj(x)  # B C Wh Ww
        if self.norm is not None:
            Wh, Ww = x.size(2), x.size(3)
            x = x.flatten(2).transpose(1, 2)
            x = self.norm(x)
            x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww)
        return x
    

    class SwinTransformer(nn.Module): """ Swin Transformer backbone. A PyTorch impl of : Swin Transformer: Hierarchical Vision Transformer using Shifted Windows - https://arxiv.org/pdf/2103.14030

    Args:
        pretrain_img_size (int): Input image size for training the pretrained model,
            used in absolute postion embedding. Default 224.
        patch_size (int | tuple(int)): Patch size. Default: 4.
        in_chans (int): Number of input image channels. Default: 3.
        embed_dim (int): Number of linear projection output channels. Default: 96.
        depths (tuple[int]): Depths of each Swin Transformer stage.
        num_heads (tuple[int]): Number of attention head of each stage.
        window_size (int): Window size. Default: 7.
        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4.
        qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True
        drop_rate (float): Dropout rate.
        attn_drop_rate (float): Attention dropout rate. Default: 0.
        drop_path_rate (float): Stochastic depth rate. Default: 0.2.
        norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm.
        ape (bool): If True, add absolute position embedding to the patch embedding. Default: False.
        patch_norm (bool): If True, add normalization after patch embedding. Default: True.
        out_indices (Sequence[int]): Output from which stages.
        frozen_stages (int): Stages to be frozen (stop grad and set eval mode).
            -1 means not freezing any parameters.
        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
    """
    
    def __init__(self,
                 pretrain_img_size=224,
                 patch_size=4,
                 in_chans=3,
                 embed_dim=96,
                 depths=[2, 2, 6, 2],
                 num_heads=[3, 6, 12, 24],
                 window_size=7,
                 mlp_ratio=4.,
                 qkv_bias=True,
                 drop_rate=0.,
                 attn_drop_rate=0.,
                 drop_path_rate=0.1,
                 norm_layer=nn.LayerNorm,
                 ape=False,
                 patch_norm=True,
                 out_indices=(0, 1, 2, 3),
                 frozen_stages=-1,
                 use_checkpoint=False,
                 pretrained_window_size=[0, 0, 0, 0]):
        super().__init__()
    
        self.pretrain_img_size = pretrain_img_size
        self.num_layers = len(depths)
        self.embed_dim = embed_dim
        self.ape = ape
        self.patch_norm = patch_norm
        self.out_indices = out_indices
        self.frozen_stages = frozen_stages
    
        # split image into non-overlapping patches
        self.patch_embed = PatchEmbed(
            patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim,
            norm_layer=norm_layer if self.patch_norm else None)
    
        # absolute position embedding
        if self.ape:
            pretrain_img_size = to_2tuple(pretrain_img_size)
            patch_size = to_2tuple(patch_size)
            patches_resolution = [pretrain_img_size[0] // patch_size[0], pretrain_img_size[1] // patch_size[1]]
    
            self.absolute_pos_embed = nn.Parameter(
                torch.zeros(1, embed_dim, patches_resolution[0], patches_resolution[1]))
            trunc_normal_(self.absolute_pos_embed, std=.02)
    
        self.pos_drop = nn.Dropout(p=drop_rate)
    
        # stochastic depth
        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]  # stochastic depth decay rule
    
        # build layers
        self.layers = nn.ModuleList()
        for i_layer in range(self.num_layers-1):
            layer = BasicLayer(
                dim=int(embed_dim * 2 ** i_layer),
                depth=depths[i_layer],
                num_heads=num_heads[i_layer],
                window_size=window_size,
                mlp_ratio=mlp_ratio,
                qkv_bias=qkv_bias,
                drop=drop_rate,
                attn_drop=attn_drop_rate,
                drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])],
                norm_layer=norm_layer,
                downsample=PatchMerging if (i_layer < self.num_layers - 2) else None,
                use_checkpoint=use_checkpoint,
                pretrained_window_size=pretrained_window_size[i_layer])
            self.layers.append(layer)
    
        num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)]
        self.num_features = num_features
        self.apply(self._init_weights)
    
        # add a norm layer for each output
        for i_layer in out_indices:
            layer = norm_layer(num_features[i_layer])
            layer_name = f'norm{i_layer}'
            self.add_module(layer_name, layer)
    
        self._freeze_stages()
    
    def _freeze_stages(self):
        if self.frozen_stages >= 0:
            self.patch_embed.eval()
            for param in self.patch_embed.parameters():
                param.requires_grad = False
    
        if self.frozen_stages >= 1 and self.ape:
            self.absolute_pos_embed.requires_grad = False
    
        if self.frozen_stages >= 2:
            self.pos_drop.eval()
            for i in range(0, self.frozen_stages - 1):
                m = self.layers[i]
                m.eval()
                for param in m.parameters():
                    param.requires_grad = False
    
    def _init_weights(self, m):
        if isinstance(m, nn.Linear):
            trunc_normal_(m.weight, std=.02)
            if isinstance(m, nn.Linear) and m.bias is not None:
                nn.init.constant_(m.bias, 0)
        elif isinstance(m, nn.LayerNorm):
            nn.init.constant_(m.bias, 0)
            nn.init.constant_(m.weight, 1.0)
    
    def forward(self, x):
        x = self.patch_embed(x)
        Wh, Ww = x.size(2), x.size(3)
        if self.ape:
            # interpolate the position embedding to the corresponding size
            absolute_pos_embed = F.interpolate(self.absolute_pos_embed, size=(Wh, Ww), mode='bicubic')
            x = (x + absolute_pos_embed).flatten(2).transpose(1, 2)  # B Wh*Ww C
        else:
            x = x.flatten(2).transpose(1, 2)
        x = self.pos_drop(x)
    
        outs = []
        for i in range(3):
            layer = self.layers[i]
            x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww)
    
            if i in self.out_indices:
                norm_layer = getattr(self, f'norm{i}')
                x_out = norm_layer(x_out)
                out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous()
                outs.append(out)
    
        outputs = OrderedDict()
        outputs['layer1'] = outs[0]
        outputs['layer2'] = outs[1]
        outputs['layer3'] = outs[2]
        return outputs
    
    def flops(self):
        flops = 0
        flops += self.patch_embed.flops()
        for i, layer in enumerate(self.layers):
            flops += layer.flops()
        flops += self.num_features * self.patches_resolution[0] * self.patches_resolution[1] // (2 ** self.num_layers)
        flops += self.num_features * self.num_classes
        return flops
    

    def get_model_config(model_name): default_config = { 'pretrain_img_size': 256, 'patch_size': 4, 'in_chans': 3, 'embed_dim': 96, 'depths': [2, 2, 6, 2], 'num_heads': [3, 6, 12, 24], 'window_size': 7, 'mlp_ratio': 4., 'qkv_bias': True, 'drop_rate': 0.0, 'drop_path_rate': 0.1, 'ape': False, 'patch_norm': True, 'use_checkpoint': False, 'out_indices': (0, 1, 2, 3), } if model_name == 'swin_t8': default_config['drop_path_rate'] = 0.2 default_config['embed_dim'] = 96 default_config['depths'] = [2, 2, 6, 2] default_config['num_heads'] = [3, 6, 12, 24] default_config['window_size'] = 8 default_config['out_indices'] = (0, 1, 2)

    elif model_name == 'swin_t16':
        default_config['drop_path_rate'] = 0.2
        default_config['embed_dim'] = 96
        default_config['depths'] = [2, 2, 6, 2]
        default_config['num_heads'] = [3, 6, 12, 24]
        default_config['window_size'] = 16
        default_config['out_indices'] = (0, 1, 2)
    elif model_name == 'swin_s':
        default_config['drop_path_rate'] = 0.3
        default_config['embed_dim'] = 96
        default_config['depths'] = [2, 2, 18, 2]
        default_config['num_heads'] = [3, 6, 12, 24]
        default_config['window_size'] = 8
        default_config['out_indices'] = (0, 1, 2)
    
    else:
        print('error')
        exit()
    
    return default_config
    

    def get_model(config): model = SwinTransformer(pretrain_img_size=config['pretrain_img_size'], patch_size=config['patch_size'], in_chans=config['in_chans'], embed_dim=config['embed_dim'], depths=config['depths'], num_heads=config['num_heads'], window_size=config['window_size'], mlp_ratio=config['mlp_ratio'], qkv_bias=config['qkv_bias'], drop_rate=config['drop_rate'], drop_path_rate=config['drop_path_rate'], ape=config['ape'], patch_norm=config['patch_norm'], out_indices=config['out_indices'], use_checkpoint=config['use_checkpoint']) return model

    def swint_v2(): """swin_t8 window size=8 """ model_config = get_model_config(model_name='swin_t8') pth_path = os.path.dirname(file) + '/swinv2_tiny_patch4_window8_256.pth'

    """swin_t16 window size=16"""
    # model_config = get_model_config(model_name='swin_t16')
    # pth_path = os.path.dirname(__file__) + '/swinv2_tiny_patch4_window16_256.pth'
    """swin_s"""
    # model_config = get_model_config(model_name='swin_s')
    # pth_path = os.path.dirname(__file__) + '/swinv2_small_patch4_window8_256.pth'
    
    model = get_model(model_config)
    pth_dict = torch.load(pth_path)
    pth_dict = pth_dict['model']
    model_dict = model.state_dict()
    jieduan_pth_dict = {}
    for k, v in pth_dict.items():
        if 'attn_mask' in str(k):
            continue
        if str(k).startswith('layers.2.downsample'):
            break
        jieduan_pth_dict[k] = v
    model_dict.update(jieduan_pth_dict)
    model.load_state_dict(model_dict)
    del jieduan_pth_dict
    del model_dict
    torch.cuda.empty_cache()
    return model
    

    if name == 'main': os.environ['CUDA_VISIBLE_DEVICES'] = '7' model = swint_v2().cuda() print(model) img = torch.ones(2, 3, 256, 256).cuda() img2 = torch.ones(2, 3, 128, 128).cuda() res = model(img) res2 = model(img2) print(res['layer1'].shape) print(res['layer2'].shape) print(res['layer3'].shape) print(res2['layer1'].shape) print(res2['layer2'].shape) print(res2['layer3'].shape)

    n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad)
    print('n_parameters:', n_parameters) 中文分析
    
  2. Assistant

    该代码实现了Swin Transformer V2,这是一个视觉变换模型,主要用于图像处理任务。以下是对代码中关键部分的中文分析:

    1. Mlp 类

    • 作用:多层感知器(MLP)模块,处理馈送前向网络。
    • 构成:包括两个全连接层(nn.Linear),一个激活函数(如GELU),和一个Dropout。

    2. window_partition 和 window_reverse 函数

    • window_partition:将图像的特征图划分为多个窗口,用于局部注意力计算。
    • window_reverse:将窗口级别的特征图恢复成原来的空间布局。

    3. WindowAttention 类

    • 作用:通过窗口方式应用自注意力机制。支持相对位置偏差。
    • 构成:包括线性层用于投影查询、键、值(qkv),用于注意力计算的softmax层及projections。

    4. SwinTransformerBlock 类

    • 作用:实现Swin Transformer的基础块,结合窗口(局部)自注意力机制和多层感知器。
    • 构成:包括窗口注意力层、MLP层、正则化层(如LayerNorm)和drop路径(DropPath)。

    5. PatchMerging 类

    • 作用:用于逐层缩小特征图的分辨率。
    • 构成:通过减少特征图的空间维度,并投影到更高的特征维度。

    6. BasicLayer 类

    • 作用:在一个阶段中应用多个Swin Transformer Block。
    • 构成:包括多个Swin Transformer Block和一个到下一个阶段的下采样层。

    7. PatchEmbed 类

    • 作用:将输入图像划分为不重叠的补丁,并进行线性嵌入。
    • 构成:使用卷积层实现图像到patch的转换。

    8. SwinTransformer 类

    • 主架构:实现Swin Transformer的完整模型。
    • 特性:
      • 支持绝对位置嵌入。
      • 分阶段的特征处理,每个阶段分别由BasicLayer构成。
      • 输出指定阶段的特征。

    9. get_model_config 和 get_model 函数

    • get_model_config:根据模型名称返回相应的默认配置。
    • get_model:根据配置实例化Swin Transformer模型。

    10. swint_v2 函数和脚本末尾部分

    • swint_v2:加载预训练模型参数,并构建和返回带有配置的SwinTransformer模型。
    • 脚本测试:创建模型实例并输入示例数据进行测试,同时计算模型的参数数量。

    总结

    该代码实现了Swin Transformer V2的PyTorch版本,通过定义多种模块实现了图像的分阶段嵌入、特征提取、窗口级注意力机制,以及特征的下采样与归一化,是用作图像分类或其他视觉任务的强有力模型。

    语言处理

    本文档代码使用了Python和PyTorch,并且非常重视模块化设计,同时在使用GPU加速运算。

    即便不熟悉视觉变换模型的细节,只要具备一定的PyTorch框架使用经验,也可以在此基础上进一步修改或实现更多定制功能。

  3. User

    再分析一下:import torch import torch.nn as nn from torch.nn.functional import linear, pad from torch import Tensor from torch.nn import MultiheadAttention

    from typing import Optional, Tuple, List import warnings

    def multi_head_attention_forward( query: Tensor, key: Tensor, value: Tensor, embed_dim_to_check: int, num_heads: int, in_proj_weight: Tensor, in_proj_bias: Tensor, bias_k: Optional[Tensor], bias_v: Optional[Tensor], add_zero_attn: bool, dropout_p: float, out_proj_weight: Tensor, out_proj_bias: Tensor, training: bool = True, key_padding_mask: Optional[Tensor] = None, need_weights: bool = True, attn_mask: Optional[Tensor] = None, use_separate_proj_weight: bool = False, q_proj_weight: Optional[Tensor] = None, k_proj_weight: Optional[Tensor] = None, v_proj_weight: Optional[Tensor] = None, static_k: Optional[Tensor] = None, static_v: Optional[Tensor] = None, gaussian: Optional[Tensor] = None, lamb = None, ) -> Tuple[Tensor, Optional[Tensor]]: r""" Args: query, key, value: map a query and a set of key-value pairs to an output. See "Attention Is All You Need" for more details. embed_dim_to_check: total dimension of the model. num_heads: parallel attention heads. in_proj_weight, in_proj_bias: input projection weight and bias. bias_k, bias_v: bias of the key and value sequences to be added at dim=0. add_zero_attn: add a new batch of zeros to the key and value sequences at dim=1. dropout_p: probability of an element to be zeroed. out_proj_weight, out_proj_bias: the output projection weight and bias. training: apply dropout if is True. key_padding_mask: if provided, specified padding elements in the key will be ignored by the attention. This is an binary mask. When the value is True, the corresponding value on the attention layer will be filled with -inf. need_weights: output attn_output_weights. attn_mask: 2D or 3D mask that prevents attention to certain positions. This is an additive mask (i.e. the values will be added to the attention layer). A 2D mask will be broadcasted for all the batches while a 3D mask allows to specify a different mask for the entries of each batch. use_separate_proj_weight: the function accept the proj. weights for query, key, and value in different forms. If false, in_proj_weight will be used, which is a combination of q_proj_weight, k_proj_weight, v_proj_weight. q_proj_weight, k_proj_weight, v_proj_weight, in_proj_bias: input projection weight and bias. static_k, static_v: static key and value used for attention operators. gaussian: the generated Gaussian-like weight map

    Shape:
        Inputs:
        - query: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is
          the embedding dimension.
        - key: :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is
          the embedding dimension.
        - value: :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is
          the embedding dimension.
        - key_padding_mask: :math:`(N, S)` where N is the batch size, S is the source sequence length.
          If a ByteTensor is provided, the non-zero positions will be ignored while the zero positions
          will be unchanged. If a BoolTensor is provided, the positions with the
          value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged.
        - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length.
          3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length,
          S is the source sequence length. attn_mask ensures that position i is allowed to attend the unmasked
          positions. If a ByteTensor is provided, the non-zero positions are not allowed to attend
          while the zero positions will be unchanged. If a BoolTensor is provided, positions with ``True``
          are not allowed to attend while ``False`` values will be unchanged. If a FloatTensor
          is provided, it will be added to the attention weight.
        - static_k: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length,
          N is the batch size, E is the embedding dimension. E/num_heads is the head dimension.
        - static_v: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length,
          N is the batch size, E is the embedding dimension. E/num_heads is the head dimension.
    
        Outputs:
        - attn_output: :math:`(L, N, E)` where L is the target sequence length, N is the batch size,
          E is the embedding dimension.
        - attn_output_weights: :math:`(N, L, S)` where N is the batch size,
          L is the target sequence length, S is the source sequence length.
    """
    tgt_len, bsz, embed_dim = query.size()
    assert embed_dim == embed_dim_to_check
    # allow MHA to have different sizes for the feature dimension
    assert key.size(0) == value.size(0) and key.size(1) == value.size(1)
    
    head_dim = embed_dim // num_heads
    assert head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads"
    scaling = float(head_dim) ** -0.5
    
    if not use_separate_proj_weight:
        if torch.equal(query, key) and torch.equal(key, value):
            # self-attention
            q, k, v = linear(query, in_proj_weight, in_proj_bias).chunk(3, dim=-1)
    
        elif torch.equal(key, value):
            # encoder-decoder attention
            # This is inline in_proj function with in_proj_weight and in_proj_bias
            _b = in_proj_bias
            _start = 0
            _end = embed_dim
            _w = in_proj_weight[_start:_end, :]
            if _b is not None:
                _b = _b[_start:_end]
            q = linear(query, _w, _b)
    
            if key is None:
                assert value is None
                k = None
                v = None
            else:
    
                # This is inline in_proj function with in_proj_weight and in_proj_bias
                _b = in_proj_bias
                _start = embed_dim
                _end = None
                _w = in_proj_weight[_start:, :]
                if _b is not None:
                    _b = _b[_start:]
                k, v = linear(key, _w, _b).chunk(2, dim=-1)
    
        else:
            # This is inline in_proj function with in_proj_weight and in_proj_bias
            _b = in_proj_bias
            _start = 0
            _end = embed_dim
            _w = in_proj_weight[_start:_end, :]
            if _b is not None:
                _b = _b[_start:_end]
            q = linear(query, _w, _b)
    
            # This is inline in_proj function with in_proj_weight and in_proj_bias
            _b = in_proj_bias
            _start = embed_dim
            _end = embed_dim * 2
            _w = in_proj_weight[_start:_end, :]
            if _b is not None:
                _b = _b[_start:_end]
            k = linear(key, _w, _b)
    
            # This is inline in_proj function with in_proj_weight and in_proj_bias
            _b = in_proj_bias
            _start = embed_dim * 2
            _end = None
            _w = in_proj_weight[_start:, :]
            if _b is not None:
                _b = _b[_start:]
            v = linear(value, _w, _b)
    else:
        q_proj_weight_non_opt = torch.jit._unwrap_optional(q_proj_weight)
        len1, len2 = q_proj_weight_non_opt.size()
        assert len1 == embed_dim and len2 == query.size(-1)
    
        k_proj_weight_non_opt = torch.jit._unwrap_optional(k_proj_weight)
        len1, len2 = k_proj_weight_non_opt.size()
        assert len1 == embed_dim and len2 == key.size(-1)
    
        v_proj_weight_non_opt = torch.jit._unwrap_optional(v_proj_weight)
        len1, len2 = v_proj_weight_non_opt.size()
        assert len1 == embed_dim and len2 == value.size(-1)
    
        if in_proj_bias is not None:
            q = linear(query, q_proj_weight_non_opt, in_proj_bias[0:embed_dim])
            k = linear(key, k_proj_weight_non_opt, in_proj_bias[embed_dim:(embed_dim * 2)])
            v = linear(value, v_proj_weight_non_opt, in_proj_bias[(embed_dim * 2):])
        else:
            q = linear(query, q_proj_weight_non_opt, in_proj_bias)
            k = linear(key, k_proj_weight_non_opt, in_proj_bias)
            v = linear(value, v_proj_weight_non_opt, in_proj_bias)
    q = q * scaling
    
    if attn_mask is not None:
        if attn_mask.dim() == 2:
            attn_mask = attn_mask.unsqueeze(0)
            if list(attn_mask.size()) != [1, query.size(0), key.size(0)]:
                raise RuntimeError('The size of the 2D attn_mask is not correct.')
        elif attn_mask.dim() == 3:
            if list(attn_mask.size()) != [bsz * num_heads, query.size(0), key.size(0)]:
                raise RuntimeError('The size of the 3D attn_mask is not correct.')
        else:
            raise RuntimeError("attn_mask's dimension {} is not supported".format(attn_mask.dim()))
        # attn_mask's dim is 3 now.
    
    if bias_k is not None and bias_v is not None:
        if static_k is None and static_v is None:
            k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
            v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
            if attn_mask is not None:
                attn_mask = pad(attn_mask, (0, 1))
            if key_padding_mask is not None:
                key_padding_mask = pad(key_padding_mask, (0, 1))
        else:
            assert static_k is None, "bias cannot be added to static key."
            assert static_v is None, "bias cannot be added to static value."
    else:
        assert bias_k is None
        assert bias_v is None
    
    q = q.contiguous().view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1)
    if k is not None:
        k = k.contiguous().view(-1, bsz * num_heads, head_dim).transpose(0, 1)
    if v is not None:
        v = v.contiguous().view(-1, bsz * num_heads, head_dim).transpose(0, 1)
    
    if static_k is not None:
        assert static_k.size(0) == bsz * num_heads
        assert static_k.size(2) == head_dim
        k = static_k
    
    if static_v is not None:
        assert static_v.size(0) == bsz * num_heads
        assert static_v.size(2) == head_dim
        v = static_v
    
    src_len = k.size(1)
    
    if key_padding_mask is not None:
        assert key_padding_mask.size(0) == bsz
        assert key_padding_mask.size(1) == src_len
    
    if add_zero_attn:
        src_len += 1
        k = torch.cat([k, torch.zeros((k.size(0), 1) + k.size()[2:], dtype=k.dtype, device=k.device)], dim=1)
        v = torch.cat([v, torch.zeros((v.size(0), 1) + v.size()[2:], dtype=v.dtype, device=v.device)], dim=1)
        if attn_mask is not None:
            attn_mask = pad(attn_mask, (0, 1))
        if key_padding_mask is not None:
            key_padding_mask = pad(key_padding_mask, (0, 1))
    
    attn_output_weights = torch.bmm(q, k.transpose(1, 2))
    assert list(attn_output_weights.size()) == [bsz * num_heads, tgt_len, src_len]
    
    if attn_mask is not None:
        attn_output_weights += attn_mask
    
    if key_padding_mask is not None:
        attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
        attn_output_weights = attn_output_weights.masked_fill(
            key_padding_mask.unsqueeze(1).unsqueeze(2), float('-inf'),)
        attn_output_weights = attn_output_weights.view(bsz * num_heads, tgt_len, src_len)
    
    attn_output_weights = attn_output_weights + gaussian[0].permute(2, 0, 1)
    attn_output_weights = torch.nn.functional.softmax(attn_output_weights, dim=-1)
    # High-Frequency Emphasis  HFE
    attn_d = torch.ones(attn_output_weights.shape[-2:], device=attn_output_weights.device)/tgt_len
    attn_d = attn_d[None, None, ...]
    attn_h = attn_output_weights.view(bsz, num_heads, tgt_len, src_len) - attn_d
    attn_h = attn_h * (1. + lamb[None, :, None, None])
    attn_output_weights = (attn_d + attn_h).view(bsz*num_heads, tgt_len, src_len)
    
    attn_output_weights = torch.nn.functional.dropout(attn_output_weights, p=dropout_p,
                                                      training=training)
    
    attn_output = torch.bmm(attn_output_weights, v)
    assert list(attn_output.size()) == [bsz * num_heads, tgt_len, head_dim]
    attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim)
    attn_output = linear(attn_output, out_proj_weight, out_proj_bias)
    
    return attn_output, attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
    

    class GaussianMultiheadAttention(MultiheadAttention): def init(self, embed_dim, num_heads, **kwargs): super(GaussianMultiheadAttention, self).init(embed_dim, num_heads, **kwargs) self.gaussian = True self.lamb = nn.Parameter(torch.zeros(num_heads), requires_grad=True)

    def forward(self, query, key, value, key_padding_mask=None,
                need_weights=False, attn_mask=None, gaussian=None):
        # type: (Tensor, Tensor, Tensor, Optional[Tensor], bool, Optional[Tensor], Optional[Tensor]) -> Tuple[Tensor, Optional[Tensor]]
        r"""
    Args:
        query, key, value: map a query and a set of key-value pairs to an output.
            See "Attention Is All You Need" for more details.
        key_padding_mask: if provided, specified padding elements in the key will
            be ignored by the attention. When given a binary mask and a value is True,
            the corresponding value on the attention layer will be ignored. When given
            a byte mask and a value is non-zero, the corresponding value on the attention
            layer will be ignored
        need_weights: output attn_output_weights.
        attn_mask: 2D or 3D mask that prevents attention to certain positions. A 2D mask will be broadcasted for all
            the batches while a 3D mask allows to specify a different mask for the entries of each batch.
        gaussian: 2D gaussian attention map that focus attention to certain object queries' initial estimations
            with handcrafted query spatial priors.
    
    Shape:
        - Inputs:
        - query: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is
          the embedding dimension.
        - key: :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is
          the embedding dimension.
        - value: :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is
          the embedding dimension.
        - key_padding_mask: :math:`(N, S)` where N is the batch size, S is the source sequence length.
          If a ByteTensor is provided, the non-zero positions will be ignored while the position
          with the zero positions will be unchanged. If a BoolTensor is provided, the positions with the
          value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged.
        - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length.
          3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length,
          S is the source sequence length. attn_mask ensure that position i is allowed to attend the unmasked
          positions. If a ByteTensor is provided, the non-zero positions are not allowed to attend
          while the zero positions will be unchanged. If a BoolTensor is provided, positions with ``True``
          is not allowed to attend while ``False`` values will be unchanged. If a FloatTensor
          is provided, it will be added to the attention weight.
        - gaussian: :math:`(L, S, nhead * batch_size)`, where nhead is the number of head in multi-head
          attention module, L is the target sequence length, S is the source sequence length.
    
        - Outputs:
        - attn_output: :math:`(L, N, E)` where L is the target sequence length, N is the batch size,
          E is the embedding dimension.
        - attn_output_weights: :math:`(N, L, S)` where N is the batch size,
          L is the target sequence length, S is the source sequence length.
        """
        if not self._qkv_same_embed_dim:
            return multi_head_attention_forward(
                query, key, value, self.embed_dim, self.num_heads,
                self.in_proj_weight, self.in_proj_bias,
                self.bias_k, self.bias_v, self.add_zero_attn,
                self.dropout, self.out_proj.weight, self.out_proj.bias,
                training=self.training,
                key_padding_mask=key_padding_mask, need_weights=need_weights,
                attn_mask=attn_mask, use_separate_proj_weight=True,
                q_proj_weight=self.q_proj_weight, k_proj_weight=self.k_proj_weight,
                v_proj_weight=self.v_proj_weight, gaussian=gaussian,
                lamb=self.lamb)
        else:
            return multi_head_attention_forward(
                query, key, value, self.embed_dim, self.num_heads,
                self.in_proj_weight, self.in_proj_bias,
                self.bias_k, self.bias_v, self.add_zero_attn,
                self.dropout, self.out_proj.weight, self.out_proj.bias,
                training=self.training,
                key_padding_mask=key_padding_mask, need_weights=need_weights,
                attn_mask=attn_mask, gaussian=gaussian,
                lamb=self.lamb)
    
  4. Assistant

    代码分析:多头注意力机制

    这段代码实现了多头注意力机制中带高斯权重的模块。代码主要分为两个部分:multi_head_attention_forward函数和GaussianMultiheadAttention类。下面将对这两个部分进行详细分析。

    1. multi_head_attention_forward 函数

    该函数是实现多头注意力机制的核心部分。其作用是计算查询、键和值之间的注意力权重,并生成输出。以下是对函数重要部分的分析:

    • 输入参数:

      • query, key, value:这是多头注意力操作中输入的三组张量,分别表示查询、键和值。
      • embed_dim_to_check:用于检查模型的嵌入维度。
      • num_heads:并行注意力头的数量。
      • in_proj_weight, in_proj_bias:输入投影的权重和偏置。
      • 其他参数用于调整注意力机制的具体细节,如bias_k, bias_v, attn_mask等。
    • 核心步骤:

      1. QKV 投影:根据输入张量query, key, value和投影权重参数,通过线性变换生成q, k, v。
      2. 调整形状:对得到的q, k, v进行形状调整,使其适合后续的多头操作。
      3. 注意力计算:通过对q和k的点积计算得到注意力权重,结合softmax激活函数以获得概率分布。
      4. 应用掩码和加权平均:如果提供了掩码,对非关注区域进行屏蔽,根据注意力权重加权求和获得注意力输出。
      5. 引入高斯权重:该部分引入了高斯权重gaussian,并对注意力权重进行了调整,这是此实现的一个特点。
      6. 生成输出:经过投影后的注意力输出作为最终输出,返回输出结果和注意力权重。

    2. GaussianMultiheadAttention 类

    这是一个继承自torch.nn.MultiheadAttention的类,主要添加了对高斯权重的支持。

    • 构造方法:

      • 定义了基础属性如embed_dim和num_heads,并增加了一个可学习参数lamb。
    • 前向方法:

      • 重写了forward方法,通过调用multi_head_attention_forward函数,使其支持使用高斯权重。
      • 当_qkv_same_embed_dim为False时,使用不同的是投影权重;否则直接调用函数。

    总结

    以上代码对标准多头注意力机制进行了扩展,增加了对高斯权重的支持。这可以在注意力权重的计算中更强调对某些特定位置的关注,尤其在需要关注具体空间位置的场景下可能提升模型表现。多头注意力机制广泛用于各类深度学习模型,尤其是变换器类模型中,这一实现为其提供了更灵活的适应能力。

  5. User

    分析一下:import math import torch import torch.nn as nn from timm.models.layers import Mlp, DropPath, trunc_normal_, lecun_normal_

    from lib.models.layers.attn import Attention

    def candidate_elimination(attn: torch.Tensor, tokens: torch.Tensor, lens_t: int, keep_ratio: float, global_index: torch.Tensor, box_mask_z: torch.Tensor): """ Eliminate potential background candidates for computation reduction and noise cancellation. Args: attn (torch.Tensor): [B, num_heads, L_t + L_s, L_t + L_s], attention weights tokens (torch.Tensor): [B, L_t + L_s, C], template and search region tokens lens_t (int): length of template keep_ratio (float): keep ratio of search region tokens (candidates) global_index (torch.Tensor): global index of search region tokens box_mask_z (torch.Tensor): template mask used to accumulate attention weights

    Returns:
        tokens_new (torch.Tensor): tokens after candidate elimination
        keep_index (torch.Tensor): indices of kept search region tokens
        removed_index (torch.Tensor): indices of removed search region tokens
    """
    lens_s = attn.shape[-1] - lens_t
    bs, hn, _, _ = attn.shape
    
    lens_keep = math.ceil(keep_ratio * lens_s)
    if lens_keep == lens_s:
        return tokens, global_index, None
    
    attn_t = attn[:, :, :lens_t, lens_t:]
    
    if box_mask_z is not None:
        box_mask_z = box_mask_z.unsqueeze(1).unsqueeze(-1).expand(-1, attn_t.shape[1], -1, attn_t.shape[-1])
        box_mask_z = torch.cat([box_mask_z, box_mask_z], dim=-2)
        # attn_t = attn_t[:, :, box_mask_z, :]
        attn_t = attn_t[box_mask_z]
        attn_t = attn_t.view(bs, hn, -1, lens_s)
        attn_t = attn_t.mean(dim=2).mean(dim=1)  # B, H, L-T, L_s --> B, L_s
    
        # attn_t = [attn_t[i, :, box_mask_z[i, :], :] for i in range(attn_t.size(0))]
        # attn_t = [attn_t[i].mean(dim=1).mean(dim=0) for i in range(len(attn_t))]
        # attn_t = torch.stack(attn_t, dim=0)
    else:
        attn_t = attn_t.mean(dim=2).mean(dim=1)  # B, H, L-T, L_s --> B, L_s
    
    # use sort instead of topk, due to the speed issue
    # https://github.com/pytorch/pytorch/issues/22812
    sorted_attn, indices = torch.sort(attn_t, dim=1, descending=True)
    
    topk_attn, topk_idx = sorted_attn[:, :lens_keep], indices[:, :lens_keep]
    non_topk_attn, non_topk_idx = sorted_attn[:, lens_keep:], indices[:, lens_keep:]
    
    keep_index = global_index.gather(dim=1, index=topk_idx)
    removed_index = global_index.gather(dim=1, index=non_topk_idx)
    
    # separate template and search tokens
    tokens_t = tokens[:, :lens_t]
    tokens_s = tokens[:, lens_t:]
    
    # obtain the attentive and inattentive tokens
    B, L, C = tokens_s.shape
    # topk_idx_ = topk_idx.unsqueeze(-1).expand(B, lens_keep, C)
    attentive_tokens = tokens_s.gather(dim=1, index=topk_idx.unsqueeze(-1).expand(B, -1, C))
    # inattentive_tokens = tokens_s.gather(dim=1, index=non_topk_idx.unsqueeze(-1).expand(B, -1, C))
    
    # compute the weighted combination of inattentive tokens
    # fused_token = non_topk_attn @ inattentive_tokens
    
    # concatenate these tokens
    # tokens_new = torch.cat([tokens_t, attentive_tokens, fused_token], dim=0)
    tokens_new = torch.cat([tokens_t, attentive_tokens], dim=1)
    
    return tokens_new, keep_index, removed_index
    

    class CEBlock(nn.Module):

    def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0.,
                 drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, keep_ratio_search=1.0,):
        super().__init__()
        self.norm1 = norm_layer(dim)
        self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop)
        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
        self.norm2 = norm_layer(dim)
        mlp_hidden_dim = int(dim * mlp_ratio)
        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
    
        self.keep_ratio_search = keep_ratio_search
    
    def forward(self, x, global_index_template, global_index_search, mask=None, ce_template_mask=None, keep_ratio_search=None):
        x_attn, attn = self.attn(self.norm1(x), mask, True)
        x = x + self.drop_path(x_attn)
        lens_t = global_index_template.shape[1]
    
        removed_index_search = None
        if self.keep_ratio_search < 1 and (keep_ratio_search is None or keep_ratio_search < 1):
            keep_ratio_search = self.keep_ratio_search if keep_ratio_search is None else keep_ratio_search
            x, global_index_search, removed_index_search = candidate_elimination(attn, x, lens_t, keep_ratio_search,
                                                                                 global_index_search, ce_template_mask)
    
        x = x + self.drop_path(self.mlp(self.norm2(x)))
        return x, global_index_template, global_index_search, removed_index_search, attn
    

    class Block(nn.Module):

    def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0.,
                 drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm):
        super().__init__()
        self.norm1 = norm_layer(dim)
        self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop)
        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
        self.norm2 = norm_layer(dim)
        mlp_hidden_dim = int(dim * mlp_ratio)
        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
    
    def forward(self, x, mask=None):
        x = x + self.drop_path(self.attn(self.norm1(x), mask))
        x = x + self.drop_path(self.mlp(self.norm2(x)))
        return x
    
  6. Assistant

    代码解析:候选消除与注意力块

    这段代码主要实现了两类神经网络模块:一个是候选消除块(CEBlock),另一个是常规的块(Block)。此外,candidate_elimination函数用于消除潜在的背景候选,从而减少计算和噪音。以下是对代码的详细分析:

    1. candidate_elimination 函数

    该函数的目的在于通过消除不重要的候选特征来简化计算并降低噪音。

    • 输入参数:

      • attn:注意力权重张量,表示模板和搜索区域之间的注意力关系。
      • tokens:表示模板和搜索区域的特征。
      • lens_t:模板的长度。
      • keep_ratio:保留的搜索区域特征的比例。
      • global_index:搜索区域特征的全局索引。
      • box_mask_z:模板的掩码。
    • 核心步骤:

      1. 设置模板和搜索区域的分割:通过lens_t分割模板和搜索特征。
      2. 消去潜在候选:
        • 计算模板部分对于搜索区域的注意力。
        • 基于指定的保持比例keep_ratio,决定保留多少搜索特征。
        • 使用排序对注意力权重进行排名,选择最重要的特征。
      3. 返回结果:返回保留后的特征、保留索引和被消除的索引。

    2. CEBlock 类

    • 作用:这是一个将注意力和MLP结合在一起的神经网络模块。特别之处在于整合了候选消除的逻辑,以减少搜索区域上的计算。

    • 构造方法:

      • 使用Attention类作为注意力机制。
      • 包含正则化、drop path和MLP来增强模型鲁棒性。
    • 前向传递:

      • 计算注意力输出并更新特征。
      • 在条件满足时,进行候选消除,从而仅保留最重要的特征。
      • 更新后特征。

    3. Block 类

    • 作用:这是一个标准的块,不如CEBlock那么复杂,没有候选消除逻辑。

    • 构造方法:

      • 类似于CEBlock,也使用了Attention、MLP、正则化和drop path。
    • 前向传递:

      • 依次应用注意力和MLP更新特征,通过drop path增强训练稳定性。

    总结

    这段代码主要在于通过注意力机制对输入特征进行变换和处理,特别在CEBlock中引入了候选消除机制,以减少多余的计算和噪音。利用注意力权重,智能化选择较为重要的搜索特征,从而优化计算效率,适用于需要对输入信息进行简化和降噪的视觉任务中。

  7. User

    那么在Block和CEBlock中是否也可以扩展?增加了对高斯权重?

  8. Assistant

    在 Block 和 CEBlock 中增加对高斯权重的支持是可行的。高斯权重可以用来调整注意力权重的分布,从而更好地关注特定的空间位置或特征。这种扩展可以通过以下步骤实现:

    步骤

    1. 修改Attention模块: 首先需要确保 Attention 类或者模块能够接受和处理高斯权重。这可能涉及到在注意力计算中引入高斯权重的影响。

    2. 添加高斯权重参数: 在 CEBlock 和 Block 的初始化方法中,可以增加与高斯权重相关的参数。例如,可以引入一个可学习的参数或外部传入的高斯地图。

    3. 更新前向方法:

      • 在前向传递中,将高斯权重作为额外参数传递给 self.attn 调用。
      • 在注意力计算过程中,将高斯权重应用到注意力权重矩阵上,例如通过元素级相乘的方式。

    下面是一个示例,展示如何在 CEBlock 中扩展高斯权重:

    class CEBlock(nn.Module):
        def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0.,
                     drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, keep_ratio_search=1.0,
                     use_gaussian=False):
            super().__init__()
            self.use_gaussian = use_gaussian
            self.norm1 = norm_layer(dim)
            self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop)
            self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
            self.norm2 = norm_layer(dim)
            mlp_hidden_dim = int(dim * mlp_ratio)
            self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
    
            self.keep_ratio_search = keep_ratio_search
    
        def forward(self, x, global_index_template, global_index_search, mask=None, ce_template_mask=None, keep_ratio_search=None, gaussian=None):
            if self.use_gaussian and gaussian is not None:
                # Pass the Gaussian map to the attention mechanism
                x_attn, attn = self.attn(self.norm1(x), mask, True, gaussian)
            else:
                x_attn, attn = self.attn(self.norm1(x), mask, True)
            
            x = x + self.drop_path(x_attn)
            lens_t = global_index_template.shape[1]
    
            removed_index_search = None
            if self.keep_ratio_search < 1 and (keep_ratio_search is None or keep_ratio_search < 1):
                keep_ratio_search = self.keep_ratio_search if keep_ratio_search is None else keep_ratio_search
                x, global_index_search, removed_index_search = candidate_elimination(attn, x, lens_t, keep_ratio_search,
                                                                                     global_index_search, ce_template_mask)
    
            x = x + self.drop_path(self.mlp(self.norm2(x)))
            return x, global_index_template, global_index_search, removed_index_search, attn
    

    注意事项

    • 在Attention类中确保对高斯权重允许: Attention 类里,需要确保在计算注意力权重时能接受和利用高斯映射。
    • 确保高斯权重的输入和处理: 适当地处理高斯权重的形状和广播,以适配注意力计算。
    • 模型兼容性: 在扩展功能时,保持对原有功能的兼容性,以便能够在没有高斯权重输入时正常工作。

    结论

    通过引入和结合高斯权重,可以进一步增强模型对特定区域或特征的关注能力。这一机制对于需要强调空间信息或具有先验知识的应用场景可能颇具价值。

  9. User

    此处的attention是:import torch import torch.nn as nn import torch.nn.functional as F from timm.models.layers import trunc_normal_

    from lib.models.layers.rpe import generate_2d_concatenated_self_attention_relative_positional_encoding_index

    class Attention(nn.Module): def init(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0., rpe=False, z_size=7, x_size=14): super().init() self.num_heads = num_heads head_dim = dim // num_heads self.scale = head_dim ** -0.5

        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
        self.attn_drop = nn.Dropout(attn_drop)
        self.proj = nn.Linear(dim, dim)
        self.proj_drop = nn.Dropout(proj_drop)
    
        self.rpe =rpe
        if self.rpe:
            relative_position_index = \
                generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size],
                                                                                           [x_size, x_size])
            self.register_buffer("relative_position_index", relative_position_index)
            # define a parameter table of relative position bias
            self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads,
                                                                          relative_position_index.max() + 1)))
            trunc_normal_(self.relative_position_bias_table, std=0.02)
    
    def forward(self, x, mask=None, return_attention=False):
        # x: B, N, C
        # mask: [B, N, ] torch.bool
        B, N, C = x.shape
        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
        q, k, v = qkv.unbind(0)   # make torchscript happy (cannot use tensor as tuple)
    
        attn = (q @ k.transpose(-2, -1)) * self.scale
    
        if self.rpe:
            relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0)
            attn += relative_position_bias
    
        if mask is not None:
            attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf'),)
    
        attn = attn.softmax(dim=-1)
        attn = self.attn_drop(attn)
    
        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
        x = self.proj(x)
        x = self.proj_drop(x)
    
        if return_attention:
            return x, attn
        else:
            return x
    

    class Attention_talking_head(nn.Module): # taken from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py # with slight modifications to add Talking Heads Attention (https://arxiv.org/pdf/2003.02436v1.pdf) def init(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., rpe=True, z_size=7, x_size=14): super().init()

        self.num_heads = num_heads
    
        head_dim = dim // num_heads
    
        self.scale = qk_scale or head_dim ** -0.5
    
        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
        self.attn_drop = nn.Dropout(attn_drop)
    
        self.proj = nn.Linear(dim, dim)
    
        self.proj_l = nn.Linear(num_heads, num_heads)
        self.proj_w = nn.Linear(num_heads, num_heads)
    
        self.proj_drop = nn.Dropout(proj_drop)
    
        self.rpe = rpe
        if self.rpe:
            relative_position_index = \
                generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size],
                                                                                           [x_size, x_size])
            self.register_buffer("relative_position_index", relative_position_index)
            # define a parameter table of relative position bias
            self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads,
                                                                          relative_position_index.max() + 1)))
            trunc_normal_(self.relative_position_bias_table, std=0.02)
    
    def forward(self, x, mask=None):
        B, N, C = x.shape
        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
        q, k, v = qkv[0] * self.scale, qkv[1], qkv[2]
    
        attn = (q @ k.transpose(-2, -1))
    
        if self.rpe:
            relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0)
            attn += relative_position_bias
    
        if mask is not None:
            attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2),
                                    float('-inf'),)
    
        attn = self.proj_l(attn.permute(0, 2, 3, 1)).permute(0, 3, 1, 2)
    
        attn = attn.softmax(dim=-1)
    
        attn = self.proj_w(attn.permute(0, 2, 3, 1)).permute(0, 3, 1, 2)
        attn = self.attn_drop(attn)
    
        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
        x = self.proj(x)
        x = self.proj_drop(x)
        return x,现在按照multi_head_attention_forward中添加高斯的方法,将高斯集成到现在的attention中。
    
  10. Assistant

    为了将高斯权重集成到现有的 Attention 模块中,我们可以参考 multi_head_attention_forward 函数中的实现,将高斯权重调整添加到注意力权重计算过程中。这可以通过一个新的参数来传递高斯矩阵,并将其应用于注意力权重计算。下面是如何在现有 Attention 和 Attention_talking_head 类中添加该特性的一个示例:

    改进后的 Attention 类

    class Attention(nn.Module):
        def __init__(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0.,
                     rpe=False, z_size=7, x_size=14):
            super().__init__()
            self.num_heads = num_heads
            head_dim = dim // num_heads
            self.scale = head_dim ** -0.5
    
            self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
            self.attn_drop = nn.Dropout(attn_drop)
            self.proj = nn.Linear(dim, dim)
            self.proj_drop = nn.Dropout(proj_drop)
    
            self.rpe =rpe
            if self.rpe:
                relative_position_index = \
                    generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size],
                                                                                               [x_size, x_size])
                self.register_buffer("relative_position_index", relative_position_index)
                # define a parameter table of relative position bias
                self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads,
                                                                              relative_position_index.max() + 1)))
                trunc_normal_(self.relative_position_bias_table, std=0.02)
    
        def forward(self, x, mask=None, return_attention=False, gaussian=None):
            # x: B, N, C
            B, N, C = x.shape
            qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
            q, k, v = qkv.unbind(0)   # make torchscript happy (cannot use tensor as tuple)
    
            attn = (q @ k.transpose(-2, -1)) * self.scale
    
            if self.rpe:
                relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0)
                attn += relative_position_bias
    
            if mask is not None:
                attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf'),)
    
            if gaussian is not None:
                attn += gaussian.unsqueeze(0).unsqueeze(1)  # Assuming gaussian has shape (B, N, N)
    
            attn = attn.softmax(dim=-1)
            attn = self.attn_drop(attn)
    
            x = (attn @ v).transpose(1, 2).reshape(B, N, C)
            x = self.proj(x)
            x = self.proj_drop(x)
    
            if return_attention:
                return x, attn
            else:
                return x
    

    改进后的 Attention_talking_head 类

    同样的调整可以应用于Attention_talking_head类:

    class Attention_talking_head(nn.Module):
        def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.,
                     rpe=True, z_size=7, x_size=14):
            super().__init__()
    
            self.num_heads = num_heads
    
            head_dim = dim // num_heads
    
            self.scale = qk_scale or head_dim ** -0.5
    
            self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
            self.attn_drop = nn.Dropout(attn_drop)
    
            self.proj = nn.Linear(dim, dim)
    
            self.proj_l = nn.Linear(num_heads, num_heads)
            self.proj_w = nn.Linear(num_heads, num_heads)
    
            self.proj_drop = nn.Dropout(proj_drop)
    
            self.rpe = rpe
            if self.rpe:
                relative_position_index = \
                    generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size],
                                                                                               [x_size, x_size])
                self.register_buffer("relative_position_index", relative_position_index)
                # define a parameter table of relative position bias
                self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads,
                                                                              relative_position_index.max() + 1)))
                trunc_normal_(self.relative_position_bias_table, std=0.02)
    
        def forward(self, x, mask=None, gaussian=None):
            B, N, C = x.shape
            qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
            q, k, v = qkv[0] * self.scale, qkv[1], qkv[2]
    
            attn = (q @ k.transpose(-2, -1))
    
            if self.rpe:
                relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0)
                attn += relative_position_bias
    
            if mask is not None:
                attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2),
                                        float('-inf'),)
    
            if gaussian is not None:
                attn += gaussian.unsqueeze(0).unsqueeze(1)  # Assuming gaussian has shape (B, N, N)
    
            attn = self.proj_l(attn.permute(0, 2, 3, 1)).permute(0, 3, 1, 2)
    
            attn = attn.softmax(dim=-1)
    
            attn = self.proj_w(attn.permute(0, 2, 3, 1)).permute(0, 3, 1, 2)
            attn = self.attn_drop(attn)
    
            x = (attn @ v).transpose(1, 2).reshape(B, N, C)
            x = self.proj(x)
            x = self.proj_drop(x)
            return x
    

    改动说明

    1. 增加 gaussian 参数: 在forward方法中多了一个gaussian参数,用于传递高斯矩阵。
    2. 应用高斯调整: 如果提供了高斯矩阵,则将其添加到注意力权重上。这一处理与相对位置偏置类似。
    3. 假设高斯的形状: 假定高斯矩阵具有与注意力权重匹配的形状,或适当的广播形状。

    通过这个扩展,模型可以在权重计算中进一步考虑空间信息或事先设定的注意力偏好。

  11. User

    我们这样修改:import torch import torch.nn as nn import torch.nn.functional as F from timm.models.layers import trunc_normal_

    from lib.models.layers.rpe import generate_2d_concatenated_self_attention_relative_positional_encoding_index

    class Attention(nn.Module): def init(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0., rpe=False, z_size=7, x_size=14): super().init() self.num_heads = num_heads head_dim = dim // num_heads self.scale = head_dim ** -0.5

        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
        self.attn_drop = nn.Dropout(attn_drop)
        self.proj = nn.Linear(dim, dim)
        self.proj_drop = nn.Dropout(proj_drop)
        self.rpe =rpe
        
        if self.rpe:
            relative_position_index = \
                generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size],
                                                                                           [x_size, x_size])
            self.register_buffer("relative_position_index", relative_position_index)
            # define a parameter table of relative position bias
            self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads,
                                                                          relative_position_index.max() + 1)))
            trunc_normal_(self.relative_position_bias_table, std=0.02)
        # 引入 高斯
        self.lamb = nn.Parameter(torch.zeros(num_heads), requires_grad=True)  # 添加高斯参数
        
    def forward(self, x, mask=None, return_attention=False):
        # x: B, N, C
        # mask: [B, N, ] torch.bool
        B, N, C = x.shape
        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
        q, k, v = qkv.unbind(0)   # make torchscript happy (cannot use tensor as tuple)
    
        attn = (q @ k.transpose(-2, -1)) * self.scale
    
        if self.rpe:
            relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0)
            attn += relative_position_bias
    
        if mask is not None:
            attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf'),)
    
        attn = attn.softmax(dim=-1)
        attn = self.attn_drop(attn)
    
        # 引入高斯权重
        gaussian = torch.exp(-((attn.unsqueeze(-1) - 0.5) ** 2) / (2 * 0.1 ** 2))  # 调整高斯函数参数
        attn = attn + gaussian * self.lamb.unsqueeze(0).unsqueeze(2).unsqueeze(3)
        
        ############
        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
        x = self.proj(x)
        x = self.proj_drop(x)
    
        if return_attention:
            return x, attn
        else:
            return x
    
  12. Assistant

    在这个修改后的 Attention 类中,我们引入了高斯权重,并对注意力权重进行了调整。具体的高斯改动如下:

    修改说明:

    1. 高斯参数 lamb:

      • 引入了一个新的可学习参数lamb,它的形状为 (num_heads,)。
      • 该参数将作用于每个注意力头的高斯调整中。
    2. 生成高斯权重:

      • 使用了高斯函数来生成一个高斯矩阵,并将其与原有的注意力权重相结合。
      • 高斯权重基于一个均值为0.5和标准差上为0.1的正态分布。这是一个经验设定,你可以根据具体需求进行调整。
      • 使用 torch.exp 计算高斯权重,之后乘以 lamb 以进行强度调整。
    3. 更新注意力权重:

      • 将得到的高斯矩阵乘以系数lamb并加到注意力权重上。这样可以在注意力权重的基础上,进一步引入空间上的先验偏置。

    使用方法:

    • 输入与输出:
      • 输入为 (B, N, C) 的特征向量,同样采取了对 mask 和 return_attention 两个参数的支持。
      • 输出将是调整后的特征向量,原则上维度不变。

    代码中的注意事项:

    • 广播机制:确保所有不同形状的张量在相加或乘之前,应适当调整形状(例如使用 unsqueeze)。
    • 高斯函数参数调整:函数中采用的默认值(如均值和方差)是一个选择的起始点。你可以根据具体数据特点来调节这些值。
    • 学习参数:你可以将高斯方差设为可学习参数,以便模型在训练过程中自行调整到最合适的状态。

    使用场景:

    这样整合了高斯权重的注意力机制可以在需要进一步突出某些特定空间位置的信息时使用,适用于如目标检测、语义分割等任务。在这类任务中,各个特征的空间关系尤为重要。

  13. User

    那么在当前修改中,与multi_head_attention_forward中使用高斯的方法有何不同?

  14. Assistant

    在当前修改的 Attention 类中,我们引入了一个高斯公式来调整注意力权重,而在 multi_head_attention_forward 函数中,对高斯的使用可能是直接与注意力权重相结合的。以下是两者之间不同点的详细分析:

    1. 高斯处理方式

    • 在 Attention 类中的高斯处理:

      • 使用高斯函数生成一个高斯分布的权重矩阵:
        gaussian = torch.exp(-((attn.unsqueeze(-1) - 0.5) ** 2) / (2 * 0.1 ** 2))
        
      • 使用这个权重来调整注意力矩阵,以此来进一步强化某一特定分布的权重。
    • 在 multi_head_attention_forward 中的高斯处理:

      • 该函数中高斯权重可能是直接从外部传入的一个张量,然后与注意力权重直接相加或点乘。
      • 这种方法依赖对高斯权重的直接使用,可能更多依赖先验知识。

    2. 高斯权重的来源

    • 自生成高斯权重:

      • 在 Attention 类中,高斯权重的生成是基于一个固定公式,且通过模型学习参数 lamb 进行调整。通过训练得到的参数,可以适应特定数据集。
    • 外部提供的高斯权重:

      • 在 multi_head_attention_forward 的实现中,高斯权重从外部传入,意味着这些权重可能是基于另一种方法或先验知识计算出的。

    3. 调整注意力权重的方式

    • 方法不同:
      • 在 Attention 类中,生成的高斯权重是通过一个预定公式,目的是创建某种中心化的注意力分布。
      • 在 multi_head_attention_forward 中,高斯权重的使用灵活,可能直接影响注意力的计算,其方式可能基于特定场景和需求。

    4. 目的和扩展性

    • 目的:

      • 两种方法的设计目的可能不同。Attention 类中的方法,更多是希望约束或引导注意力权重到一个高斯分布上。
      • multi_head_attention_forward 的方法,更强调灵活应用现有先验知识来处理注意力权重。
    • 扩展性:

      • Attention 采用固定公式扩展性更强,可通过调整均值方差等参数定制化。
      • multi_head_attention_forward 可以接收不同形态的高斯信息,更灵活。

    通过这种分析,你可以看到,在实现与设计中选择合适的方式,往往需要考虑到具体任务的需求、模型的灵活性和数据特征。

  15. User

    那么现在根据multi_head_attention_forward中高斯的计算和使用,修改Attention中的高斯计算

  16. Assistant

    为了将 Attention 类中的高斯计算方式与 multi_head_attention_forward 中的高斯使用方式对齐,我们需要更改高斯权重的计算,并在处理注意力权重时更紧密地结合高斯调整。以下是可能的步骤:

    1. 高斯权重的传入

    首先,我们需要接收外部的高斯权重信息。这意味着我们可能需要在 forward 函数中增加一个参数,专门用于传入高斯权重。

    2. 高斯权重的处理方式

    根据 multi_head_attention_forward 中的方法,外部的高斯权重通常用于直接调整注意力权重。在 Attention 类的实现中,我们可以考虑高斯权重如何适应和调整注意力权重。

    代码示例

    class Attention(nn.Module):
        def __init__(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0.,
                     rpe=False, z_size=7, x_size=14):
            super().__init__()
            self.num_heads = num_heads
            head_dim = dim // num_heads
            self.scale = head_dim ** -0.5
    
            self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
            self.attn_drop = nn.Dropout(attn_drop)
            self.proj = nn.Linear(dim, dim)
            self.proj_drop = nn.Dropout(proj_drop)
            self.rpe = rpe
    
            if self.rpe:
                relative_position_index = \
                    generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size],
                                                                                               [x_size, x_size])
                self.register_buffer("relative_position_index", relative_position_index)
                self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads,
                                                                              relative_position_index.max() + 1)))
                trunc_normal_(self.relative_position_bias_table, std=0.02)
    
            # 引入高斯比例参数
            self.lamb = nn.Parameter(torch.zeros(num_heads), requires_grad=True)
            
        def forward(self, x, mask=None, return_attention=False, gaussian=None):
            B, N, C = x.shape
            qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
            q, k, v = qkv.unbind(0)
    
            attn = (q @ k.transpose(-2, -1)) * self.scale
    
            if self.rpe:
                relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0)
                attn += relative_position_bias
    
            # 根据 mask 设置注意力分数
            if mask is not None:
                attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf'))
    
            # 应用高斯权重
            if gaussian is not None:
                attn += gaussian * self.lamb.unsqueeze(0).unsqueeze(2).unsqueeze(3)
    
            attn = attn.softmax(dim=-1)
            attn = self.attn_drop(attn)
    
            x = (attn @ v).transpose(1, 2).reshape(B, N, C)
            x = self.proj(x)
            x = self.proj_drop(x)
    
            if return_attention:
                return x, attn
            else:
                return x
    

    修改说明

    1. 传入高斯权重: 添加一个 gaussian 的参数至 forward 方法,以接收外部计算好的高斯权重。

    2. 高斯权重应用: 在计算好的注意力矩阵上应用传入的高斯权重,乘以系数 lamb 后与注意力矩阵相加。这与 multi_head_attention_forward 耦合高斯权重的方式一致。

    3. 灵活性: 外部传入的高斯权重可以根据实际需要灵活调整,Attention 类本身不亲自生成高斯权重,却能对其进行有效整合和应用。

    通过这样的调整,高斯权重的适应性得以提高,应用场景更为广泛,适用于在有先验高斯信息的注意力机制中增强对特定空间或特征的关注。

  17. User

    将import torch import torch.nn as nn from torch.nn.functional import linear, pad from torch import Tensor from torch.nn import MultiheadAttention

    from typing import Optional, Tuple, List import warnings

    def multi_head_attention_forward( query: Tensor, key: Tensor, value: Tensor, embed_dim_to_check: int, num_heads: int, in_proj_weight: Tensor, in_proj_bias: Tensor, bias_k: Optional[Tensor], bias_v: Optional[Tensor], add_zero_attn: bool, dropout_p: float, out_proj_weight: Tensor, out_proj_bias: Tensor, training: bool = True, key_padding_mask: Optional[Tensor] = None, need_weights: bool = True, attn_mask: Optional[Tensor] = None, use_separate_proj_weight: bool = False, q_proj_weight: Optional[Tensor] = None, k_proj_weight: Optional[Tensor] = None, v_proj_weight: Optional[Tensor] = None, static_k: Optional[Tensor] = None, static_v: Optional[Tensor] = None, gaussian: Optional[Tensor] = None, lamb = None, ) -> Tuple[Tensor, Optional[Tensor]]: r""" Args: query, key, value: map a query and a set of key-value pairs to an output. See "Attention Is All You Need" for more details. embed_dim_to_check: total dimension of the model. num_heads: parallel attention heads. in_proj_weight, in_proj_bias: input projection weight and bias. bias_k, bias_v: bias of the key and value sequences to be added at dim=0. add_zero_attn: add a new batch of zeros to the key and value sequences at dim=1. dropout_p: probability of an element to be zeroed. out_proj_weight, out_proj_bias: the output projection weight and bias. training: apply dropout if is True. key_padding_mask: if provided, specified padding elements in the key will be ignored by the attention. This is an binary mask. When the value is True, the corresponding value on the attention layer will be filled with -inf. need_weights: output attn_output_weights. attn_mask: 2D or 3D mask that prevents attention to certain positions. This is an additive mask (i.e. the values will be added to the attention layer). A 2D mask will be broadcasted for all the batches while a 3D mask allows to specify a different mask for the entries of each batch. use_separate_proj_weight: the function accept the proj. weights for query, key, and value in different forms. If false, in_proj_weight will be used, which is a combination of q_proj_weight, k_proj_weight, v_proj_weight. q_proj_weight, k_proj_weight, v_proj_weight, in_proj_bias: input projection weight and bias. static_k, static_v: static key and value used for attention operators. gaussian: the generated Gaussian-like weight map

    Shape:
        Inputs:
        - query: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is
          the embedding dimension.
        - key: :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is
          the embedding dimension.
        - value: :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is
          the embedding dimension.
        - key_padding_mask: :math:`(N, S)` where N is the batch size, S is the source sequence length.
          If a ByteTensor is provided, the non-zero positions will be ignored while the zero positions
          will be unchanged. If a BoolTensor is provided, the positions with the
          value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged.
        - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length.
          3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length,
          S is the source sequence length. attn_mask ensures that position i is allowed to attend the unmasked
          positions. If a ByteTensor is provided, the non-zero positions are not allowed to attend
          while the zero positions will be unchanged. If a BoolTensor is provided, positions with ``True``
          are not allowed to attend while ``False`` values will be unchanged. If a FloatTensor
          is provided, it will be added to the attention weight.
        - static_k: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length,
          N is the batch size, E is the embedding dimension. E/num_heads is the head dimension.
        - static_v: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length,
          N is the batch size, E is the embedding dimension. E/num_heads is the head dimension.
    
        Outputs:
        - attn_output: :math:`(L, N, E)` where L is the target sequence length, N is the batch size,
          E is the embedding dimension.
        - attn_output_weights: :math:`(N, L, S)` where N is the batch size,
          L is the target sequence length, S is the source sequence length.
    """
    tgt_len, bsz, embed_dim = query.size()
    assert embed_dim == embed_dim_to_check
    # allow MHA to have different sizes for the feature dimension
    assert key.size(0) == value.size(0) and key.size(1) == value.size(1)
    
    head_dim = embed_dim // num_heads
    assert head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads"
    scaling = float(head_dim) ** -0.5
    
    if not use_separate_proj_weight:
        if torch.equal(query, key) and torch.equal(key, value):
            # self-attention
            q, k, v = linear(query, in_proj_weight, in_proj_bias).chunk(3, dim=-1)
    
        elif torch.equal(key, value):
            # encoder-decoder attention
            # This is inline in_proj function with in_proj_weight and in_proj_bias
            _b = in_proj_bias
            _start = 0
            _end = embed_dim
            _w = in_proj_weight[_start:_end, :]
            if _b is not None:
                _b = _b[_start:_end]
            q = linear(query, _w, _b)
    
            if key is None:
                assert value is None
                k = None
                v = None
            else:
    
                # This is inline in_proj function with in_proj_weight and in_proj_bias
                _b = in_proj_bias
                _start = embed_dim
                _end = None
                _w = in_proj_weight[_start:, :]
                if _b is not None:
                    _b = _b[_start:]
                k, v = linear(key, _w, _b).chunk(2, dim=-1)
    
        else:
            # This is inline in_proj function with in_proj_weight and in_proj_bias
            _b = in_proj_bias
            _start = 0
            _end = embed_dim
            _w = in_proj_weight[_start:_end, :]
            if _b is not None:
                _b = _b[_start:_end]
            q = linear(query, _w, _b)
    
            # This is inline in_proj function with in_proj_weight and in_proj_bias
            _b = in_proj_bias
            _start = embed_dim
            _end = embed_dim * 2
            _w = in_proj_weight[_start:_end, :]
            if _b is not None:
                _b = _b[_start:_end]
            k = linear(key, _w, _b)
    
            # This is inline in_proj function with in_proj_weight and in_proj_bias
            _b = in_proj_bias
            _start = embed_dim * 2
            _end = None
            _w = in_proj_weight[_start:, :]
            if _b is not None:
                _b = _b[_start:]
            v = linear(value, _w, _b)
    else:
        q_proj_weight_non_opt = torch.jit._unwrap_optional(q_proj_weight)
        len1, len2 = q_proj_weight_non_opt.size()
        assert len1 == embed_dim and len2 == query.size(-1)
    
        k_proj_weight_non_opt = torch.jit._unwrap_optional(k_proj_weight)
        len1, len2 = k_proj_weight_non_opt.size()
        assert len1 == embed_dim and len2 == key.size(-1)
    
        v_proj_weight_non_opt = torch.jit._unwrap_optional(v_proj_weight)
        len1, len2 = v_proj_weight_non_opt.size()
        assert len1 == embed_dim and len2 == value.size(-1)
    
        if in_proj_bias is not None:
            q = linear(query, q_proj_weight_non_opt, in_proj_bias[0:embed_dim])
            k = linear(key, k_proj_weight_non_opt, in_proj_bias[embed_dim:(embed_dim * 2)])
            v = linear(value, v_proj_weight_non_opt, in_proj_bias[(embed_dim * 2):])
        else:
            q = linear(query, q_proj_weight_non_opt, in_proj_bias)
            k = linear(key, k_proj_weight_non_opt, in_proj_bias)
            v = linear(value, v_proj_weight_non_opt, in_proj_bias)
    q = q * scaling
    
    if attn_mask is not None:
        if attn_mask.dim() == 2:
            attn_mask = attn_mask.unsqueeze(0)
            if list(attn_mask.size()) != [1, query.size(0), key.size(0)]:
                raise RuntimeError('The size of the 2D attn_mask is not correct.')
        elif attn_mask.dim() == 3:
            if list(attn_mask.size()) != [bsz * num_heads, query.size(0), key.size(0)]:
                raise RuntimeError('The size of the 3D attn_mask is not correct.')
        else:
            raise RuntimeError("attn_mask's dimension {} is not supported".format(attn_mask.dim()))
        # attn_mask's dim is 3 now.
    
    if bias_k is not None and bias_v is not None:
        if static_k is None and static_v is None:
            k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
            v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
            if attn_mask is not None:
                attn_mask = pad(attn_mask, (0, 1))
            if key_padding_mask is not None:
                key_padding_mask = pad(key_padding_mask, (0, 1))
        else:
            assert static_k is None, "bias cannot be added to static key."
            assert static_v is None, "bias cannot be added to static value."
    else:
        assert bias_k is None
        assert bias_v is None
    
    q = q.contiguous().view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1)
    if k is not None:
        k = k.contiguous().view(-1, bsz * num_heads, head_dim).transpose(0, 1)
    if v is not None:
        v = v.contiguous().view(-1, bsz * num_heads, head_dim).transpose(0, 1)
    
    if static_k is not None:
        assert static_k.size(0) == bsz * num_heads
        assert static_k.size(2) == head_dim
        k = static_k
    
    if static_v is not None:
        assert static_v.size(0) == bsz * num_heads
        assert static_v.size(2) == head_dim
        v = static_v
    
    src_len = k.size(1)
    
    if key_padding_mask is not None:
        assert key_padding_mask.size(0) == bsz
        assert key_padding_mask.size(1) == src_len
    
    if add_zero_attn:
        src_len += 1
        k = torch.cat([k, torch.zeros((k.size(0), 1) + k.size()[2:], dtype=k.dtype, device=k.device)], dim=1)
        v = torch.cat([v, torch.zeros((v.size(0), 1) + v.size()[2:], dtype=v.dtype, device=v.device)], dim=1)
        if attn_mask is not None:
            attn_mask = pad(attn_mask, (0, 1))
        if key_padding_mask is not None:
            key_padding_mask = pad(key_padding_mask, (0, 1))
    
    attn_output_weights = torch.bmm(q, k.transpose(1, 2))
    assert list(attn_output_weights.size()) == [bsz * num_heads, tgt_len, src_len]
    
    if attn_mask is not None:
        attn_output_weights += attn_mask
    
    if key_padding_mask is not None:
        attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
        attn_output_weights = attn_output_weights.masked_fill(
            key_padding_mask.unsqueeze(1).unsqueeze(2), float('-inf'),)
        attn_output_weights = attn_output_weights.view(bsz * num_heads, tgt_len, src_len)
    
    attn_output_weights = attn_output_weights + gaussian[0].permute(2, 0, 1)
    attn_output_weights = torch.nn.functional.softmax(attn_output_weights, dim=-1)
    # High-Frequency Emphasis  HFE
    attn_d = torch.ones(attn_output_weights.shape[-2:], device=attn_output_weights.device)/tgt_len
    attn_d = attn_d[None, None, ...]
    attn_h = attn_output_weights.view(bsz, num_heads, tgt_len, src_len) - attn_d
    attn_h = attn_h * (1. + lamb[None, :, None, None])
    attn_output_weights = (attn_d + attn_h).view(bsz*num_heads, tgt_len, src_len)
    
    attn_output_weights = torch.nn.functional.dropout(attn_output_weights, p=dropout_p,
                                                      training=training)
    
    attn_output = torch.bmm(attn_output_weights, v)
    assert list(attn_output.size()) == [bsz * num_heads, tgt_len, head_dim]
    attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim)
    attn_output = linear(attn_output, out_proj_weight, out_proj_bias)
    
    return attn_output, attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
    

    class GaussianMultiheadAttention(MultiheadAttention): def init(self, embed_dim, num_heads, **kwargs): super(GaussianMultiheadAttention, self).init(embed_dim, num_heads, **kwargs) self.gaussian = True self.lamb = nn.Parameter(torch.zeros(num_heads), requires_grad=True)

    def forward(self, query, key, value, key_padding_mask=None,
                need_weights=False, attn_mask=None, gaussian=None):
        # type: (Tensor, Tensor, Tensor, Optional[Tensor], bool, Optional[Tensor], Optional[Tensor]) -> Tuple[Tensor, Optional[Tensor]]
        r"""
    Args:
        query, key, value: map a query and a set of key-value pairs to an output.
            See "Attention Is All You Need" for more details.
        key_padding_mask: if provided, specified padding elements in the key will
            be ignored by the attention. When given a binary mask and a value is True,
            the corresponding value on the attention layer will be ignored. When given
            a byte mask and a value is non-zero, the corresponding value on the attention
            layer will be ignored
        need_weights: output attn_output_weights.
        attn_mask: 2D or 3D mask that prevents attention to certain positions. A 2D mask will be broadcasted for all
            the batches while a 3D mask allows to specify a different mask for the entries of each batch.
        gaussian: 2D gaussian attention map that focus attention to certain object queries' initial estimations
            with handcrafted query spatial priors.
    
    Shape:
        - Inputs:
        - query: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is
          the embedding dimension.
        - key: :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is
          the embedding dimension.
        - value: :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is
          the embedding dimension.
        - key_padding_mask: :math:`(N, S)` where N is the batch size, S is the source sequence length.
          If a ByteTensor is provided, the non-zero positions will be ignored while the position
          with the zero positions will be unchanged. If a BoolTensor is provided, the positions with the
          value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged.
        - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length.
          3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length,
          S is the source sequence length. attn_mask ensure that position i is allowed to attend the unmasked
          positions. If a ByteTensor is provided, the non-zero positions are not allowed to attend
          while the zero positions will be unchanged. If a BoolTensor is provided, positions with ``True``
          is not allowed to attend while ``False`` values will be unchanged. If a FloatTensor
          is provided, it will be added to the attention weight.
        - gaussian: :math:`(L, S, nhead * batch_size)`, where nhead is the number of head in multi-head
          attention module, L is the target sequence length, S is the source sequence length.
    
        - Outputs:
        - attn_output: :math:`(L, N, E)` where L is the target sequence length, N is the batch size,
          E is the embedding dimension.
        - attn_output_weights: :math:`(N, L, S)` where N is the batch size,
          L is the target sequence length, S is the source sequence length.
        """
        if not self._qkv_same_embed_dim:
            return multi_head_attention_forward(
                query, key, value, self.embed_dim, self.num_heads,
                self.in_proj_weight, self.in_proj_bias,
                self.bias_k, self.bias_v, self.add_zero_attn,
                self.dropout, self.out_proj.weight, self.out_proj.bias,
                training=self.training,
                key_padding_mask=key_padding_mask, need_weights=need_weights,
                attn_mask=attn_mask, use_separate_proj_weight=True,
                q_proj_weight=self.q_proj_weight, k_proj_weight=self.k_proj_weight,
                v_proj_weight=self.v_proj_weight, gaussian=gaussian,
                lamb=self.lamb)
        else:
            return multi_head_attention_forward(
                query, key, value, self.embed_dim, self.num_heads,
                self.in_proj_weight, self.in_proj_bias,
                self.bias_k, self.bias_v, self.add_zero_attn,
                self.dropout, self.out_proj.weight, self.out_proj.bias,
                training=self.training,
                key_padding_mask=key_padding_mask, need_weights=need_weights,
                attn_mask=attn_mask, gaussian=gaussian,
                lamb=self.lamb)中涉及到高斯的部分单独列出来
    
  18. Assistant

    在给定的 multi_head_attention_forward 函数中,高斯权重部分主要涉及到如何利用一个高斯映射来影响注意力权重的计算。这些高斯权重的整合部分可以单独拉出来分析和归纳。以下是代码中与高斯相关的部分:

    高斯权重整合步骤

    1. 高斯权重参数:

      • 函数接收一个名为 gaussian 的参数,用于传递一个外部计算好的高斯权重矩阵。
      • 定义了 lamb,这是一个用于调整高斯影响的参数。
    2. 高斯权重应用到注意力权重:

      • 在计算完成原始注意力权重 attn_output_weights 后,将高斯权重直接与注意力权重相加。
      • 注意,这里假设 gaussian 张量的形状经过变换后可以与 attn_output_weights 对应。
    3. 高斯调整与高频强调 (High-Frequency Emphasis, HFE):

      • 增加了一个高频强调的步骤,即:
        • 计算平均注意力分布 attn_d。
        • 对原始加权和高斯调整后的注意力做微调,提升某些高频特征。

    高斯相关的主要代码段

    # Apply Gaussian-related adjustments
    attn_output_weights = attn_output_weights + gaussian[0].permute(2, 0, 1)
    
    attn_output_weights = torch.nn.functional.softmax(attn_output_weights, dim=-1)
    
    # High-Frequency Emphasis (HFE)
    attn_d = torch.ones(attn_output_weights.shape[-2:], device=attn_output_weights.device) / tgt_len
    attn_d = attn_d[None, None, ...]
    attn_h = attn_output_weights.view(bsz, num_heads, tgt_len, src_len) - attn_d
    attn_h = attn_h * (1. + lamb[None, :, None, None])
    attn_output_weights = (attn_d + attn_h).view(bsz * num_heads, tgt_len, src_len)
    

    解释

    • 高斯整合:

      • attn_output_weights = attn_output_weights + gaussian[0].permute(2, 0, 1) 是整合高斯权重的核心步骤,假设 gaussian 张量可以通过调整形状与注意力权重相加。
    • HFE 步骤:

      • 先取得平均注意力分布 attn_d 。
      • 通过计算注意力差异 attn_h 并结合 lamb 进行调整。
      • 将调整后的结果再平滑整合到最终权重中,达到提升某些特征的目的。

    此设计使用了高斯分布和高频强调来增强对特定特征的区分能力,使得模型可以在某些位置上更敏感、更准确。结合了高斯权重的设计主要目的是为了解决注意力普遍性和局部强关注之间的关系。

  19. Assistant

    如果您还有什么问题或者需要进一步的帮助,请随时告诉我!

  20. User

    那么attn_output_weights是如何得到?在import torch import torch.nn as nn import torch.nn.functional as F from timm.models.layers import trunc_normal_

    from lib.models.layers.rpe import generate_2d_concatenated_self_attention_relative_positional_encoding_index

    class Attention(nn.Module): def init(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0., rpe=False, z_size=7, x_size=14): super().init() self.num_heads = num_heads head_dim = dim // num_heads self.scale = head_dim ** -0.5

        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
        self.attn_drop = nn.Dropout(attn_drop)
        self.proj = nn.Linear(dim, dim)
        self.proj_drop = nn.Dropout(proj_drop)
        self.rpe =rpe
        
        if self.rpe:
            relative_position_index = \
                generate_2d_concatenated_self_attention_relative_positional_encoding_index([z_size, z_size],
                                                                                           [x_size, x_size])
            self.register_buffer("relative_position_index", relative_position_index)
            # define a parameter table of relative position bias
            self.relative_position_bias_table = nn.Parameter(torch.empty((num_heads,
                                                                          relative_position_index.max() + 1)))
            trunc_normal_(self.relative_position_bias_table, std=0.02)
        # 引入 高斯比例参数
        self.lamb = nn.Parameter(torch.zeros(num_heads), requires_grad=True)  # 添加高斯参数
        
    def forward(self, x, mask=None, return_attention=False, gaussian=None):
        # x: B, N, C
        # mask: [B, N, ] torch.bool
        B, N, C = x.shape
        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
        q, k, v = qkv.unbind(0)   # make torchscript happy (cannot use tensor as tuple)
    
        attn = (q @ k.transpose(-2, -1)) * self.scale
    
        if self.rpe:
            relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0)
            attn += relative_position_bias
    
        if mask is not None:
            attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf'),)
        # 应用高斯权重
        if gaussian is not None:
            attn += gaussian*self.lamb.unsqueeze(0).unsqueeze(2).unsqueeze(3)
            
        attn = attn.softmax(dim=-1)
        attn = self.attn_drop(attn)
        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
        x = self.proj(x)
        x = self.proj_drop(x)
    
        if return_attention:
            return x, attn
        else:
            return x中如何计算的高斯?不一样吗?
    
  21. Assistant

    在这段 Attention 类代码中,attn_output_weights 可以理解为计算注意力权重的矩阵,即变量 attn。该矩阵是在前向传递 forward 函数中计算的:

    计算 attn_output_weights

    1. 线性变换和重整形:

      • 使用线性变换 self.qkv 从输入 x 中提取查询(query)、键(key)、值(value):
        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads)
        q, k, v = qkv.unbind(0)  # 将 qkv 拆分成单独的 q, k, v
        
    2. 计算注意力权重:

      • 计算查询 q 和键 k 的内积,并缩放结果:
        attn = (q @ k.transpose(-2, -1)) * self.scale
        
      • 如果有位置偏差(rpe),则加入到注意力值中:
        if self.rpe:
            relative_position_bias = self.relative_position_bias_table[:, self.relative_position_index].unsqueeze(0)
            attn += relative_position_bias
        
      • 如果有mask时,将指定位置的注意力权重设为负无穷:
        if mask is not None:
            attn = attn.masked_fill(mask.unsqueeze(1).unsqueeze(2), float('-inf'))
        
    3. 应用高斯权重:

      • 加入高斯权重选项,调整注意力值,根据外部提供的gaussian和可学习参数lamb:
        if gaussian is not None:
            attn += gaussian * self.lamb.unsqueeze(0).unsqueeze(2).unsqueeze(3)
        
    4. 得到归一化权重:

      • 对注意力权重进行softmax归一化,得到 attn_output_weights:
        attn = attn.softmax(dim=-1)
        attn = self.attn_drop(attn)
        

    比较 multi_head_attention_forward

    在 multi_head_attention_forward 中的高斯使用方式,两者之间的差别主要在计算逻辑及目的上:

    • 生成方式:Attention 类中的高斯权重来自外部输入变量 gaussian,并在内部用 lamb 进行增益调整,而 multi_head_attention_forward 直接将高斯矩阵与注意力矩阵进行相加,可能是一个基于先验或特定偏好设计的高斯分布。

    • 应用步骤:Attention 类中通过高斯执行真正的微调,而在 multi_head_attention_forward 函数中,给出了高斯整合算法段落并利用特定手段(如 HFE)进一步加强某些数据的权值,注重更细致的特征调整。

    两者在处理流程上,这使得 Attention 可以灵活地处理、调节外部提供的高斯信息,而 multi_head_attention_forward 则侧重于更通用的实现方式,直接利用高斯权重进行调制。这意味着在实践中,使用哪一种方法,将取决于数据特征以及特定任务需求。

  22. User

    但是Attention中并未对高斯定义

  23. Assistant

    您说得对,在给定的 Attention 类实现中,高斯权重 gaussian 只是作为一个可能的输入参数被引入到 forward 方法中,但没有在方法内对其进行具体的定义或生成。这意味着在使用该 Attention 类时,需要从外部提供高斯权重。这种设计使得 Attention 层能够通过参数灵活地利用高斯信息,而不需要在类内部生成高斯权重。

    处理高斯权重的方式

    1. 外部生成:

      • 高斯权重(即参数gaussian)应在调用 Attention 类的 forward 方法时,从外部计算和提供。这可以通过预处理的步骤或在模型其他层中生成。
    2. 注意力权重的调节:

      • 如果提供了 gaussian,模型会将其与注意力矩阵相加,并乘以可学习参数 lamb,这可以看作是对注意力分布的一种偏向性调整。

    拓展与实现

    如果希望让 Attention 类自身生成某种形式的高斯权重,需要在类内部添加额外的逻辑。例如:

    # 伪代码示例:Attention类内部生成简单高斯权重
    
    import torch
    
    class Attention(nn.Module):
        def __init__(self, ...):
            ...
            self.gaussian_sigma = 0.5 # 自定义的标准差
    
        def forward(self, x, mask=None, return_attention=False):
            ...
            
            # 如果没有提供高斯权重,从内部生成
            if gaussian is None:
                B, N, _ = x.shape
                gaussian = torch.exp(-((torch.arange(N, dtype=torch.float32) - N/2) ** 2) / (2 * self.gaussian_sigma ** 2))
                gaussian = gaussian.unsqueeze(0).unsqueeze(1).to(x.device)  # (1, 1, N)
            
            # 应用生成或外部传入的高斯权重
            attn += gaussian * self.lamb.unsqueeze(0).unsqueeze(2).unsqueeze(3)
            ...
    
    

    选择性地使用

    通过这种方法,您可以在需要时从外部输入精确的高斯矩阵,而在其他情况下,允许 Attention 类自适应创建基本高斯分布。这赋予了模型设计高度的灵活性,以适应不同的任务和场景需求。这样就可以利用预定义的或动态生成的高斯信息来微调注意力机制造成的注意力模式。