分析:import os import torch import torch.nn as nn import torch.nn.functional as F import…
- User
分析:import os import torch import torch.nn as nn import torch.nn.functional as F import math from .utils import cosine_beta_schedule, default, extract from .UNet import UNet from .UViT import UViT
from SiamB.core.defaults import cfg
import matplotlib.pyplot as plt import gc def get_l2(f): c, h, w = f.shape mean = torch.mean(f, dim=0).unsqueeze(0).repeat(c, 1, 1) f = torch.sum(torch.pow((f - mean), 2), dim=0) / c f = f return (f - f.min()) / (f.max() - f.min())
def draw(f, name): plt.imshow(get_l2(f).detach().cpu().numpy()) plt.title(name) plt.show()
class SinusoidalPositionEmbeddings(nn.Module): def init(self, dim): super().init() self.dim = dim
def forward(self, time): # device = time.device half_dim = self.dim // 2 embeddings = math.log(10000) / (half_dim - 1) embeddings = torch.exp(torch.arange(half_dim).cuda() * -embeddings) embeddings = time[:, None] * embeddings[None, :] embeddings = torch.cat((embeddings.sin(), embeddings.cos()), dim=-1) return embeddingsclass Time_Embedding(nn.Module): def init(self, cfg): super(Time_Embedding, self).init() self.time_mlp = nn.Sequential( SinusoidalPositionEmbeddings(int(cfg.Time_Embedding.Dim / 4)), nn.Linear(int(cfg.Time_Embedding.Dim / 4), cfg.Time_Embedding.Dim), nn.GELU(), nn.Linear(cfg.Time_Embedding.Dim, cfg.Time_Embedding.Dim), ) def forward(self, t): return self.time_mlp(t)
class LDDiffuse(nn.Module): def init(self, Total=1000, Sample=5): super(LDDiffuse, self).init() # define a track head # self.time_embedding = Time_Embedding(cfg) ## set T self.total_timesteps = Total sampling_timesteps = Sample self.objective = 'pred_x0' ##prepare beta and alpha betas = cosine_beta_schedule(self.total_timesteps) # beta alphas = 1. - betas # 1-beta = alpha alphas_cumprod = torch.cumprod(alphas, dim=0) # (1-beta)! = alpha! = alpha bar alphas_cumprod_prev = F.pad(alphas_cumprod[:-1], (1, 0), value=1.) timesteps, = betas.shape
self.sampling_timesteps = default(sampling_timesteps, timesteps) assert self.sampling_timesteps <= timesteps self.is_ddim_sampling = self.sampling_timesteps < timesteps self.ddim_sampling_eta = 1. self.condition = True # self.scale = cfg.Diffusion.SNR_SCALE self.box_renewal = True self.use_ensemble = True self.register_buffer('betas', betas) self.register_buffer('alphas_cumprod', alphas_cumprod) self.register_buffer('alphas_cumprod_prev', alphas_cumprod_prev) self.register_buffer('sqrt_alphas_cumprod', torch.sqrt(alphas_cumprod)) self.register_buffer('sqrt_one_minus_alphas_cumprod', torch.sqrt(1. - alphas_cumprod)) self.register_buffer('log_one_minus_alphas_cumprod', torch.log(1. - alphas_cumprod)) self.register_buffer('sqrt_recip_alphas_cumprod', torch.sqrt(1. / alphas_cumprod)) self.register_buffer('sqrt_recipm1_alphas_cumprod', torch.sqrt(1. / alphas_cumprod - 1)) posterior_variance = betas * (1. - alphas_cumprod_prev) / (1. - alphas_cumprod) self.register_buffer('posterior_variance', posterior_variance) self.register_buffer('posterior_log_variance_clipped', torch.log(posterior_variance.clamp(min=1e-20))) self.register_buffer('posterior_mean_coef1', betas * torch.sqrt(alphas_cumprod_prev) / (1. - alphas_cumprod)) self.register_buffer('posterior_mean_coef2', (1. - alphas_cumprod_prev) * torch.sqrt(alphas) / (1. - alphas_cumprod)) self.num_timesteps = Total self.sample_timesteps = Sample self.set_loss('l2') self.unet = UNet(conditional=True) # self.unet = UViT(conditional=True) def set_loss(self, type): if type == 'l1': self.loss_func = nn.L1Loss(reduction='sum') elif type == 'l2': self.loss_func = nn.MSELoss(reduction='sum') else: raise NotImplementedError() def predict_noise_from_start(self, x_t, t, x0): return ( (extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - x0) / extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) ).float() # forward diffusion def q_sample(self, x_start, t, noise=None): if noise is None: noise = torch.randn_like(x_start) sqrt_alphas_cumprod_t = extract(self.sqrt_alphas_cumprod, t, x_start.shape) sqrt_one_minus_alphas_cumprod_t = extract(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) return (sqrt_alphas_cumprod_t * x_start + sqrt_one_minus_alphas_cumprod_t * noise).float(), sqrt_alphas_cumprod_t.float(), sqrt_one_minus_alphas_cumprod_t.float() # backward generation def p_sample(self, time, time_next, before, pred_x0, pred_noises): if time_next == -1: return before alpha = self.alphas_cumprod[time] alpha_next = self.alphas_cumprod[time_next] sigma = self.ddim_sampling_eta * ((1 - alpha / alpha_next) * (1 - alpha_next) / (1 - alpha)).sqrt() c = (1 - alpha_next - sigma ** 2).sqrt() noise = torch.randn_like(before) # x_start = pred[-1][:, 1, :, :] # pred_noise = self.predict_noise_from_start(before, time, pred[-1]) # predict_x0 = self.get_predict_x0(before, pred, [beta], [one_beta]) out = alpha_next.sqrt() * pred_x0 + sigma * noise + c * pred_noises # xt_next = alpha_next.sqrt() * pred + # out = pred[-1] * alpha_next.sqrt() + \ # c * pred_noise + \ # sigma * noise return out ## get the random t and noise for each sample within the batch def prepare_targets(self, num, template): ts = [] noises = [] for i in range(num): t = torch.randint(0, self.num_timesteps, (1,)).long().cuda() noise = torch.randn_like(template).cuda() ts.append(t) noises.append(noise) return ts, noises def get_predict_x0(self, xt, noises, betas): x0 = [(xt[i] - (1-betas[i]).sqrt() * noises[i]) / betas[i].sqrt() for i in range(len(betas))] return x0 def predict(self, fea_rgb, fea_tir): # get time steps times = torch.linspace(-1, self.num_timesteps - 1, self.sample_timesteps + 1).cuda() times = list(reversed(times.int().tolist())) # times = [999,2,-1] time_pairs = list(zip(times[:-1], times[1:])) # time_pairs = time_pairs[1:] ol = [] pl = [] xl = [] xt = torch.randn_like(fea_rgb).cuda() for time, time_next in time_pairs: # time_cond = torch.full((1,), time, dtype=torch.long).cuda() # time = 500 xl.append(xt) time = torch.full((1,), time, dtype=torch.long).cuda() beta = extract(self.alphas_cumprod, time, xt.shape).float() # one_beta = extract(self.sqrt_one_minus_alphas_cumprod, time, xt.shape).float() tem = torch.cat((torch.cat((xt, fea_rgb), dim=1), fea_tir), dim=1) out = self.unet(tem, beta, fea_rgb) ol.append(out) pred_n = self.predict_noise_from_start(xt, time, out) pl.append(pred_n) # predict_x0 = self.get_predict_x0(xt, out_noises, [beta])[-1] # xt = self.p_sample(time, time_next, xt, predict_x0, out_noises).float() if time_next < 0: # img = out_cls[:,1,:,:] return out else: xt = self.p_sample(time, time_next, xt, out, pred_n).float() # del out, pred_n # del out, pred_n # gc.collect() # att = self.diffusion_model.p_sample(time_cond, time_next, att, out_cls) def get_predicts(self, qs, ts, noises, betas): # out = [] # for i in range(qs.size(0)): out = self.get_predict_x0(qs, noises, betas) # out = torch.stack(out) out = [self.p_sample(ts[i], ts[i]-1, qs[i], out[i], noises[i]).float() for i in range(betas.size(0))] return torch.stack(out) def write_times(self, times): s = '' for time in times: s = s + str(time.item()) + ' ' s = s + '\n' def forward(self, fused_fea, fea_rgb, fea_tir): sample_num = fused_fea.size(0) # prepare ts, noises = self.prepare_targets(sample_num, fused_fea[0]) self.write_times(ts) qs = [] betas = [] one_betas = [] for i in range(sample_num): q, beta, one_beta = self.q_sample(fused_fea[i], ts[i], noises[i]) qs.append(q) betas.append(beta) one_betas.append(one_beta) if not self.condition: out = self.unet(qs, betas) else: qs = torch.stack(qs) betas = torch.stack(betas) con = torch.cat((fea_rgb, fea_tir), dim=1) out = self.unet(torch.cat((qs, con), dim=1), betas, fea_rgb) # self.loss_func.cuda() noises = torch.stack(noises) pred_n = [self.predict_noise_from_start(qs[i], ts[i], out[i]) for i in range(len(ts))] pred_n = torch.stack(pred_n) loss = self.loss_func(noises, pred_n).cuda() # predicted_x0s = self.get_predicts(qs, ts, out_noises, betas) return loss, out中的noise设计
- Assistant
The code you've shared appears to implement a class for a neural network model that performs diffusion-based generative modeling, potentially related to a denoising diffusion probabilistic model (DDPM) or similar frameworks. In the context of this model, noise design plays a crucial role in both training and inference phases. Let's break down the design and role of noise in this implementation:
Design and Role of Noise
-
Forward Diffusion Process (
q_samplefunction):- Purpose: During training, the forward diffusion process gradually adds noise to the data. The model is trained to predict this noise, eventually learning how to reverse the process to generate data.
- Noise Addition: In the
q_samplefunction, Gaussian noise is added to the input data (x_start) based on a timestep (t). The noise is generated usingtorch.randn_like(x_start), which creates a tensor of random numbers from a standard normal distribution, shaped likex_start. - Equation: The function calculates a noisy version of the input at time
tusing: [ q_t(x_t \mid x_0) = \sqrt{\alpha_t} \cdot x_0 + \sqrt{1 - \alpha_t} \cdot \text{noise} ] where\(\alpha_t\)is the cumulative product of ((1 - \beta_t)).
-
Noise Prediction (
predict_noise_from_startfunction):- Objective: The model is tasked with predicting the noise added at each timestep, rather than directly reconstructing the original data. This prediction is crucial for the denoising process during inference.
-
Reverse Process (Sampling):
- In the reverse diffusion process (
p_samplefunction), the model uses its predicted noise to iteratively denoise and reconstruct the data starting from pure noise: [ x_{t-1} = \sqrt{\alpha_{t-1}} \cdot \hat{x}_0 + \sigma \cdot \epsilon + c \cdot \hat{\epsilon}_t ] Here, (\hat{x}_0) is the estimated clean data, (\epsilon) is freshly drawn noise, and (\hat{\epsilon}_t) is the noise predicted by the model.
- In the reverse diffusion process (
-
Training Setup (prepare_targets function):
- Sampling Noise and Timestamps: During training, the
prepare_targetsfunction randomly selects time points (t) and generates Gaussian noise for each instance in the batch. This provides the noisy inputs and targets at various timesteps for training.
- Sampling Noise and Timestamps: During training, the
-
Loss Function:
- The model is trained using an L2 loss (
nn.MSELoss) between the predicted noise and the actual noise added during the forward process. Minimizing this loss helps the model learn the reverse process effectively.
- The model is trained using an L2 loss (
Conclusion: The design and use of noise in this model are integral to the diffusion process. By progressively injecting noise during training and learning to denoise, the model is able to generate new data from random noise. This stochastic nature is what allows diffusion models to generate diverse and high-quality outputs. The noise acts as both a training signal and a generative mechanism, enabling the model to synthesize data in a controlled manner.
-
- User
中文回答
- Assistant
该代码实现了一个基于扩散模型的神经网络,用于生成建模,可能与去噪扩散概率模型(DDPM)或类似框架相关。在这个模型中,噪声的设计在训练和推理阶段都扮演着重要角色。以下是对该实现中噪声设计和作用的中文解读:
噪声设计与作用
-
正向扩散过程 (
q_sample函数):- 目的: 在训练过程中,正向扩散过程逐渐向数据添加噪声。模型被训练去预测这些噪声,最终学习如何逆转该过程以生成数据。
- 噪声添加: 在
q_sample函数中,基于时间步长 (t) 向输入数据 (x_start) 添加高斯噪声。噪声通过torch.randn_like(x_start)生成,它创建了一个与x_start形状相同的标准正态分布的随机数张量。 - 公式: 该函数计算了在时间
t的噪声版本: [ q_t(x_t \mid x_0) = \sqrt{\alpha_t} \cdot x_0 + \sqrt{1 - \alpha_t} \cdot \text{noise} ] 其中\(\alpha_t\)是 ((1 - \beta_t)) 的累积乘积。
-
噪声预测 (
predict_noise_from_start函数):- 目标: 模型的任务不是直接重建原始数据,而是预测每个时间步添加的噪声。这个预测对于推理过程中的去噪至关重要。
-
逆向过程(采样):
- 在逆向扩散过程中(
p_sample函数),模型使用其预测的噪声从纯噪声开始迭代去噪和重建数据: [ x_{t-1} = \sqrt{\alpha_{t-1}} \cdot \hat{x}_0 + \sigma \cdot \epsilon + c \cdot \hat{\epsilon}_t ] 其中,(\hat{x}_0) 是估计的干净数据,(\epsilon) 是新抽取的噪声,而(\hat{\epsilon}_t) 是模型预测的噪声。
- 在逆向扩散过程中(
-
训练设置(
prepare_targets函数):- 采样噪声和时间戳: 在训练过程中,
prepare_targets函数为批处理中的每个实例随机选择时间点 (t) 并生成高斯噪声。这为训练提供了不同时间步下的噪声输入和目标。
- 采样噪声和时间戳: 在训练过程中,
-
损失函数:
- 模型使用预测噪声与前向过程实际添加噪声之间的 L2 损失(
nn.MSELoss)进行训练。最小化这个损失帮助模型有效学习逆向过程。
- 模型使用预测噪声与前向过程实际添加噪声之间的 L2 损失(
总结: 在该模型中,噪声的设计和使用是扩散过程的核心组成部分。通过在训练中逐步注入噪声并学习去噪,模型能够从随机噪声生成新的数据。这种随机性使得扩散模型能够控制性地产生多样且高质量的输出。噪声不仅作为训练信号,亦是生成机制,促使模型以可控方式合成数据。
-
- User
将噪声设计的部分单独提取出来
- Assistant
在扩散模型中,噪声设计对于模型的训练和数据生成具有重要意义。以下是噪声设计的部分单独提取和描述:
噪声设计
-
噪声生成:
- 在正向扩散过程中,噪声是通过
torch.randn_like(x_start)生成的。这一函数会根据输入x_start的形状创建一个标准正态分布(均值为0,方差为1)的随机数张量。噪声的作用是逐步扰乱数据,使模型学习如何从噪声信号中还原原始数据。
- 在正向扩散过程中,噪声是通过
-
噪声添加过程:
- 在
q_sample函数中,通过以下公式向数据添加噪声: [ q_t(x_t \mid x_0) = \sqrt{\alpha_t} \cdot x_0 + \sqrt{1 - \alpha_t} \cdot \text{noise} ] 其中,\(\alpha_t\)是通过(1-β)的累积乘积计算得到的变量,表示每一步添加噪声后的保持比例;1-\alpha_t则是噪声成分的比例。
- 在
-
噪声预测:
- 模型的核心任务是在各个时间步
t上预测加入的噪声。这种预测不仅帮助模型学习如何从噪声生成数据,也在逆向过程中指导去噪。
- 模型的核心任务是在各个时间步
-
逆向生成(推理)过程中的噪声使用:
- 在逆向生成流程中,模型同样会利用新生成的噪声来逐步减少图像中的噪声成分,从而复现原始数据: [ x_{t-1} = \sqrt{\alpha_{t-1}} \cdot \hat{x}_0 + \sigma \cdot \epsilon + c \cdot \hat{\epsilon}_t ] 其中,(\epsilon) 是从标准正态分布重新采样的噪声,帮助在生成过程中引入新的随机性,以获取多样化的输出。
总的来说,噪声在扩散模型中扮演了至关重要的双重角色:一方面作为训练信号,指导模型逐步学习去噪过程;另一方面作为生成机制的关键部分,引导模型在推理阶段从噪声中合成新的数据。
-
- User
我的意思是将代码提出出来
- Assistant
以下是从您的代码中提取出来的与噪声设计相关的部分:
import torch import torch.nn.functional as F from .utils import extract class LDDiffuse(nn.Module): def __init__(self, Total=1000): super(LDDiffuse, self).__init__() self.total_timesteps = Total # 创建扩散过程中的beta和alpha序列 betas = cosine_beta_schedule(self.total_timesteps) alphas = 1. - betas alphas_cumprod = torch.cumprod(alphas, dim=0) alphas_cumprod_prev = F.pad(alphas_cumprod[:-1], (1, 0), value=1.) # 注册这些变量为缓冲,避免在训练过程中被更改 self.register_buffer('betas', betas) self.register_buffer('alphas_cumprod', alphas_cumprod) self.register_buffer('alphas_cumprod_prev', alphas_cumprod_prev) self.register_buffer('sqrt_alphas_cumprod', torch.sqrt(alphas_cumprod)) self.register_buffer('sqrt_one_minus_alphas_cumprod', torch.sqrt(1. - alphas_cumprod)) self.register_buffer('sqrt_recip_alphas_cumprod', torch.sqrt(1. / alphas_cumprod)) self.register_buffer('sqrt_recipm1_alphas_cumprod', torch.sqrt(1. / alphas_cumprod - 1)) # 用于正向扩散过程的函数 def q_sample(self, x_start, t, noise=None): if noise is None: noise = torch.randn_like(x_start) sqrt_alphas_cumprod_t = extract(self.sqrt_alphas_cumprod, t, x_start.shape) sqrt_one_minus_alphas_cumprod_t = extract(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) return (sqrt_alphas_cumprod_t * x_start + sqrt_one_minus_alphas_cumprod_t * noise).float() # 用于预测噪声的函数 def predict_noise_from_start(self, x_t, t, x0): return ( (extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - x0) / extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) ).float()说明
-
betas和alphas: 这些参数用于定义扩散过程中的噪声力度。betas是预定义的噪声增益序列,alphas是 (1 - \beta),代表了每一时间步噪声减弱的比例。 -
缓冲区注册: 利用
self.register_buffer()保存模型中不需要梯度的固定参数。 -
q_sample: 该函数实现了向输入数据中添加噪声的过程。噪声是通过torch.randn_like(x_start)产生的随机高斯噪声。这个过程随着时间步t的推进,使输入数据逐渐被噪声污染。 -
predict_noise_from_start: 该函数用于从噪声图像预测原始无噪声图像所需的真实噪声。这是模型在训练中学习的核心任务。
这些模块组合在一起,形成了在正向扩散过程中对数据的逐步加噪声以及在反向生成过程中过滤噪声的机制。
-
- User
那么LDDiffuse的调用是如何?在哪里使用,且输入输出是什么?起到什么作用?
- Assistant
LDDiffuse类在代码中实现了一个基于扩散模型的组件,用于增量地添加噪声并反向去噪来生成数据。以下是该类的调用方式、使用场景、输入输出及其作用的详细说明:使用场景
LDDiffuse类主要用于以扩散模型的方法来生成数据,通常用于生成模型任务中,尤其是那些需要递归去噪的场景,比如生成图像、声音或其他需要重构信号的领域。调用方式
-
初始化:
在创建
LDDiffuse对象时,您需要指定总的时间步数,总时间步数会影响扩散过程中的噪声调度和生成质量。ld_diffuse = LDDiffuse(Total=1000)在实际应用中,
Total参数可以根据需求调整。常见的时间步数为 1000,但根据实际任务复杂性和计算资源,可以选择不同的值。 -
正向扩散(噪声添加):
在训练阶段,可以使用
q_sample方法实现数据的正向扩散,从而生成带噪声的数据样本,用于训练模型去预测噪声。x_noisy, sqrt_alphas_cumprod_t, sqrt_one_minus_alphas_cumprod_t = ld_diffuse.q_sample(x_start, t)- 输入:
x_start: 原始的干净数据(通常是图像或信号)。t: 时间步张量(通常以整型标识当前“扩散时间”)。
- 输出:
x_noisy: 添加噪声后的数据。
- 输入:
-
反向去噪(噪声预测与去除):
在生成阶段或训练步骤的目标函数计算时,需要预测噪声。可以使用以下的噪声预测函数:
predicted_noise = ld_diffuse.predict_noise_from_start(x_t, t, x0)- 输入:
x_t: 当前时间步带有噪声的数据。t: 时间步张量。x0: 原始数据或模型预测的清晰数据。
- 输出:
predicted_noise: 预测的加入噪声。
- 输入:
作用解释
-
数据扰动和合成: 通过噪声注入打乱数据分布,并通过神经网络学习去噪过程,使得模型在反向生成时可以合成新的数据样本。
-
模型训练: 在训练过程中,通过模拟正向扩散过程来生成训练数据,并通过预测噪声来训练模型。
-
生成任务: 在推理阶段,利用训练好的模型,通过迭代去除噪声逐步生成新的、高质量的数据。
总之,
LDDiffuse在扩散模型架构中起到了加噪和去噪的双重作用,通过这种机制,模型可以在构造性生成任务中实现高效的数据建模和生成。 -
- User
那么在 # Copyright (c) 2015-present, Facebook, Inc.
All rights reserved.
####################################### 等同于 vit_ce ############################# import torch import torch.nn as nn from functools import partial from torch import Tensor from typing import Optional
from timm.models.vision_transformer import VisionTransformer, cfg from timm.models.registry import register_model from timm.models.layers import trunc_normal, lecun_normal_
from timm.models.layers import DropPath, to_2tuple from timm.models.vision_transformer import _load_weights
import math
from collections import namedtuple
from mamba_ssm.modules.mamba_simple import Mamba from mamba_ssm.utils.generation import GenerationMixin from mamba_ssm.utils.hf import load_config_hf, load_state_dict_hf #################### 结合当前项目引入 ######################### from lib.models.mamba_fetrack.rope import * import random from lib.models.layers.head import build_box_head import importlib import lib.train.admin.settings as ws_settings from .utils import combine_tokens, recover_tokens ################################################################# try: from mamba_ssm.ops.triton.layernorm import RMSNorm, layer_norm_fn, rms_norm_fn except ImportError: RMSNorm, layer_norm_fn, rms_norm_fn = None, None, None
all = [ 'vim_tiny_patch16_224', 'vim_small_patch16_224', 'vim_base_patch16_224', 'vim_tiny_patch16_384', 'vim_small_patch16_384', 'vim_base_patch16_384', ]
class PatchEmbed(nn.Module): """ 2D Image to Patch Embedding """ def init(self, img_size=224, patch_size=16, stride=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True): super().init() img_size = to_2tuple(img_size) patch_size = to_2tuple(patch_size) self.img_size = img_size self.patch_size = patch_size self.grid_size = ((img_size[0] - patch_size[0]) // stride + 1, (img_size[1] - patch_size[1]) // stride + 1) self.num_patches = self.grid_size[0] * self.grid_size[1] self.flatten = flatten
self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=stride) self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() def forward(self, x): B, C, H, W = x.shape # assert H == self.img_size[0] and W == self.img_size[1], \ # f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." x = self.proj(x) if self.flatten: x = x.flatten(2).transpose(1, 2) # BCHW -> BNC x = self.norm(x) return xclass Block(nn.Module): def init( self, dim, mixer_cls, norm_cls=nn.LayerNorm, fused_add_norm=False, residual_in_fp32=False,drop_path=0., ): """ Simple block wrapping a mixer class with LayerNorm/RMSNorm and residual connection"
This Block has a slightly different structure compared to a regular prenorm Transformer block. The standard block is: LN -> MHA/MLP -> Add. [Ref: https://arxiv.org/abs/2002.04745] Here we have: Add -> LN -> Mixer, returning both the hidden_states (output of the mixer) and the residual. This is purely for performance reasons, as we can fuse add and LayerNorm. The residual needs to be provided (except for the very first block). """ super().__init__() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.mixer = mixer_cls(dim) self.norm = norm_cls(dim) self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() if self.fused_add_norm: assert RMSNorm is not None, "RMSNorm import fails" assert isinstance( self.norm, (nn.LayerNorm, RMSNorm) ), "Only LayerNorm and RMSNorm are supported for fused_add_norm" def forward( self, hidden_states: Tensor, residual: Optional[Tensor] = None, inference_params=None ): r"""Pass the input through the encoder layer. Args: hidden_states: the sequence to the encoder layer (required). residual: hidden_states = Mixer(LN(residual)) """ if not self.fused_add_norm: if residual is None: residual = hidden_states else: residual = residual + self.drop_path(hidden_states) hidden_states = self.norm(residual.to(dtype=self.norm.weight.dtype)) if self.residual_in_fp32: residual = residual.to(torch.float32) else: fused_add_norm_fn = rms_norm_fn if isinstance(self.norm, RMSNorm) else layer_norm_fn if residual is None: hidden_states, residual = fused_add_norm_fn( hidden_states, self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) else: hidden_states, residual = fused_add_norm_fn( self.drop_path(hidden_states), self.norm.weight, self.norm.bias, residual=residual, prenorm=True, residual_in_fp32=self.residual_in_fp32, eps=self.norm.eps, ) hidden_states = self.mixer(hidden_states, inference_params=inference_params) return hidden_states, residual def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return self.mixer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs)def create_block( d_model, ssm_cfg=None, norm_epsilon=1e-5, drop_path=0., rms_norm=False, residual_in_fp32=False, fused_add_norm=False, layer_idx=None, device=None, dtype=None, if_bimamba=False, bimamba_type="none", if_devide_out=False, init_layer_scale=None, ): if if_bimamba: bimamba_type = "v1" if ssm_cfg is None: ssm_cfg = {} factory_kwargs = {"device": device, "dtype": dtype} mixer_cls = partial(Mamba, layer_idx=layer_idx, bimamba_type=bimamba_type, if_devide_out=if_devide_out, init_layer_scale=init_layer_scale, **ssm_cfg, **factory_kwargs) norm_cls = partial( nn.LayerNorm if not rms_norm else RMSNorm, eps=norm_epsilon, **factory_kwargs ) block = Block( d_model, mixer_cls, norm_cls=norm_cls, drop_path=drop_path, fused_add_norm=fused_add_norm, residual_in_fp32=residual_in_fp32, ) block.layer_idx = layer_idx return block
https://github.com/huggingface/transformers/blob/c28d04e9e252a1a099944e325685f14d242ecdcd/src/transformers/models/gpt2/modeling_gpt2.py#L454
def init_weights( module, n_layer, initializer_range=0.02, # Now only used for embedding layer. rescale_prenorm_residual=True, n_residuals_per_layer=1, # Change to 2 if we have MLP ): if isinstance(module, nn.Linear): if module.bias is not None: if not getattr(module.bias, "no_reinit", False): nn.init.zeros(module.bias) elif isinstance(module, nn.Embedding): nn.init.normal(module.weight, std=initializer_range)
if rescale_prenorm_residual: # Reinitialize selected weights subject to the OpenAI GPT-2 Paper Scheme: # > A modified initialization which accounts for the accumulation on the residual path with model depth. Scale # > the weights of residual layers at initialization by a factor of 1/√N where N is the # of residual layers. # > -- GPT-2 :: https://openai.com/blog/better-language-models/ # # Reference (Megatron-LM): https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/model/gpt_model.py for name, p in module.named_parameters(): if name in ["out_proj.weight", "fc2.weight"]: # Special Scaled Initialization --> There are 2 Layer Norms per Transformer Block # Following Pytorch init, except scale by 1/sqrt(2 * n_layer) # We need to reinit p since this code could be called multiple times # Having just p *= scale would repeatedly scale it down nn.init.kaiming_uniform_(p, a=math.sqrt(5)) with torch.no_grad(): p /= math.sqrt(n_residuals_per_layer * n_layer)def segm_init_weights(m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=0.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, nn.Conv2d): # NOTE conv was left to pytorch default in my original init lecun_normal_(m.weight) if m.bias is not None: nn.init.zeros_(m.bias) elif isinstance(m, (nn.LayerNorm, nn.GroupNorm, nn.BatchNorm2d)): nn.init.zeros_(m.bias) nn.init.ones_(m.weight)
class VisionMamba(nn.Module): def init(self, img_size=224, patch_size=16, stride=16, depth=24, embed_dim=192, channels=3, num_classes=1000, ssm_cfg=None, drop_rate=0., drop_path_rate=0.1, norm_epsilon: float = 1e-5, rms_norm: bool = False, initializer_cfg=None, fused_add_norm=False, residual_in_fp32=False, device=None, dtype=None, ft_seq_len=None, pt_hw_seq_len=14, if_bidirectional=False, final_pool_type='none', if_abs_pos_embed=False, if_rope=False, if_rope_residual=False, flip_img_sequences_ratio=-1., if_bimamba=False, bimamba_type="none", if_cls_token=False, if_devide_out=False, init_layer_scale=None, use_double_cls_token=False, use_middle_cls_token=False, **kwargs): factory_kwargs = {"device": device, "dtype": dtype} # add factory_kwargs into kwargs kwargs.update(factory_kwargs) super().init() self.residual_in_fp32 = residual_in_fp32 self.fused_add_norm = fused_add_norm self.if_bidirectional = if_bidirectional self.final_pool_type = final_pool_type self.if_abs_pos_embed = if_abs_pos_embed self.if_rope = if_rope self.if_rope_residual = if_rope_residual self.flip_img_sequences_ratio = flip_img_sequences_ratio # self.if_cls_token = if_cls_token (此处与Mamba源代码不同) self.if_cls_token = False self.use_double_cls_token = use_double_cls_token self.use_middle_cls_token = use_middle_cls_token self.num_tokens = 1 if if_cls_token else 0
# pretrain parameters self.num_classes = num_classes self.d_model = self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models self.patch_embed = PatchEmbed( img_size=img_size, patch_size=patch_size, stride=stride, in_chans=channels, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches if if_cls_token: if use_double_cls_token: self.cls_token_head = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) self.cls_token_tail = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) self.num_tokens = 2 else: self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) # self.num_tokens = 1 if if_abs_pos_embed: # self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, self.embed_dim)) # 此处与mamba不同,划分了x和z的位置编码 self.pos_embed_x = nn.Parameter(torch.zeros(1, 256, self.embed_dim)) self.pos_embed_z = nn.Parameter(torch.zeros(1, 64, self.embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) if if_rope: half_head_dim = embed_dim // 2 hw_seq_len = img_size // patch_size self.rope = VisionRotaryEmbeddingFast( dim=half_head_dim, pt_seq_len=pt_hw_seq_len, ft_seq_len=hw_seq_len ) # 此处将head去掉了。 # self.head = nn.Linear(self.num_features, num_classes) if num_classes > 0 else nn.Identity() # settings = ws_settings.Settings() # config_module = importlib.import_module("lib.config.%s.config" % settings.script_name) # cfg = config_module.cfg # self.head = build_box_head(cfg, embed_dim) # TODO: release this comment dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule # import ipdb;ipdb.set_trace() inter_dpr = [0.0] + dpr self.drop_path = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity() # transformer blocks self.layers = nn.ModuleList( [ create_block( embed_dim, ssm_cfg=ssm_cfg, norm_epsilon=norm_epsilon, rms_norm=rms_norm, residual_in_fp32=residual_in_fp32, fused_add_norm=fused_add_norm, layer_idx=i, if_bimamba=if_bimamba, bimamba_type=bimamba_type, drop_path=inter_dpr[i], if_devide_out=if_devide_out, init_layer_scale=init_layer_scale, **factory_kwargs, ) for i in range(depth) ] ) # output head self.norm_f = (nn.LayerNorm if not rms_norm else RMSNorm)( embed_dim, eps=norm_epsilon, **factory_kwargs ) # self.pre_logits = nn.Identity() # original init self.patch_embed.apply(segm_init_weights) # 此处与mamba不同,问题在于没有执行head相关操作 # self.head.apply(segm_init_weights) # 在pos_embed中也进行了模板和搜索的划分 if if_abs_pos_embed: trunc_normal_(self.pos_embed_x, std=.02) trunc_normal_(self.pos_embed_z, std=.02) if if_cls_token: if use_double_cls_token: trunc_normal_(self.cls_token_head, std=.02) trunc_normal_(self.cls_token_tail, std=.02) else: trunc_normal_(self.cls_token, std=.02) # mamba init self.apply( partial( _init_weights, n_layer=depth, **(initializer_cfg if initializer_cfg is not None else {}), ) ) def allocate_inference_cache(self, batch_size, max_seqlen, dtype=None, **kwargs): return { i: layer.allocate_inference_cache(batch_size, max_seqlen, dtype=dtype, **kwargs) for i, layer in enumerate(self.layers) } @torch.jit.ignore def no_weight_decay(self): return {"pos_embed", "cls_token", "dist_token", "cls_token_head", "cls_token_tail"} @torch.jit.ignore() def load_pretrained(self, checkpoint_path, prefix=""): _load_weights(self, checkpoint_path, prefix) def forward_features(self, z, x, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False): # taken from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py # with slight modifications to add the dist_token x = self.patch_embed(x) #x.shape = torch.Size([B, 3, 256, 256]) -> torch.Size([2, 256, 384]) z = self.patch_embed(z) #z.shape = torch.Size([B, 3, 128, 128]) -> torch.Size([2, 64, 384]) B, M, _ = x.shape if self.if_cls_token: # False if self.use_double_cls_token: cls_token_head = self.cls_token_head.expand(B, -1, -1) cls_token_tail = self.cls_token_tail.expand(B, -1, -1) token_position = [0, M + 1] x = torch.cat((cls_token_head, x, cls_token_tail), dim=1) M = x.shape[1] else: if self.use_middle_cls_token: cls_token = self.cls_token.expand(B, -1, -1) token_position = M // 2 # add cls token in the middle x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1) elif if_random_cls_token_position: cls_token = self.cls_token.expand(B, -1, -1) token_position = random.randint(0, M) x = torch.cat((x[:, :token_position, :], cls_token, x[:, token_position:, :]), dim=1) print("token_position: ", token_position) else: cls_token = self.cls_token.expand(B, -1, -1) # stole cls_tokens impl from Phil Wang, thanks token_position = 0 x = torch.cat((cls_token, x), dim=1) M = x.shape[1] # 此处由于划分了模板和搜索,pos_embed也需要单独执行 if self.if_abs_pos_embed: # True x = x + self.pos_embed_x # x = x + positon_embemding =torch.Size([B, 256, 384]) + torch.Size([1, 256, 384]) = torch.Size([B, 256, 384]) z = z + self.pos_embed_z # z = z + positon_embemding =torch.Size([B, 64, 384]) + torch.Size([1, 64, 384]) = torch.Size([B, 64, 384]) x = torch.cat((z, x), dim=1) # torch.Size([B, 320, 384]) x = self.pos_drop(x) # x.shape = torch.Size([B, 320, 384]) if if_random_token_rank: #False # 生成随机 shuffle 索引 shuffle_indices = torch.randperm(M) if isinstance(token_position, list): print("original value: ", x[0, token_position[0], 0], x[0, token_position[1], 0]) else: print("original value: ", x[0, token_position, 0]) print("original token_position: ", token_position) # 执行 shuffle x = x[:, shuffle_indices, :] if isinstance(token_position, list): # 找到 cls token 在 shuffle 之后的新位置 new_token_position = [torch.where(shuffle_indices == token_position[i])[0].item() for i in range(len(token_position))] token_position = new_token_position else: # 找到 cls token 在 shuffle 之后的新位置 token_position = torch.where(shuffle_indices == token_position)[0].item() if isinstance(token_position, list): print("new value: ", x[0, token_position[0], 0], x[0, token_position[1], 0]) else: print("new value: ", x[0, token_position, 0]) print("new token_position: ", token_position) if_flip_img_sequences = False if self.flip_img_sequences_ratio > 0 and (self.flip_img_sequences_ratio - random.random()) > 1e-5: # False x = x.flip([1]) if_flip_img_sequences = True # mamba impl residual = None hidden_states = x if not self.if_bidirectional: # True for layer in self.layers: if if_flip_img_sequences and self.if_rope: # False hidden_states = hidden_states.flip([1]) if residual is not None: residual = residual.flip([1]) # rope about if self.if_rope: # False hidden_states = self.rope(hidden_states) if residual is not None and self.if_rope_residual: residual = self.rope(residual) if if_flip_img_sequences and self.if_rope: # False hidden_states = hidden_states.flip([1]) if residual is not None: residual = residual.flip([1]) hidden_states, residual = layer( hidden_states, residual, inference_params=inference_params ) else: # False # get two layers in a single for-loop for i in range(len(self.layers) // 2): if self.if_rope: hidden_states = self.rope(hidden_states) if residual is not None and self.if_rope_residual: residual = self.rope(residual) hidden_states_f, residual_f = self.layers[i * 2]( hidden_states, residual, inference_params=inference_params ) hidden_states_b, residual_b = self.layers[i * 2 + 1]( hidden_states.flip([1]), None if residual == None else residual.flip([1]), inference_params=inference_params ) hidden_states = hidden_states_f + hidden_states_b.flip([1]) residual = residual_f + residual_b.flip([1]) if not self.fused_add_norm: #False if residual is None: residual = hidden_states else: residual = residual + self.drop_path(hidden_states) hidden_states = self.norm_f(residual.to(dtype=self.norm_f.weight.dtype)) else: #True # Set prenorm=False here since we don't need the residual fused_add_norm_fn = rms_norm_fn if isinstance(self.norm_f, RMSNorm) else layer_norm_fn hidden_states = fused_add_norm_fn( # hidden_states.shape = torch.Size([B, 320, 384]) self.drop_path(hidden_states), self.norm_f.weight, self.norm_f.bias, eps=self.norm_f.eps, residual=residual, prenorm=False, residual_in_fp32=self.residual_in_fp32, ) # return only cls token if it exists if self.if_cls_token: #False if self.use_double_cls_token: return (hidden_states[:, token_position[0], :] + hidden_states[:, token_position[1], :]) / 2 else: if self.use_middle_cls_token: return hidden_states[:, token_position, :] elif if_random_cls_token_position: return hidden_states[:, token_position, :] else: return hidden_states[:, token_position, :] if self.final_pool_type == 'none': return hidden_states[:, -1, :] elif self.final_pool_type == 'mean': #True return hidden_states.mean(dim=1) #hidden_states.shape = torch.Size([2, 384]) elif self.final_pool_type == 'max': return hidden_states elif self.final_pool_type == 'all': return hidden_states else: raise NotImplementedError def forward(self, x, return_features=False, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False): x = self.forward_features(x, inference_params, if_random_cls_token_position=if_random_cls_token_position, if_random_token_rank=if_random_token_rank) if return_features: return x x = self.head(x) if self.final_pool_type == 'max': x = x.max(dim=1)[0] return x@register_model def vim_tiny_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model
@register_model def vim_tiny_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=192, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model
@register_model def vim_small_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='all', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg()
if pretrained: # checkpoint = torch.hub.load_state_dict_from_url( # url="to.do", # map_location="cpu", check_hash=True # ) # model.load_state_dict(checkpoint["model"]) checkpoint = torch.load(pretrained, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False) print('Load pretrained model from: ' + pretrained) return model@register_model def vim_small_patch16_stride8_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2(pretrained=False, **kwargs): model = VisionMamba( patch_size=16, stride=8, embed_dim=384, depth=24, rms_norm=True, residual_in_fp32=True, fused_add_norm=True, final_pool_type='mean', if_abs_pos_embed=True, if_rope=False, if_rope_residual=False, bimamba_type="v2", if_cls_token=True, if_devide_out=True, use_middle_cls_token=True, **kwargs) model.default_cfg = _cfg() if pretrained: checkpoint = torch.hub.load_state_dict_from_url( url="to.do", map_location="cpu", check_hash=True ) model.load_state_dict(checkpoint["model"]) return model 和 """ Basic mamda_fetrack model. """ import math import os from typing import List
import torch from torch import nn from torch.nn.modules.transformer import _get_clones
from lib.models.layers.head import build_box_head from lib.utils.box_ops import box_xyxy_to_cxcywh from lib.models.mamba_fetrack.models_mamba import create_block from timm.models import create_model import torch.nn.functional as F from lib.models.mamba_fetrack.mamba_cross import CrossMamba from thop import profile
class Mamba_FEtrack(nn.Module): """ This is the base class for mamda_fetrack """
def __init__(self, visionmamba, cross_mamba, box_head, aux_loss=False, head_type="CORNER"): """ Initializes the model. Parameters: transformer: torch module of the transformer architecture. aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. """ super().__init__() self.backbone = visionmamba self.cross_mamba= cross_mamba self.box_head = box_head self.aux_loss = aux_loss self.head_type = head_type if head_type == "CORNER" or head_type == "CENTER": self.feat_sz_s = int(box_head.feat_sz) self.feat_len_s = int(box_head.feat_sz ** 2) if self.aux_loss: self.box_head = _get_clones(self.box_head, 6) def forward(self, template: torch.Tensor, search: torch.Tensor, event_template: torch.Tensor, event_search: torch.Tensor, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False, ): rgb_feature = self.backbone.forward_features( z=template, x=search, #[B, 320, 384] inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False) event_feature = self.backbone.forward_features(z=event_template, x=event_search, #[B, 320, 384] inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False) residual_event_f = 0 residual_rgb_f = 0 event_f = self.cross_mamba(event_feature,residual_event_f,rgb_feature) + event_feature rgb_f = self.cross_mamba(rgb_feature,residual_rgb_f,event_feature) + rgb_feature event_searh = event_f[:, -self.feat_len_s:] rgb_search = rgb_f[:, -self.feat_len_s:] x = torch.cat((event_searh,rgb_search),dim=-1) feat_last = x if isinstance(x, list): feat_last = x[-1] out = self.forward_head(feat_last, None) out['backbone_feat'] = x return out def forward_head(self, cat_feature, gt_score_map=None): """ cat_feature: output embeddings of the backbone, it can be (HW1+HW2, B, C) or (HW2, B, C) """ search_feature = cat_feature opt = (search_feature.unsqueeze(-1)).permute((0, 3, 2, 1)).contiguous() # opt.shape = torch.Size([B, 1, 384, 256]) bs, Nq, C, HW = opt.size() opt_feat = opt.view(-1, C, self.feat_sz_s, self.feat_sz_s) # opt_feat.shape = torch.Size([B, 384, 16, 16]) if self.head_type == "CORNER": # run the corner head pred_box, score_map = self.box_head(opt_feat, True) outputs_coord = box_xyxy_to_cxcywh(pred_box) outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map, } return out elif self.head_type == "CENTER": # run the center head score_map_ctr, bbox, size_map, offset_map = self.box_head(opt_feat, gt_score_map) outputs_coord = bbox outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map_ctr, 'size_map': size_map, 'offset_map': offset_map} return out else: raise NotImplementedErrordef build_mamba_fetrack(cfg, training=True): current_dir = os.path.dirname(os.path.abspath(file)) # This is your Project Root pretrained_path = os.path.join(current_dir, 'pretrained_models') # pretrained_path = os.path.join(current_dir, '../../../pretrained_models') if cfg.MODEL.PRETRAIN_FILE and ('Mamba_FETrack' not in cfg.MODEL.PRETRAIN_FILE) and training: pretrained = os.path.join(pretrained_path, cfg.MODEL.PRETRAIN_FILE) else: pretrained = ''
backbone = create_model( model_name= cfg.MODEL.BACKBONE.TYPE, pretrained= pretrained, num_classes=1000, drop_rate=0.0, drop_path_rate=cfg.TRAIN.DROP_PATH_RATE, drop_block_rate=None, img_size=256 ) hidden_dim = 384 cross_mamba = CrossMamba(hidden_dim) box_head = build_box_head(cfg, hidden_dim*2) model = Mamba_FEtrack( backbone, cross_mamba, box_head, aux_loss=False, head_type=cfg.MODEL.HEAD.TYPE, ) if 'Mamba_FETrack' in cfg.MODEL.PRETRAIN_FILE and training: checkpoint = torch.load(cfg.MODEL.PRETRAIN_FILE, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["net"], strict=False) print('Load pretrained model from: ' + cfg.MODEL.PRETRAIN_FILE) return model中是否可以集成LDDiffuse?起到什么作用?
- Assistant
要将
LDDiffuse集成到您描述的现有代码中,并让它起作用,我们需要了解该项目的当前结构和目的。一般而言,LDDiffuse的集成可能涉及以下几个步骤和注意事项:可能的集成场景
-
数据前处理或增强:
-
目的: 您可以在模型的输入数据上使用
LDDiffuse来生成不同程度的噪声版本。这适合用于数据增强,增强模型的泛化能力。 -
实现: 在输入(例如,模板和搜索区域)进入模型前,使用
q_sample将数据变换为带有噪声的数据。noisy_template, _, _ = ld_diffuse.q_sample(template, t) noisy_search, _, _ = ld_diffuse.q_sample(search, t) # 然后使用这些噪声数据进行训练
-
-
特征生成和增强:
-
目的: 在
VisionMamba生成的特征后,加入噪声生成过程。类似于对特征进行扰动,以通过噪声学习使模型更具鲁棒性。 -
实现: 使用从
forward_features生成的特征,再应用LDDiffuse。features = self.backbone.forward_features(...) noisy_features, _, _ = ld_diffuse.q_sample(features, t)
-
-
对比学习与复合损失:
-
目的: 使用
LDDiffuse生成的噪声特征,用于对比学习的负样本或复合损失函数的构建中。 -
实现: 可在损失计算部分引入,如构造损失来最小化去除噪声后的特征与目标特征之间的距离。
-
具体集成步骤
-
添加
LDDiffuse组件: 在现有代码的适当位置初始化LDDiffuse,如在Mamba_FEtrack类的构造函数中进行初始化。self.ld_diffuse = LDDiffuse(Total=1000) -
选择合适的集成位置:
-
在输入数据: 可以在输入数据模板和搜索区域上使用,在
forward方法内处理输入。 -
在特征生成: 可以在特征生成后即
forward_features的发出后。
-
-
修改
forward方法或其他相关函数:- 将
LDDiffuse噪声采样过程嵌入到forward方法中,使输入或特征带有噪声。
- 将
可能的作用
-
鲁棒性增强: 通过对输入或特征的噪声扰动,使模型学习到更稳健的特征表达。
-
数据多样性: 噪声注入增加了数据样本的多样性,提高模型的泛化能力。
-
特征对齐: 通过学习去噪特征,有助于更好地对齐不同模态的数据表示。
需要注意的是集成后需通过实验验证其效果,具体表现会因任务目标和数据特点而异。
-
- User
原始的噪声设计及调用是:import os import torch import torch.nn as nn import torch.nn.functional as F import math from .utils import cosine_beta_schedule, default, extract from .UNet import UNet from .UViT import UViT
from SiamB.core.defaults import cfg
import matplotlib.pyplot as plt import gc def get_l2(f): c, h, w = f.shape mean = torch.mean(f, dim=0).unsqueeze(0).repeat(c, 1, 1) f = torch.sum(torch.pow((f - mean), 2), dim=0) / c f = f return (f - f.min()) / (f.max() - f.min())
def draw(f, name): plt.imshow(get_l2(f).detach().cpu().numpy()) plt.title(name) plt.show()
class SinusoidalPositionEmbeddings(nn.Module): def init(self, dim): super().init() self.dim = dim
def forward(self, time): # device = time.device half_dim = self.dim // 2 embeddings = math.log(10000) / (half_dim - 1) embeddings = torch.exp(torch.arange(half_dim).cuda() * -embeddings) embeddings = time[:, None] * embeddings[None, :] embeddings = torch.cat((embeddings.sin(), embeddings.cos()), dim=-1) return embeddingsclass Time_Embedding(nn.Module): def init(self, cfg): super(Time_Embedding, self).init() self.time_mlp = nn.Sequential( SinusoidalPositionEmbeddings(int(cfg.Time_Embedding.Dim / 4)), nn.Linear(int(cfg.Time_Embedding.Dim / 4), cfg.Time_Embedding.Dim), nn.GELU(), nn.Linear(cfg.Time_Embedding.Dim, cfg.Time_Embedding.Dim), ) def forward(self, t): return self.time_mlp(t)
class LDDiffuse(nn.Module): def init(self, Total=1000, Sample=5): super(LDDiffuse, self).init() # define a track head # self.time_embedding = Time_Embedding(cfg) ## set T self.total_timesteps = Total sampling_timesteps = Sample self.objective = 'pred_x0' ##prepare beta and alpha betas = cosine_beta_schedule(self.total_timesteps) # beta alphas = 1. - betas # 1-beta = alpha alphas_cumprod = torch.cumprod(alphas, dim=0) # (1-beta)! = alpha! = alpha bar alphas_cumprod_prev = F.pad(alphas_cumprod[:-1], (1, 0), value=1.) timesteps, = betas.shape
self.sampling_timesteps = default(sampling_timesteps, timesteps) assert self.sampling_timesteps <= timesteps self.is_ddim_sampling = self.sampling_timesteps < timesteps self.ddim_sampling_eta = 1. self.condition = True # self.scale = cfg.Diffusion.SNR_SCALE self.box_renewal = True self.use_ensemble = True self.register_buffer('betas', betas) self.register_buffer('alphas_cumprod', alphas_cumprod) self.register_buffer('alphas_cumprod_prev', alphas_cumprod_prev) self.register_buffer('sqrt_alphas_cumprod', torch.sqrt(alphas_cumprod)) self.register_buffer('sqrt_one_minus_alphas_cumprod', torch.sqrt(1. - alphas_cumprod)) self.register_buffer('log_one_minus_alphas_cumprod', torch.log(1. - alphas_cumprod)) self.register_buffer('sqrt_recip_alphas_cumprod', torch.sqrt(1. / alphas_cumprod)) self.register_buffer('sqrt_recipm1_alphas_cumprod', torch.sqrt(1. / alphas_cumprod - 1)) posterior_variance = betas * (1. - alphas_cumprod_prev) / (1. - alphas_cumprod) self.register_buffer('posterior_variance', posterior_variance) self.register_buffer('posterior_log_variance_clipped', torch.log(posterior_variance.clamp(min=1e-20))) self.register_buffer('posterior_mean_coef1', betas * torch.sqrt(alphas_cumprod_prev) / (1. - alphas_cumprod)) self.register_buffer('posterior_mean_coef2', (1. - alphas_cumprod_prev) * torch.sqrt(alphas) / (1. - alphas_cumprod)) self.num_timesteps = Total self.sample_timesteps = Sample self.set_loss('l2') self.unet = UNet(conditional=True) # self.unet = UViT(conditional=True) def set_loss(self, type): if type == 'l1': self.loss_func = nn.L1Loss(reduction='sum') elif type == 'l2': self.loss_func = nn.MSELoss(reduction='sum') else: raise NotImplementedError() def predict_noise_from_start(self, x_t, t, x0): return ( (extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - x0) / extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) ).float() # forward diffusion def q_sample(self, x_start, t, noise=None): if noise is None: noise = torch.randn_like(x_start) sqrt_alphas_cumprod_t = extract(self.sqrt_alphas_cumprod, t, x_start.shape) sqrt_one_minus_alphas_cumprod_t = extract(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) return (sqrt_alphas_cumprod_t * x_start + sqrt_one_minus_alphas_cumprod_t * noise).float(), sqrt_alphas_cumprod_t.float(), sqrt_one_minus_alphas_cumprod_t.float() # backward generation def p_sample(self, time, time_next, before, pred_x0, pred_noises): if time_next == -1: return before alpha = self.alphas_cumprod[time] alpha_next = self.alphas_cumprod[time_next] sigma = self.ddim_sampling_eta * ((1 - alpha / alpha_next) * (1 - alpha_next) / (1 - alpha)).sqrt() c = (1 - alpha_next - sigma ** 2).sqrt() noise = torch.randn_like(before) # x_start = pred[-1][:, 1, :, :] # pred_noise = self.predict_noise_from_start(before, time, pred[-1]) # predict_x0 = self.get_predict_x0(before, pred, [beta], [one_beta]) out = alpha_next.sqrt() * pred_x0 + sigma * noise + c * pred_noises # xt_next = alpha_next.sqrt() * pred + # out = pred[-1] * alpha_next.sqrt() + \ # c * pred_noise + \ # sigma * noise return out ## get the random t and noise for each sample within the batch def prepare_targets(self, num, template): ts = [] noises = [] for i in range(num): t = torch.randint(0, self.num_timesteps, (1,)).long().cuda() noise = torch.randn_like(template).cuda() ts.append(t) noises.append(noise) return ts, noises def get_predict_x0(self, xt, noises, betas): x0 = [(xt[i] - (1-betas[i]).sqrt() * noises[i]) / betas[i].sqrt() for i in range(len(betas))] return x0 def predict(self, fea_rgb, fea_tir): # get time steps times = torch.linspace(-1, self.num_timesteps - 1, self.sample_timesteps + 1).cuda() times = list(reversed(times.int().tolist())) # times = [999,2,-1] time_pairs = list(zip(times[:-1], times[1:])) # time_pairs = time_pairs[1:] ol = [] pl = [] xl = [] xt = torch.randn_like(fea_rgb).cuda() for time, time_next in time_pairs: # time_cond = torch.full((1,), time, dtype=torch.long).cuda() # time = 500 xl.append(xt) time = torch.full((1,), time, dtype=torch.long).cuda() beta = extract(self.alphas_cumprod, time, xt.shape).float() # one_beta = extract(self.sqrt_one_minus_alphas_cumprod, time, xt.shape).float() tem = torch.cat((torch.cat((xt, fea_rgb), dim=1), fea_tir), dim=1) out = self.unet(tem, beta, fea_rgb) ol.append(out) pred_n = self.predict_noise_from_start(xt, time, out) pl.append(pred_n) # predict_x0 = self.get_predict_x0(xt, out_noises, [beta])[-1] # xt = self.p_sample(time, time_next, xt, predict_x0, out_noises).float() if time_next < 0: # img = out_cls[:,1,:,:] return out else: xt = self.p_sample(time, time_next, xt, out, pred_n).float() # del out, pred_n # del out, pred_n # gc.collect() # att = self.diffusion_model.p_sample(time_cond, time_next, att, out_cls) def get_predicts(self, qs, ts, noises, betas): # out = [] # for i in range(qs.size(0)): out = self.get_predict_x0(qs, noises, betas) # out = torch.stack(out) out = [self.p_sample(ts[i], ts[i]-1, qs[i], out[i], noises[i]).float() for i in range(betas.size(0))] return torch.stack(out) def write_times(self, times): s = '' for time in times: s = s + str(time.item()) + ' ' s = s + '\n' def forward(self, fused_fea, fea_rgb, fea_tir): sample_num = fused_fea.size(0) # prepare ts, noises = self.prepare_targets(sample_num, fused_fea[0]) self.write_times(ts) qs = [] betas = [] one_betas = [] for i in range(sample_num): q, beta, one_beta = self.q_sample(fused_fea[i], ts[i], noises[i]) qs.append(q) betas.append(beta) one_betas.append(one_beta) if not self.condition: out = self.unet(qs, betas) else: qs = torch.stack(qs) betas = torch.stack(betas) con = torch.cat((fea_rgb, fea_tir), dim=1) out = self.unet(torch.cat((qs, con), dim=1), betas, fea_rgb) # self.loss_func.cuda() noises = torch.stack(noises) pred_n = [self.predict_noise_from_start(qs[i], ts[i], out[i]) for i in range(len(ts))] pred_n = torch.stack(pred_n) loss = self.loss_func(noises, pred_n).cuda() # predicted_x0s = self.get_predicts(qs, ts, out_noises, betas) return loss, out和 """ TBSI_Track model. Developed on OSTrack. """ import math from operator import ipow import os from typing import List
import torch from torch import nn from torch.nn.modules.transformer import _get_clones
from lib.models.layers.head import build_box_head, conv from lib.models.tbsi_track.vit_tbsi_care import vit_base_patch16_224_tbsi from lib.utils.box_ops import box_xyxy_to_cxcywh from lib.models.lddifuse.lddifuse import LDDiffuse
''' add token transfer to feature ''' def token2feature(tokens): B,L,D=tokens.shape H=W=int(L**0.5) x = tokens.permute(0, 2, 1).view(B, D, W, H).contiguous() return x
''' feature2token ''' def feature2token(x): B,C,W,H = x.shape L = W*H tokens = x.view(B, C, L).permute(0, 2, 1).contiguous() return tokens
class TBSIGMMTTrack(nn.Module): """ This is the base class for TBSITrack developed on OSTrack (Ye et al. ECCV 2022) """
def __init__(self, transformer, box_head, ldd_head, aux_loss=False, head_type="CORNER", cfg=None): """ Initializes the model. Parameters: transformer: torch module of the transformer architecture. aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. """ super().__init__() hidden_dim = transformer.embed_dim self.backbone = transformer self.tbsi_fuse_search = conv(hidden_dim * 2, hidden_dim) # Fuse RGB and T search regions, random initialized self.box_head = box_head self.aux_loss = aux_loss self.head_type = head_type if head_type == "CORNER" or head_type == "CENTER": self.feat_sz_s = int(box_head.feat_sz) self.feat_len_s = int(box_head.feat_sz ** 2) if self.aux_loss: self.box_head = _get_clones(self.box_head, 6) ####----------LDDiffuse-----------#### self.lddiffuse = LDDiffuse(cfg.TRAIN.T_STEP, cfg.TEST.SAMPLE) self.ldd_head = ldd_head def forward(self, template: torch.Tensor, search: torch.Tensor, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False, ): x, aux_dict = self.backbone(z=template, x=search, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate, return_last_attn=return_last_attn) # Forward head feat_last = x if isinstance(x, list): feat_last = x[-1] out, loss_lddifuse = self.forward_head(feat_last, None) out.update(aux_dict) out['backbone_feat'] = x out['loss_lddifuse'] = loss_lddifuse return out def forward_head(self, cat_feature, gt_score_map=None): """ cat_feature: output embeddings of the backbone, it can be (HW1+HW2, B, C) or (HW2, B, C) """ num_template_token = 64 num_search_token = 256 # encoder outputs for the visible and infrared search regions, both are (B, HW, C) enc_opt1 = cat_feature[:, num_template_token:num_template_token + num_search_token, :] enc_opt2 = cat_feature[:, -num_search_token:, :] enc_opt = torch.cat([enc_opt1, enc_opt2], dim=2) opt = (enc_opt.unsqueeze(-1)).permute((0, 3, 2, 1)).contiguous() bs, Nq, C, HW = opt.size() HW = int(HW/2) opt_feat = opt.view(-1, C, self.feat_sz_s, self.feat_sz_s) opt_feat = self.tbsi_fuse_search(opt_feat) loss_lddifuse, out = self.lddiffuse(opt_feat, token2feature(enc_opt1), token2feature(enc_opt2)) loss_lddifuse = loss_lddifuse / (out.shape[0] * out.shape[1] * out.shape[2] * out.shape[3]) if self.head_type == "CORNER": # run the corner head # pred_box, score_map = self.box_head(opt_feat, True) pred_box, score_map = self.ldd_head(opt_feat, True) # score_map_ctr, bbox, offset_map = self.ldd_head(opt_feat, gt_score_map) outputs_coord = box_xyxy_to_cxcywh(pred_box) outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map, } return out, loss_lddifuse elif self.head_type == "CENTER": # run the center head # score_map_ctr, bbox, size_map, offset_map = self.box_head(opt_feat, gt_score_map) score_map_ctr, bbox, size_map, offset_map = self.ldd_head(out, gt_score_map) # outputs_coord = box_xyxy_to_cxcywh(bbox) outputs_coord = bbox outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map_ctr, 'size_map': size_map, 'offset_map': offset_map} return out, loss_lddifuse else: raise NotImplementedError def test(self, template: torch.Tensor, search: torch.Tensor, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False, ): x, aux_dict = self.backbone(z=template, x=search, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate, return_last_attn=return_last_attn) # Forward head feat_last = x if isinstance(x, list): feat_last = x[-1] out = self.forward_head_test(feat_last, None) out.update(aux_dict) out['backbone_feat'] = x return out def forward_head_test(self, cat_feature, gt_score_map=None): """ cat_feature: output embeddings of the backbone, it can be (HW1+HW2, B, C) or (HW2, B, C) """ num_template_token = 64 num_search_token = 256 # encoder outputs for the visible and infrared search regions, both are (B, HW, C) enc_opt1 = cat_feature[:, num_template_token:num_template_token + num_search_token, :] enc_opt2 = cat_feature[:, -num_search_token:, :] enc_opt = torch.cat([enc_opt1, enc_opt2], dim=2) opt = (enc_opt.unsqueeze(-1)).permute((0, 3, 2, 1)).contiguous() bs, Nq, C, HW = opt.size() HW = int(HW/2) out = self.lddiffuse.predict(token2feature(enc_opt1), token2feature(enc_opt2)) if self.head_type == "CORNER": # run the corner head pred_box, score_map = self.ldd_head(out, True) outputs_coord = box_xyxy_to_cxcywh(pred_box) outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map, } return out elif self.head_type == "CENTER": # run the center head # score_map_ctr, bbox, size_map, offset_map = self.box_head(opt_feat, gt_score_map) score_map_ctr, bbox, size_map, offset_map = self.ldd_head(out, gt_score_map) outputs_coord = bbox outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map_ctr, 'size_map': size_map, 'offset_map': offset_map} return out else: raise NotImplementedErrordef build_tbsi_gmmt_track(cfg, training=True): current_dir = os.path.dirname(os.path.abspath(file)) # This is your Project Root pretrained_path = os.path.join(current_dir, '../../../pretrained_models') # if cfg.MODEL.PRETRAIN_FILE and ('TBSITrack' not in cfg.MODEL.PRETRAIN_FILE) and training: # pretrained = os.path.join(pretrained_path, cfg.MODEL.PRETRAIN_FILE) # print('Load pretrained model from: ' + pretrained) # else: # pretrained = '' pretrained = '' if cfg.MODEL.BACKBONE.TYPE == 'vit_base_patch16_224_tbsi': backbone = vit_base_patch16_224_tbsi(pretrained, drop_path_rate=cfg.TRAIN.DROP_PATH_RATE, tbsi_loc=cfg.MODEL.BACKBONE.TBSI_LOC, tbsi_drop_path=cfg.TRAIN.TBSI_DROP_PATH ) else: raise NotImplementedError
hidden_dim = backbone.embed_dim patch_start_index = 1 backbone.finetune_track(cfg=cfg, patch_start_index=patch_start_index) box_head = build_box_head(cfg, hidden_dim) ldd_head = build_box_head(cfg, hidden_dim) model = TBSIGMMTTrack( backbone, box_head, ldd_head, aux_loss=False, head_type=cfg.MODEL.HEAD.TYPE, cfg=cfg ) if 'TBSITrack' in cfg.MODEL.PRETRAIN_FILE and training: pretrained_file = os.path.join(pretrained_path, cfg.MODEL.PRETRAIN_FILE) checkpoint = torch.load(pretrained_file, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["net"], strict=False) print('Load pretrained model from: ' + cfg.MODEL.PRETRAIN_FILE) return model结合分析 LDDiffuse的作用
- Assistant
LDDiffuse类在您提供的代码中起着关键作用,尤其是在结合感知和生成任务的一些复杂模型中。以下是LDDiffuse的具体作用分析及其在模型中的作用:LDDiffuse的核心作用
-
噪声生成与处理:
LDDiffuse的主要功能是实现扩散模型中的噪声生成和去噪过程。它利用了一系列的公式,通过逐步添加噪声来训练模型,并在推理阶段去噪还原输入数据。
-
多模态特征集成:
- 在
TBSIGMMTTrack类中,LDDiffuse被用作融合不同模态(如 RGB 和热成像特征)的方法。它通过将噪声加到这些特征中,并训练去噪过程,使得这些模态特征能够相互补充并提高模型的鲁棒性和泛化性能。
- 在
-
数据提升与泛化:
LDDiffuse通过在特征中注入随机噪声,增强了训练数据的多样性。这有助于减少过拟合,提升模型在不同输入数据上的泛化能力。
-
复合损失过程:
- 在前向传播计算主损失的同时,
LDDiffuse还参与了复合损失的计算(如生成损失或对比损失)。其输出的loss_lddifuse提供了额外的训练信号,使得模型更加稳健。
- 在前向传播计算主损失的同时,
在TBSIGMMTTrack中的作用
-
特征处理:
- 在
TBSIGMMTTrack的前向过程里,LDDiffuse处理来自编码器的特征,增强了特征之间的互补性,特别是在forward_head中处理后的特征则被用于更为复杂的推理任务中。
- 在
-
创建生成对比:
LDDiffuse的功能不仅限于噪声处理,还在创建中间特征表示时发挥作用,生成用于评估差异和相似性的特征空间。
-
训练与推理环节:
- 在训练阶段,
LDDiffuse的输出被用来衡量生成过程的有效性。在推理阶段,则较多使用其去噪能力,以逐步去掉输入特征中的噪声,产生更高质量的输出。
- 在训练阶段,
总结
LDDiffuse在该模型中起到了多模态特征集成、提升数据多样性、为训练提供复合损失等多种关键性作用。通过在模型的不同层次和阶段对特征进行有控制地加噪和去噪处理,LDDiffuse能够有效改善特征的对齐,对挑战性的识别和感知任务提供支持。届时,这种机制为模型带来了鲁棒性及更好的泛化能力。 -
- User
那么同样的,在""" Basic mamda_fetrack model. """ import math import os from typing import List
import torch from torch import nn from torch.nn.modules.transformer import _get_clones
from lib.models.layers.head import build_box_head from lib.utils.box_ops import box_xyxy_to_cxcywh from lib.models.mamba_fetrack.models_mamba import create_block from timm.models import create_model import torch.nn.functional as F from lib.models.mamba_fetrack.mamba_cross import CrossMamba from thop import profile
class Mamba_FEtrack(nn.Module): """ This is the base class for mamda_fetrack """
def __init__(self, visionmamba, cross_mamba, box_head, aux_loss=False, head_type="CORNER"): """ Initializes the model. Parameters: transformer: torch module of the transformer architecture. aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. """ super().__init__() self.backbone = visionmamba self.cross_mamba= cross_mamba self.box_head = box_head self.aux_loss = aux_loss self.head_type = head_type if head_type == "CORNER" or head_type == "CENTER": self.feat_sz_s = int(box_head.feat_sz) self.feat_len_s = int(box_head.feat_sz ** 2) if self.aux_loss: self.box_head = _get_clones(self.box_head, 6) def forward(self, template: torch.Tensor, search: torch.Tensor, event_template: torch.Tensor, event_search: torch.Tensor, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False, ): rgb_feature = self.backbone.forward_features( z=template, x=search, #[B, 320, 384] inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False) event_feature = self.backbone.forward_features(z=event_template, x=event_search, #[B, 320, 384] inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False) residual_event_f = 0 residual_rgb_f = 0 event_f = self.cross_mamba(event_feature,residual_event_f,rgb_feature) + event_feature rgb_f = self.cross_mamba(rgb_feature,residual_rgb_f,event_feature) + rgb_feature event_searh = event_f[:, -self.feat_len_s:] rgb_search = rgb_f[:, -self.feat_len_s:] x = torch.cat((event_searh,rgb_search),dim=-1) feat_last = x if isinstance(x, list): feat_last = x[-1] out = self.forward_head(feat_last, None) out['backbone_feat'] = x return out def forward_head(self, cat_feature, gt_score_map=None): """ cat_feature: output embeddings of the backbone, it can be (HW1+HW2, B, C) or (HW2, B, C) """ search_feature = cat_feature opt = (search_feature.unsqueeze(-1)).permute((0, 3, 2, 1)).contiguous() # opt.shape = torch.Size([B, 1, 384, 256]) bs, Nq, C, HW = opt.size() opt_feat = opt.view(-1, C, self.feat_sz_s, self.feat_sz_s) # opt_feat.shape = torch.Size([B, 384, 16, 16]) if self.head_type == "CORNER": # run the corner head pred_box, score_map = self.box_head(opt_feat, True) outputs_coord = box_xyxy_to_cxcywh(pred_box) outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map, } return out elif self.head_type == "CENTER": # run the center head score_map_ctr, bbox, size_map, offset_map = self.box_head(opt_feat, gt_score_map) outputs_coord = bbox outputs_coord_new = outputs_coord.view(bs, Nq, 4) out = {'pred_boxes': outputs_coord_new, 'score_map': score_map_ctr, 'size_map': size_map, 'offset_map': offset_map} return out else: raise NotImplementedErrordef build_mamba_fetrack(cfg, training=True): current_dir = os.path.dirname(os.path.abspath(file)) # This is your Project Root pretrained_path = os.path.join(current_dir, 'pretrained_models') # pretrained_path = os.path.join(current_dir, '../../../pretrained_models') if cfg.MODEL.PRETRAIN_FILE and ('Mamba_FETrack' not in cfg.MODEL.PRETRAIN_FILE) and training: pretrained = os.path.join(pretrained_path, cfg.MODEL.PRETRAIN_FILE) else: pretrained = ''
backbone = create_model( model_name= cfg.MODEL.BACKBONE.TYPE, pretrained= pretrained, num_classes=1000, drop_rate=0.0, drop_path_rate=cfg.TRAIN.DROP_PATH_RATE, drop_block_rate=None, img_size=256 ) hidden_dim = 384 cross_mamba = CrossMamba(hidden_dim) box_head = build_box_head(cfg, hidden_dim*2) model = Mamba_FEtrack( backbone, cross_mamba, box_head, aux_loss=False, head_type=cfg.MODEL.HEAD.TYPE, ) if 'Mamba_FETrack' in cfg.MODEL.PRETRAIN_FILE and training: checkpoint = torch.load(cfg.MODEL.PRETRAIN_FILE, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["net"], strict=False) print('Load pretrained model from: ' + cfg.MODEL.PRETRAIN_FILE) return model中集成 LDDiffuse
- Assistant
要在
Mamba_FEtrack模型中集成LDDiffuse,我们可以采用和之前类似的方法,将LDDiffuse用于卷积特征或其他阶段的特征处理。LDDiffuse的主要作用是将扩散模型用于特征噪声和去噪,这在视觉任务中可以增强模型的鲁棒性和泛化能力。以下是一些可能的集成方法:集成步骤
-
初始化
LDDiffuse:在
Mamba_FEtrack类的初始化过程中添加LDDiffuse实例。需要从配置中获取步骤数和采样数参数。class Mamba_FEtrack(nn.Module): def __init__(self, visionmamba, cross_mamba, box_head, aux_loss=False, head_type="CORNER", cfg=None): super().__init__() self.backbone = visionmamba self.cross_mamba = cross_mamba self.box_head = box_head self.aux_loss = aux_loss self.head_type = head_type if head_type in ["CORNER", "CENTER"]: self.feat_sz_s = int(box_head.feat_sz) self.feat_len_s = int(box_head.feat_sz ** 2) if self.aux_loss: self.box_head = _get_clones(self.box_head, 6) # 初始化 LDDiffuse self.lddiffuse = LDDiffuse(cfg.TRAIN.T_STEP, cfg.TEST.SAMPLE) -
在
forward方法中使用LDDiffuse:可以在
backbone特征生成之后、特征融合之前使用LDDiffuse来处理特征。这可以推动模型学习更具鲁棒性的特征表示。def forward(self, template: torch.Tensor, search: torch.Tensor, event_template: torch.Tensor, event_search: torch.Tensor, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False): rgb_feature = self.backbone.forward_features( z=template, x=search, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False ) event_feature = self.backbone.forward_features( z=event_template, x=event_search, inference_params=None, if_random_cls_token_position=False, if_random_token_rank=False ) # 使用 LDDiffuse 处理特征 loss_lddiffuse, new_rgb_feature = self.lddiffuse(rgb_feature, rgb_feature, event_feature) event_f = self.cross_mamba(new_rgb_feature, 0, event_feature) + event_feature rgb_f = self.cross_mamba(new_rgb_feature, 0, event_feature) + new_rgb_feature event_search = event_f[:, -self.feat_len_s:] rgb_search = rgb_f[:, -self.feat_len_s:] x = torch.cat((event_search, rgb_search), dim=-1) feat_last = x if isinstance(x, list): feat_last = x[-1] out = self.forward_head(feat_last, None) out['backbone_feat'] = x out['loss_lddifuse'] = loss_lddiffuse return out -
补充配置文件:
确保您的配置文件(
cfg)中包含LDDiffuse需要的参数(如TRAIN.T_STEP和TEST.SAMPLE)。
整合后的作用
-
特征增强:
LDDiffuse的集成可以在特征级别对输入数据进行噪声增强和去噪处理,从而在特征层次增强模型的泛化能力。 -
鲁棒性提升: 通过反复的噪声去噪学习,模型可以有效增强其对 noisy 数据的处理能力,提高检测和预测的准确性。
-
差异性学习: 通过噪声扰动促进多模态特征的融合,有效利用多源信息以提高监测及跟踪的效果。
这种集成使得模型从实时的视觉任务中获益,提高了模型在噪声环境下的表现稳定性。
-