参考import math import logging from functools import partial from collections import…
- User
参考import math import logging from functools import partial from collections import OrderedDict from copy import deepcopy
import torch import torch.nn as nn import torch.nn.functional as F
from timm.models.layers import to_2tuple
from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise from .utils import combine_tokens, recover_tokens from .vit import VisionTransformer from ..layers.attn_blocks import CEBlock from math import sqrt
_logger = logging.getLogger(name)
class VisionTransformerCE(VisionTransformer): """ Vision Transformer with candidate elimination (CE) module
A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` - https://arxiv.org/abs/2010.11929 Includes distillation token & head support for `DeiT: Data-efficient Image Transformers` - https://arxiv.org/abs/2012.12877 """ def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None, act_layer=None, weight_init='', ce_loc=None, ce_keep_ratio=None): """ Args: img_size (int, tuple): input image size patch_size (int, tuple): patch size in_chans (int): number of input channels num_classes (int): number of classes for classification head embed_dim (int): embedding dimension depth (int): depth of transformer num_heads (int): number of attention heads mlp_ratio (int): ratio of mlp hidden dim to embedding dim qkv_bias (bool): enable bias for qkv if True representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set distilled (bool): model includes a distillation token and head as in DeiT models drop_rate (float): dropout rate attn_drop_rate (float): attention dropout rate drop_path_rate (float): stochastic depth rate embed_layer (nn.Module): patch embedding layer norm_layer: (nn.Module): normalization layer weight_init: (str): weight init scheme """ # super().__init__() super().__init__() if isinstance(img_size, tuple): self.img_size = img_size else: self.img_size = to_2tuple(img_size) self.patch_size = patch_size self.in_chans = in_chans self.num_classes = num_classes self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models self.num_tokens = 2 if distilled else 1 norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) act_layer = act_layer or nn.GELU self.patch_embed = embed_layer( img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) num_patches = self.patch_embed.num_patches self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) self.pos_drop = nn.Dropout(p=drop_rate) self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4) # self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4) # self.pos_embed_event_z = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=3, stride=1) # attn = CrossAttn(768, 4, 3072, 0.1, 'relu') # self.cross_attn = Iter_attn(attn, 2) dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule blocks = [] ce_index = 0 self.ce_loc = ce_loc for i in range(depth): ce_keep_ratio_i = 1.0 if ce_loc is not None and i in ce_loc: ce_keep_ratio_i = ce_keep_ratio[ce_index] ce_index += 1 blocks.append( CEBlock( dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer, keep_ratio_search=ce_keep_ratio_i) ) self.blocks = nn.Sequential(*blocks) # hopufield_layer num_prototype = 1000 self.lookup = nn.Linear(embed_dim, num_prototype ,bias=False) self.beta = 1./sqrt(embed_dim) self.content = nn.Sequential( nn.Linear(num_prototype, embed_dim,bias=False), nn.LayerNorm(embed_dim), ) ############################################################## self.norm = norm_layer(embed_dim) self.init_weights(weight_init) def forward_features(self, z, x, event_z, event_x, mask_z=None, mask_x=None, ce_template_mask=None, ce_keep_rate=None, return_last_attn=False ): B, H, W = x.shape[0], x.shape[2], x.shape[3] event_z = self.pos_embed_event(event_z) # [:,:,:,:1000] event_x = self.pos_embed_event(event_x) # B 768 1024 x = self.patch_embed(x) z = self.patch_embed(z) event_z += self.pos_embed_z event_x += self.pos_embed_x z += self.pos_embed_z x += self.pos_embed_x # attention mask handling # B, H, W if mask_z is not None and mask_x is not None: mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_z = mask_z.flatten(1).unsqueeze(-1) mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0] mask_x = mask_x.flatten(1).unsqueeze(-1) mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode) mask_x = mask_x.squeeze(-1) if self.add_cls_token: cls_tokens = self.cls_token.expand(B, -1, -1) cls_tokens = cls_tokens + self.cls_pos_embed if self.add_sep_seg: x += self.search_segment_pos_embed z += self.template_segment_pos_embed x = combine_tokens(z, event_z, x, event_x, mode=self.cat_mode) # 64+64+256+256=640 # x = combine_tokens(z, x, event_z, event_x, mode=self.cat_mode) # 64+64+256+256=640 if self.add_cls_token: x = torch.cat([cls_tokens, x], dim=1) x = self.pos_drop(x) lens_z = self.pos_embed_z.shape[1] lens_x = self.pos_embed_x.shape[1] global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device) global_index_t = global_index_t.repeat(B, 1) global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device) global_index_s = global_index_s.repeat(B, 1) removed_indexes_s = [] for i, blk in enumerate(self.blocks): x, global_index_t, global_index_s, removed_index_s, attn = \ blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate, i) if self.ce_loc is not None and i in self.ce_loc: removed_indexes_s.append(removed_index_s) # The last hopufield layer x = torch.softmax(self.lookup(x)*self.beta, dim=-1) x = self.content(x) x = self.norm(x) lens_x_new = global_index_s.shape[1] lens_z_new = global_index_t.shape[1] z = x[:, :lens_z_new*2] x = x[:, lens_z_new*2:] if removed_indexes_s and removed_indexes_s[0] is not None: removed_indexes_cat = torch.cat(removed_indexes_s, dim=1) pruned_lens_x = lens_x - lens_x_new pad_x = torch.zeros([B, pruned_lens_x, x.shape[2]], device=x.device) x = torch.cat([x, pad_x], dim=1) index_all = torch.cat([global_index_s, removed_indexes_cat], dim=1) # recover original token order C = x.shape[-1] x = torch.zeros_like(x).scatter_(dim=1, index=index_all.unsqueeze(-1).expand(B, -1, C).to(torch.int64), src=x) x = recover_tokens(x, lens_z_new, lens_x, mode=self.cat_mode) x = x[:, :lens_x] # RGB head x = torch.cat([event_x, x], dim=1) # x = x[:, lens_x//2:] # event head # x = torch.cat([z, x], dim=1) # re-concatenate with the template, which may be further used by other modules # x, event_x = x[:, :lens_x//2], x[:, lens_x//2:] # x = x[:, -lens_x//2:] aux_dict = { "attn": attn, "removed_indexes_s": removed_indexes_s, # used for visualization } return x, aux_dict def forward(self, z, x, event_z, event_x, ce_template_mask=None, ce_keep_rate=None, tnc_keep_rate=None, return_last_attn=False): x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,) return x, aux_dictdef _create_vision_transformer(pretrained=False, **kwargs): model = VisionTransformerCE(**kwargs)
if pretrained: if 'npz' in pretrained: model.load_pretrained(pretrained, prefix='') else: checkpoint = torch.load(pretrained, map_location="cpu") missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False) print('Load pretrained model from: ' + pretrained) return modeldef vit_base_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model
def vit_large_patch16_224_ce(pretrained=False, **kwargs): """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). """ model_kwargs = dict( patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) model = _create_vision_transformer(pretrained=pretrained, **model_kwargs) return model 中的 # hopufield_layer num_prototype = 1000 self.lookup = nn.Linear(embed_dim, num_prototype ,bias=False)
self.beta = 1./sqrt(embed_dim)self.content = nn.Sequential( nn.Linear(num_prototype, embed_dim,bias=False), nn.LayerNorm(embed_dim), )添加的位置,分析一下添加在此处的作用是什么? - Assistant
The code fragment you are referring to is adding a Hopfield layer to the
VisionTransformerCEclass, which is a specialized version of the Vision Transformer with candidate elimination (CE) module. Let's break down the added code and analyze its purpose:-
Hopfield Networks:
- Hopfield networks are a type of recurrent neural network that can serve as associative memory systems with binary threshold nodes.
- They can store one or more "memories" or "prototypes" and are capable of retrieving or completing a pattern by converging to one of the stored prototypes.
-
Code Analysis:
# hopufield_layer num_prototype = 1000 self.lookup = nn.Linear(embed_dim, num_prototype ,bias=False) self.beta = 1./sqrt(embed_dim) self.content = nn.Sequential( nn.Linear(num_prototype, embed_dim,bias=False), nn.LayerNorm(embed_dim), ) -
Breakdown:
-
num_prototype = 1000: This sets the number of prototypes that the Hopfield layer can learn and store. The choice of 1000 could be based on the number of classes or some other criteria specific to the task. -
self.lookup: A linear layer that projects the input embedding dimension (embed_dim) to the number of prototypes (num_prototype). This operation is similar to calculating the energy (or matching score) between the input and each prototype stored in the Hopfield network. -
self.beta = 1./sqrt(embed_dim): This is a scaling factor. The reasoning behind this choice often goes back to stabilizing the variance of the inputs to subsequent layers, which is common in the initialization of weights in neural networks. Scaling the output in this way can help prevent gradient explosions or vanishing gradients during training. -
self.content: A sequential container with a linear layer and layer normalization. After computing the interaction with prototypes, this step transforms the results back to the original embedding dimension (embed_dim).nn.LayerNormhelps to stabilize the training by normalizing the output for each batch.
-
-
Added Layer Purpose:
- Essentially, this additional Hopfield layer acts as a form of associative memory.
- During the forward pass, the model computes the similarity between input features and learned prototypes using
self.lookup. - The activations are scaled by
self.betaand transformed back into the feature space byself.content. - The purpose of these operations is to enhance the model's ability to capture and utilize meaningful patterns or features that might correspond to the learned prototypes. This can lead to improved generalization and robustness, especially if the prototypes capture relevant class-specific or contextual information.
Overall, the Hopfield layer is an enhancement to the Vision Transformer that adds a mechanism for learning and retrieving prototype patterns, potentially leading to better performance on tasks requiring complex pattern recognition.
-
- User
中文回答
- Assistant
在
VisionTransformerCE类中添加的代码片段实现了一个Hopfield层,这是一种增强模型能力的机制。我们来逐步分析这些代码及其目的:-
Hopfield网络:
- Hopfield网络是一种递归神经网络,可以作为具备二值阈值单元的联想记忆系统。
- 它们能够存储一个或多个“记忆”或“原型”,并通过收敛到存储的某个原型来检索或补全模式。
-
代码分析:
# hopufield_layer num_prototype = 1000 self.lookup = nn.Linear(embed_dim, num_prototype ,bias=False) self.beta = 1./sqrt(embed_dim) self.content = nn.Sequential( nn.Linear(num_prototype, embed_dim,bias=False), nn.LayerNorm(embed_dim), ) -
详细分析:
-
num_prototype = 1000:这定义了Hopfield层可以学习和存储的原型数量。选择1000可能与类别数量有关,或者是基于任务的某些其他标准。 -
self.lookup:这是一个线性层,将输入的嵌入维度(embed_dim)转化为原型的数量(num_prototype)。这个操作类似于计算输入与每个存储原型之间的能量(或匹配得分)。 -
self.beta = 1./sqrt(embed_dim):这是一个缩放因子。选择这种缩放常与稳定传入下一层的输入方差有关,类似于神经网络中权重初始化时的做法。通过这样缩放输出,可以防止训练中的梯度爆炸或消失。 -
self.content:是一个包含线性层和层归一化的顺序容器。在计算与原型的交互之后,这一步将结果转换回原始的嵌入维度(embed_dim)。nn.LayerNorm有助于通过对每个批次中的输出进行归一化来稳定训练。
-
-
添加层的目的:
- 基本上,这个添加的Hopfield层起到了联想记忆的作用。
- 在前向传播过程中,模型计算输入特征与学习到的原型之间的相似性,使用
self.lookup进行。 - 激活结果通过
self.beta进行缩放,并通过self.content转换回特征空间。 - 这些操作的目的是增强模型捕获和利用重要模式或特征的能力,这些模式或特征可能对应于学习到的原型。这可以提高模型的泛化能力和鲁棒性,尤其是在原型能够捕获相关的类别特定或上下文信息的情况下。
总体而言,Hopfield层对于Vision Transformer是一种增强,增加了一种学习和检索原型模式的机制,可能会提升在需要复杂模式识别的任务上的性能。
-
- User
理解:import abc import random import numpy as np import torch import torch.nn.functional as F import math
class THOR_Wrapper(): def init(self, net, st_capacity, lt_capacity, sample_interval, lb=0.90, response_threshold=0.8): self.response_threshold = response_threshold self.st_capacity = st_capacity self.lt_capacity = lt_capacity self._curr_type = 'st' self._frame_no = 0 self.interval = sample_interval self.net = net self.lb = lb self.lt_count = 0 self.st_count = 0
def setup(self, rawImage): """ initialize the short-term and long-term module """ self.st_module = ST_Module(template_capacity=self.st_capacity, network=self.net) self.st_module.fill(rawImage) self.lt_module = LT_Module(template_capacity=self.lt_capacity, network=self.net, lb=self.lb) self.lt_module.fill(rawImage) def update(self, rawImage): """ update the short-term and long-term module """ ''' 相似度方法 # 使用相似度来作为判断依据,发现最终都会使用LT ''' if self._frame_no % self.interval == 0: div_scale = self.st_module.update(rawImage) # print('frame no: {}, div_scale: {}'.format(self._frame_no, div_scale)) self.lt_module.update(rawImage, div_scale=div_scale, time=self._frame_no) cur_feature = self.net.backbone.forward_dynamic_features_inference([rawImage]) st_curr_sims = self.st_module.pairwise_similarities(cur_feature) lt_curr_sims = self.lt_module.pairwise_similarities(cur_feature) if torch.max(st_curr_sims) >= torch.max(lt_curr_sims): self._curr_type = 'st' self.st_count += 1 return self.st_module.get_dynamic_template_features() else: self._curr_type = 'lt' self.lt_count += 1 return self.lt_module.get_dynamic_template_features()class TemplateModule(): def init(self, network, template_capacity, feat_len_z=64): self.is_full = False self.template_capacity = template_capacity # 模板数量 self.template_raw_list = [] self.sorted_template_raw_list = [] self._templates_features_stack = None self._templates_features_stack_sim = None self._base_sim = 0 self._gram_matrix = None self.feat_len_z = feat_len_z self.network = network self.timestamp_list = []
def __len__(self): return self.template_capacity def pairwise_similarities(self, T_n, to_cpu=False): """ 卷积法计算相似度 calculate similarity of given template to all templates in memory 将一个(1, N, C)的模板T_n与内存中的所有模板进行比较,返回一个(M,)的相似度向量 assert isinstance(T_n, torch.Tensor) B, N, C = T_n.size() H = W = int(math.sqrt(N)) T_n = T_n.view(1, C, H, W) # nums = int(self._templates_features_stack.size()[1]/self.feat_len_z) # self._templates_features_stack = self._templates_features_stack.view(nums, C, H, W) sims = F.conv2d(T_n, self._templates_features_stack_sim) sims = sims.flatten() return sims """ # 将A的形状从(1, N, C)调整为(N, C) T = T_n.clone().squeeze(0) # 将LIST的形状从(1, N*m, C)调整为(N*m, C) LIST = self._templates_features_stack.clone().squeeze(0) # 初始化sims变量,用于存储每个LIST元素与A的皮尔逊相关性系数 sims = torch.empty((LIST.size(0) // T.size(0),)) # 计算A的均值和标准差 mean_T_n = torch.mean(T, dim=0) std_T_n = torch.std(T, dim=0) # 遍历LIST中的每个元素(假设每个元素大小也是(N, C)) for i in range(0, LIST.size(0), T.size(0)): # 提取LIST中的对应元素 element = LIST[i:i+T.size(0)] # 计算元素的均值和标准差 mean_element = torch.mean(element, dim=0) std_element = torch.std(element, dim=0) # 计算皮尔逊相关性系数 cov = torch.sum((T - mean_T_n) * (element - mean_element), dim=0) pearson_coeff = cov / (std_T_n * std_element) # 将计算结果赋值给sims sims[i // T.size(0)] = torch.mean(pearson_coeff) # 取平均是为了处理多通道C的情况 return sims def _calculate_gram_matrix(self): ''' self._templates_features_stack.shape : (n*64, 384) ''' # 使用PyTorch计算所有模板的成对相似度 nums = int(self._templates_features_stack.size()[1]/self.feat_len_z) dists = [] for idx in range(nums): left = idx*self.feat_len_z right = (idx+1)*self.feat_len_z dists.append(self.pairwise_similarities(self._templates_features_stack[:,left:right])) # dists.append(self.pairwise_similarities(self._templates_features_stack[:,idx*self.feat_len_z:(idx+1)*self.feat_len_z])) # dists = [self.pairwise_similarities(self._templates_features_stack[:,idx*self.feat_len_z:(idx+1)*self.feat_len_z]) for idx in range(nums)] # 使用PyTorch的squeeze函数,并且不需要转换成numpy数组 gram_matrices = [dists[i].squeeze() for i in range(len(dists))] gram_matrices = torch.stack(gram_matrices, dim=0) return gram_matrices def _set_temp(self, temp, idx, time): """ switch out the template at idx """ self.template_raw_list[idx] = temp self.timestamp_list[idx] = time @abc.abstractmethod def update(self, temp): """ check if template should be taken into memory """ raise NotImplementedError("Must be implemented in subclass.") @abc.abstractmethod def fill(self, temp): """ fill all slots in the memory with the given template # 填充函数 """ for _ in range(self.template_capacity): self.template_raw_list.append(temp) self.sorted_template_raw_list = self.template_raw_list self._gen_templates_features_stack() self._gram_matrix = self._calculate_gram_matrix() self._base_sim = self._gram_matrix[0, 0] self.timestamp_list = [0] * self.template_capacity def _gen_templates_features_stack(self): self._templates_features_stack = [] for temp in self.template_raw_list: self._templates_features_stack.append(self.network.backbone.forward_dynamic_features_inference([temp])) self._templates_features_stack = torch.cat(self._templates_features_stack, dim=1) B, N, C = self._templates_features_stack.size() N = N//self.feat_len_z H = W = int(math.sqrt(self.feat_len_z)) self._templates_features_stack_sim = self._templates_features_stack.clone().view(N, C, H, W) def get_dynamic_template_features(self): return self.network.backbone.forward_dynamic_features_inference(self.template_raw_list)[:, -self.feat_len_z:] # return self._templates_features_stack[:, -self.feat_len_z:]class ST_Module(TemplateModule): def init(self, network, template_capacity): super(ST_Module, self).init(network, template_capacity)
@staticmethod def normed_div_measure(matrix): """ calculate the normed diversity measure of matirx, the higher the more diverse """ # 确认输入张量t是方阵 assert matrix.shape[0] == matrix.shape[1], "Input tensor must be square" dim = matrix.shape[0] - 1 triu_no = int(dim * (dim + 1) / 2) # 注意这个 dim=n-1, n为方阵维度,此处计算的是原方阵上三角数目 # 使用PyTorch的triu函数来计算上三角矩阵的和 triu_sum = torch.sum(torch.triu(matrix, 1)) # 计算归一化多样性度量 # measure = 1 - triu_sum / (matirx.max() * triu_no) measure = triu_sum / (matrix[0, 0] * triu_no) # 多样性计算公式, np.triu(t, 1)提取上三角部分。公式为: 上三角元素和 / 首元素*上三角元素数目 return measure def _update_gram_matrix(self, new_template): ''' new_template.shape -> (B, N, C) ''' # 计算当前的距离 curr_sims = self.pairwise_similarities(new_template) # sorted_indices = torch.argsort(curr_sims, descending=True) # # 使用排序索引对self.template_raw_list进行排序 # self.sorted_template_raw_list = [self.template_raw_list[i] for i in sorted_indices] curr_sims = curr_sims.unsqueeze(1) # 使用 PyTorch 的 unsqueeze 方法 # 更新 Gram 矩阵 all_dists_new = torch.cat([ self._gram_matrix, curr_sims], dim=1) # 水平拼接 curr_sims = torch.cat((torch.zeros(1, 1).to(curr_sims.device), curr_sims.T), dim=1) # 水平拼接 all_dists_new = torch.cat([ all_dists_new, curr_sims], dim=0) # 垂直拼接 # 删除索引为 0 的行和列,即最旧的模板 self._gram_matrix = all_dists_new[1:, 1:] def update(self, temp): """ append to the current memory and rebuild canvas return div_scale (diversity of the current memory) """ # update distance matrix and calculate the div scale self.template_raw_list.append(temp) self.template_raw_list.pop(0) self._gen_templates_features_stack() temp = self.network.backbone.forward_dynamic_features_inference([temp]) self._update_gram_matrix(temp) # self._templates_features_stack = self.network.backbone.forward_dynamic_features_inference(self.template_raw_list) return self.normed_div_measure(matrix=self._gram_matrix)class LT_Module(TemplateModule): def init(self, network, template_capacity, lb=0.883665, lb_type='dynamic'): ''' lb : 0.90, 0.75, 0.60, 0.45 '''
super(LT_Module, self).__init__(network, template_capacity) self._lb = lb self._lb_type = lb_type self._filled_idx = 0 # def get_dynamic_template_features(self): # # shuffle_list = self.template_raw_list.copy() # sorted_indices = sorted(range(len(self.timestamp_list)), key=lambda k: self.timestamp_list[k]) # sorted_timestamp_list = [self.template_raw_list[i] for i in sorted_indices] # return self.network.backbone.forward_dynamic_features_inference(sorted_timestamp_list)[:, -self.feat_len_z:] def _throw_away_or_keep(self, curr_sims, self_sim, div_scale): """ determine if we keep the template or not if the template is rejected: return -1 (not better) or -2 (rejected by lower bound) if we keep the template: return idx where to switch """ base_sim = self._base_sim curr_sims = curr_sims.unsqueeze(1) # normalize the gram_matrix, otherwise determinants are huge gram_matrix_norm = self._gram_matrix / base_sim curr_sims_norm = curr_sims / base_sim # check if distance to base template is below lower bound if self._lb_type == 'static': reject = (curr_sims[0] < self._lb * base_sim) elif self._lb_type == 'dynamic': lb = self._lb - (1 - div_scale) lb = torch.clamp(lb, 0.0, 1.0) reject = (curr_sims[0] < lb * base_sim) # print('Reject: ', reject, 'curr_sims: ', curr_sims[0].item(), # 'lb: ', lb.item(), 'base_sim: ', base_sim.item(), 'lb * base_sim: ', (lb * base_sim).item()) elif self._lb_type == 'ensemble': reject = not torch.all(curr_sims_norm > self._lb) else: raise TypeError(f"lower boundary type {self._lb_type} not known.") if reject: return -2 # fill the memory with adjacent frames if they are not # populated with something different than the base frame yet if self._filled_idx < (self.template_capacity - 1): self._filled_idx += 1 throwaway_idx = self._filled_idx # determine if and in which spot the template increases the current gram determinant else: curr_det = torch.det(gram_matrix_norm) # start at 1 so we never throwaway the base template dets = torch.zeros((self.template_capacity - 1,)) for i in range(self.template_capacity - 1): mat = gram_matrix_norm.clone() mat[i + 1, :] = curr_sims_norm.T mat[:, i + 1] = curr_sims_norm.T mat[i + 1, i + 1] = self_sim / base_sim dets[i] = torch.det(mat) # check if any of the new combinations is better than the prev. gram_matrix max_idx = torch.argmax(dets) if curr_det > dets[max_idx]: throwaway_idx = -1 else: throwaway_idx = max_idx + 1 assert throwaway_idx != 0 return throwaway_idx if throwaway_idx != self.template_capacity else -1 # @staticmethod # def save_det(d, p): # if os.path.exists(p): # old_det = np.load(p) # else: # old_det = np.array([]) # np.save(p, np.concatenate([old_det, d.reshape(-1)])) def _update_gram_matrix(self, curr_sims, self_sim, idx): """ update the current gram matrix """ curr_sims = curr_sims.unsqueeze(1) # add the self similarity at throwaway_idx spot curr_sims[idx] = self_sim self._gram_matrix[idx, :] = curr_sims.T self._gram_matrix[:, idx] = curr_sims.T # gram_matrix_norm = self._gram_matrix/self._base_sim # curr_det = np.linalg.det(gram_matrix_norm) # self.save_det(curr_det, 'determinants_dyn.npy') def update(self, temp, div_scale, time): """ decide if the templates is taken into the lt module """ # calculate the "throwaway_idx", the spot that the new template will take temp_features = self.network.backbone.forward_dynamic_features_inference([temp]) curr_templatae = temp_features.clone() B, N, C = curr_templatae.size() H = W = int(math.sqrt(N)) curr_templatae = curr_templatae.view(1, C, H, W) curr_sims = self.pairwise_similarities(temp_features) self_sim = F.conv2d(curr_templatae, curr_templatae).squeeze().item() throwaway_idx = self._throw_away_or_keep(curr_sims=curr_sims, self_sim=self_sim, div_scale=div_scale) # if the idx is -2 or -1, the template is rejected, otherwise we update if throwaway_idx == -2 or throwaway_idx == -1: # print('div_scale: {}, reject: {}'.format(div_scale, 'True')) pass # rejected else: # print('div_scale: {}, reject: {}'.format(div_scale, 'False')) self._set_temp(temp=temp, idx=throwaway_idx, time=time) # sorted_indices = torch.argsort(curr_sims, descending=True) # 使用排序索引对self.template_raw_list进行排序 # self.sorted_template_raw_list = [self.template_raw_list[i] for i in sorted_indices] self._update_gram_matrix(curr_sims=curr_sims, self_sim=self_sim, idx=throwaway_idx) self._gen_templates_features_stack() # self._templates_features_stack = self.network.backbone.forward_dynamic_features_inference(self.sorted_template_raw_list)中的函数。
- Assistant
这个代码实现了一个称为THOR的模板管理器,其主要目的是将视觉网络中的动态模板管理分为短期(short-term, ST)和长期(long-term, LT)两类模块。每个模块都有相应的类实现,并使用了多个函数来确保有效的管理和更新模板。以下是一些关键函数的解释:
THOR_Wrapper 类
setup(self, rawImage)- 初始化短期和长期模板模块。
- 用输入的初始图像填充短期和长期模块。
update(self, rawImage)- 例行更新短期和长期模块。
- 通过计算当前特征与短期和长期模块中的模板特征的相似度来确定使用哪个模块的模板。
- 返回当前所选模块的模板特征。
TemplateModule 类
pairwise_similarities(self, T_n, to_cpu=False)- 计算输入模板与内存中所有模板之间的相似性。
- 使用皮尔逊相关系数来计算相似度。
_calculate_gram_matrix(self)- 计算并返回模板的格拉姆矩阵。格拉姆矩阵用于度量空间中向量(在这里是模板特征)的相似性。
- 用于衡量和更新模块中保留的模板的多样性。
update(self, temp)(抽象方法)- 检查是否应将模板加入内存。
- 这是一个抽象方法,在子类中具体实现。
fill(self, temp)(抽象方法)- 使用提供的模板填充内存的所有槽。
- 在子类中具体实现。
ST_Module 类(短期模块)
normed_div_measure(matrix)(静态方法)- 计算矩阵的归一化多样性度量,值越高表示越多样化。
- 多样性通过上三角矩阵的和与矩阵首元素乘以上三角元素个数的商计算得出。
update(self, temp)- 更新模板内存,并返回当前内存的多样性量度。
- 在短期模块中,新的模板每次都会替换最早的模板。
LT_Module 类(长期模块)
_throw_away_or_keep(self, curr_sims, self_sim, div_scale)- 决定是否保留新模板。
- 使用格拉姆矩阵行列式和多样性尺度来衡量当前模板与现有模板的相似性和多样性。
update(self, temp, div_scale, time)- 决定是否将模板加入长期模块。
- 通过调用
_throw_away_or_keep函数判断,决定是否及如何更新格拉姆矩阵和模板特征栈。
综上,这个代码实现了一种具有短期记忆和长期记忆的动态模板管理系统。在实时或快速变化的场景中,这种结构帮助模型采用最相关的参考模板来进行推断,提高识别的准确性和效率。
- User
那么根据上文,继续分析:import math
from lib.models.ostrack import build_ostrack from lib.models.thor import THOR_Wrapper
from lib.test.tracker.basetracker import BaseTracker import torch
from lib.test.tracker.vis_utils import gen_visualization from lib.test.utils.hann import hann2d from lib.train.data.processing_utils import sample_target
for debug
import cv2 import os
from lib.test.tracker.data_utils import Preprocessor from lib.utils.box_ops import clip_box from lib.utils.ce_utils import generate_mask_cond import random import time import torch.nn.functional as F import numpy as np
from .utils import to_numpy, print_color
''' 曾经使用的置信度measure # print('----'*40) # print(response.flatten().sort(descending=True)[0][:10])
# top1response = response.max() # top1response = 0 if top1response < self.params.cfg.TEST.THRESHOLD else top1response # self.threshold = update_threshold(self.threshold, response.flatten().sort(descending=True)[0][:5]) # print(self.threshold) # typical_confidence = get_bbox_confidence(*(pred_boxes.mean(dim=0)*self.params.search_size), response[0][0]) # 代表性置信度 # self.threshold = self.params.cfg.TEST.alpha * self.threshold + (1 - self.params.cfg.TEST.alpha) * typical_confidence # typical_confidence = 0 if typical_confidence < self.threshold else typical_confidence '''class ModelManager: _instance = None _model = None
def __new__(cls, cfg, training=True): if cls._instance is None: cls._instance = super(ModelManager, cls).__new__(cls) cls._model = build_ostrack(cfg, training) return cls._instance def get_model(self): return self._modeldef update_threshold(current_threshold, confidence_values, k=5, alpha=0.95): weights = torch.exp(torch.linspace(-1., 0., k))[::-1] # 指数加权并逆序 weighted_confidences = confidence_values[:k] * weights average_confidence = torch.sum(weighted_confidences) / torch.sum(weights) new_threshold = alpha * current_threshold + (1 - alpha) * average_confidence return new_threshold
def get_bbox_confidence(cx, cy, w, h, confidence_map): # 计算 bbox 的边界 x_min = cx - w / 2 x_max = cx + w / 2 y_min = cy - h / 2 y_max = cy + h / 2
# 映射到特征图坐标 x_min_idx = torch.clamp(torch.floor(x_min/16).int(), min=0, max=15) x_max_idx = torch.clamp(torch.ceil(x_max/16).int(), min=0, max=15) y_min_idx = torch.clamp(torch.floor(y_min/16).int(), min=0, max=15) y_max_idx = torch.clamp(torch.ceil(y_max/16).int(), min=0, max=15) # 提取覆盖区域的置信度值 covered_confidences = confidence_map[y_min_idx:y_max_idx+1, x_min_idx:x_max_idx+1] # 计算最终置信度 bbox_confidence = torch.max(covered_confidences) return bbox_confidenceclass OSTrack(BaseTracker): def init(self, params, dataset_name): super(OSTrack, self).init(params) self.LONGTERM_LIBRARY_NUMS = params.cfg.TEST.LONGTERM_LIBRARY_NUMS self.SHORTTERM_LIBRARY_NUMS = params.cfg.TEST.SHORTTERM_LIBRARY_NUMS # net, st_capacity, lt_capacity, sample_interval
# self.DYNAMIC_NUMS = self.params.cfg.TEST.DYNAMIC_NUMS # self.threshold = .5 model_manager = ModelManager(params.cfg, training=False) network = model_manager.get_model() # network = build_ostrack(params.cfg, training=False) # state_dict = torch.load(params.checkpoint, map_location='cpu') # for key in list(state_dict['net'].keys()): # if 'total_' in key: # del state_dict['net'][key] # network.load_state_dict(state_dict['net'], strict=True) network.load_state_dict(torch.load(self.params.checkpoint, map_location='cpu')['net'], strict=True) self.cfg = params.cfg self.network = network.cuda() self.network.eval() self.preprocessor = Preprocessor() self.state = None self.feat_sz = self.cfg.TEST.SEARCH_SIZE // self.cfg.MODEL.BACKBONE.STRIDE # motion constrain self.output_window = hann2d(torch.tensor([self.feat_sz, self.feat_sz]).long(), centered=True).cuda() # for debug self.debug = params.debug self.use_visdom = params.debug self.frame_id = 0 if self.debug: if not self.use_visdom: self.save_dir = "debug" if not os.path.exists(self.save_dir): os.makedirs(self.save_dir) else: # self.add_hook() self._init_visdom(None, 1) # for save boxes from all queries self.save_all_boxes = params.save_all_boxes self.z_dict1 = {} self.z_list = [] self.thor_wrapper = THOR_Wrapper(net=self.network, st_capacity=params.cfg.TEST.SHORTTERM_LIBRARY_NUMS, lt_capacity=params.cfg.TEST.LONGTERM_LIBRARY_NUMS, sample_interval=params.cfg.TEST.SAMPLE_INTERVAL, lb=params.cfg.TEST.THOR_LOWER_BOUND) def initialize(self, image, start_frame_idx, info: dict): self.frame_no = 0 # forward the template once z_patch_arr, resize_factor, z_amask_arr = sample_target(image, info['init_bbox'], self.params.template_factor, output_sz=self.params.template_size) self.z_patch_arr = z_patch_arr template = self.preprocessor.process(z_patch_arr, z_amask_arr) # template.tensors -> shape:(3, 128, 128) with torch.no_grad(): self.static_z = template.tensors self.z_dict1 = template self.z_rawimg_list = [] self.z_rawimg_list.append(template.tensors) self.stacked_z = template.tensors self.stacked_z_features = self.network.backbone.foward_dynamic_features(template.tensors.unsqueeze(0)) self.normed_div_measure_value = 0.0 # template.tensors -> shape:(1, 3, 128, 128) self.thor_wrapper.setup(template.tensors) self.box_mask_z = None if self.cfg.MODEL.BACKBONE.CE_LOC: template_bbox = self.transform_bbox_to_crop(info['init_bbox'], resize_factor, template.tensors.device).squeeze(1) self.box_mask_z = generate_mask_cond(self.cfg, 1, template.tensors.device, template_bbox) # save states self.state = info['init_bbox'] self.frame_id = start_frame_idx if self.save_all_boxes: '''save all predicted boxes''' all_boxes_save = info['init_bbox'] * self.cfg.MODEL.NUM_OBJECT_QUERIES return {"all_boxes": all_boxes_save} def track(self, image, info: dict = None, feat_z_len=64): self.info = info self.frame_no += 1 H, W, _ = image.shape self.frame_id += 1 x_patch_arr, resize_factor, x_amask_arr = sample_target(image, self.state, self.params.search_factor, output_sz=self.params.search_size) # (x1, y1, w, h) search = self.preprocessor.process(x_patch_arr, x_amask_arr) with torch.no_grad(): x_dict = search if len(info['previous_output']) == 0: # 根据有序字典长度判断是否是第一帧 dynamic_template = self.thor_wrapper.update(self.stacked_z) else: dynamic_template = self.thor_wrapper.update(info['previous_output']['prediction']) out_dict = self.network.inference( dynamic_template=dynamic_template, static_template=self.static_z, search=x_dict.tensors) # add hann windows pred_score_map = out_dict['score_map'] response = self.output_window * pred_score_map pred_boxes = self.network.box_head.cal_bbox(response, out_dict['size_map'], out_dict['offset_map']) pred_boxes = pred_boxes.view(-1, 4) # Baseline: Take the mean of all pred boxes as the final result pred_box = (pred_boxes.mean( dim=0) * self.params.search_size / resize_factor).tolist() # (cx, cy, w, h) [0,1] # get the final box result self.state = clip_box(self.map_box_back(pred_box, resize_factor), H, W, margin=10) # extract the tracking result image -> H,W,C tracking_result_arr, resize_factor, tracking_result_amask_arr = sample_target(image, self.state, self.params.template_factor, output_sz=self.params.template_size) prediction = self.preprocessor.process(tracking_result_arr, tracking_result_amask_arr) top1response = response.max() # self.getCAM2(response, image, self.frame_id) # for debug if self.debug: if not self.use_visdom: x1, y1, w, h = self.state image_BGR = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) cv2.rectangle(image_BGR, (int(x1),int(y1)), (int(x1+w),int(y1+h)), color=(0,0,255), thickness=2) save_path = os.path.join(self.save_dir, "%04d.jpg" % self.frame_id) cv2.imwrite(save_path, image_BGR) else: self.visdom.register((image, info['gt_bbox'].tolist(), self.state), 'Tracking', 1, 'Tracking') self.visdom.register(torch.from_numpy(x_patch_arr).permute(2, 0, 1), 'image', 1, 'search_region') self.visdom.register(torch.from_numpy(self.z_patch_arr).permute(2, 0, 1), 'image', 1, 'template') self.visdom.register(pred_score_map.view(self.feat_sz, self.feat_sz), 'heatmap', 1, 'score_map') self.visdom.register((pred_score_map * self.output_window).view(self.feat_sz, self.feat_sz), 'heatmap', 1, 'score_map_hann') if 'removed_indexes_s' in out_dict and out_dict['removed_indexes_s']: removed_indexes_s = out_dict['removed_indexes_s'] removed_indexes_s = [removed_indexes_s_i.cpu().numpy() for removed_indexes_s_i in removed_indexes_s] masked_search = gen_visualization(x_patch_arr, removed_indexes_s) self.visdom.register(torch.from_numpy(masked_search).permute(2, 0, 1), 'image', 1, 'masked_search') while self.pause_mode: if self.step: self.step = False break if self.save_all_boxes: '''save all predictions''' all_boxes = self.map_box_back_batch(pred_boxes * self.params.search_size / resize_factor, resize_factor) all_boxes_save = all_boxes.view(-1).tolist() # (4N, ) return {"target_bbox": self.state, "all_boxes": all_boxes_save} else: return {"target_bbox": self.state, # "prediction": prediction.tensors} "prediction": prediction.tensors, "response": response} def getCAM2(features, img, idx): save_path = 'prj/save/' if not os.path.exists(save_path): os.makedirs(save_path) # os.path = /home/tcm/PycharmProjects/siamft/pysot_toolkit/trackers features = features.to("cpu") features = features.squeeze(1).detach().numpy() img = cv2.resize(img, (256, 256)) img = img img = np.array(img, dtype=np.uint8) # mask = features.sum(dim=0, keepdims=False) mask = features # mask = mask.detach().cpu().numpy() mask = mask.transpose((1, 2, 0)) mask = (mask - mask.min()) / (mask.max() - mask.min()) mask = cv2.resize(mask, (256,256)) mask = 255 * mask mask = mask.astype(np.uint8) heatmap = cv2.applyColorMap(255-mask, cv2.COLORMAP_JET) img = cv2.addWeighted(src1=img, alpha=0.6, src2=heatmap, beta=0.4, gamma=0) name = '/attn_%d.png' % idx cv2.imwrite(save_path + name, img) # cv2.imwrite(name, img) def pltshow(self, pred_map, name): import matplotlib.pyplot as plt plt.figure(2) pred_frame = plt.gca() plt.imshow(pred_map, 'jet') pred_frame.axes.get_yaxis().set_visible(False) pred_frame.axes.get_xaxis().set_visible(False) pred_frame.spines['top'].set_visible(False) pred_frame.spines['bottom'].set_visible(False) pred_frame.spines['left'].set_visible(False) pred_frame.spines['right'].set_visible(False) pred_name = os.path.dirname(__file__) + '/response/' + str(name) + '.png' plt.savefig(pred_name, bbox_inches='tight', pad_inches=0, dpi=150) plt.close(100) # import matplotlib.pyplot as plt # import numpy as np # import cv2 # # 提取分数图 # score_map = out_dict['score_map'] # score_map = score_map.squeeze().cpu().numpy() # 转换为 (16, 16) # # 使用 matplotlib 生成热力图 # plt.imshow(score_map, cmap='jet', interpolation='bicubic') # 使用 'bicubic' 插值 # plt.axis('off') # # 保存热力图到文件 # plt.savefig('heatmap.png', bbox_inches='tight', pad_inches=0, transparent=True) # plt.close() # # 读取热力图 # heatmap = cv2.imread('heatmap.png', cv2.IMREAD_UNCHANGED) # 使用透明背景 # heatmap = cv2.resize(heatmap, (image.shape[1], image.shape[0])) # 调整大小为 (W, H) # # 提取 alpha 通道作为遮罩 # alpha_channel = heatmap[:, :, 3] / 255.0 # 归一化到 [0, 1] # # 复制 event image 以便融合 # fused_image = image.copy() # # 使用简单加权平均方法进行融合 # fused_image = (heatmap[:, :, :3] // 2 + image // 2).astype(np.uint8) # # 保存结果到文件 # cv2.imwrite('heatmap_overlay.jpg', fused_image) # heatmap[:,:,:3]//2 + image//2 # self.activation_map() # for debug if self.debug: if not self.use_visdom: x1, y1, w, h = self.state image_BGR = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) cv2.rectangle(image_BGR, (int(x1),int(y1)), (int(x1+w),int(y1+h)), color=(0,0,255), thickness=2) save_path = os.path.join(self.save_dir, "%04d.jpg" % self.frame_id) cv2.imwrite(save_path, image_BGR) else: self.visdom.register((image, info['gt_bbox'].tolist(), self.state), 'Tracking', 1, 'Tracking') self.visdom.register(torch.from_numpy(x_patch_arr).permute(2, 0, 1), 'image', 1, 'search_region') self.visdom.register(torch.from_numpy(self.z_patch_arr).permute(2, 0, 1), 'image', 1, 'template') self.visdom.register(pred_score_map.view(self.feat_sz, self.feat_sz), 'heatmap', 1, 'score_map') self.visdom.register((pred_score_map * self.output_window).view(self.feat_sz, self.feat_sz), 'heatmap', 1, 'score_map_hann') if 'removed_indexes_s' in out_dict and out_dict['removed_indexes_s']: removed_indexes_s = out_dict['removed_indexes_s'] removed_indexes_s = [removed_indexes_s_i.cpu().numpy() for removed_indexes_s_i in removed_indexes_s] masked_search = gen_visualization(x_patch_arr, removed_indexes_s) self.visdom.register(torch.from_numpy(masked_search).permute(2, 0, 1), 'image', 1, 'masked_search') while self.pause_mode: if self.step: self.step = False break if self.save_all_boxes: '''save all predictions''' all_boxes = self.map_box_back_batch(pred_boxes * self.params.search_size / resize_factor, resize_factor) all_boxes_save = all_boxes.view(-1).tolist() # (4N, ) return {"target_bbox": self.state, "all_boxes": all_boxes_save} else: return {"target_bbox": self.state, # "prediction": prediction.tensors} "prediction": prediction.tensors} def map_box_back(self, pred_box: list, resize_factor: float): cx_prev, cy_prev = self.state[0] + 0.5 * self.state[2], self.state[1] + 0.5 * self.state[3] cx, cy, w, h = pred_box half_side = 0.5 * self.params.search_size / resize_factor cx_real = cx + (cx_prev - half_side) cy_real = cy + (cy_prev - half_side) return [cx_real - 0.5 * w, cy_real - 0.5 * h, w, h] def map_box_back_batch(self, pred_box: torch.Tensor, resize_factor: float): cx_prev, cy_prev = self.state[0] + 0.5 * self.state[2], self.state[1] + 0.5 * self.state[3] cx, cy, w, h = pred_box.unbind(-1) # (N,4) --> (N,) half_side = 0.5 * self.params.search_size / resize_factor cx_real = cx + (cx_prev - half_side) cy_real = cy + (cy_prev - half_side) return torch.stack([cx_real - 0.5 * w, cy_real - 0.5 * h, w, h], dim=-1) def add_hook(self): conv_features, enc_attn_weights, dec_attn_weights = [], [], [] for i in range(12): self.network.backbone.blocks[i].attn.register_forward_hook( # lambda self, input, output: enc_attn_weights.append(output[1]) lambda self, input, output: enc_attn_weights.append(output[1]) ) self.enc_attn_weights = enc_attn_weightsdef get_tracker_class(): return OSTrack
- Assistant
这段代码实现的是一个名为
OSTrack的跟踪器。在高层次上,它结合了动态模板管理功能,以高效地在视频中跟踪目标。这里使用了THOR_Wrapper来管理短期和长期的模板。根据代码,可以将其理解为一个用于追踪的深度学习系统,主要由以下部分组成:类
ModelManager- 单例模式:确保在程序运行中,模型只被构建一次。
- 方法
get_model:返回已经构建好的模型实例。
函数
update_threshold- 用于更新置信度阈值,使用指数加权平均来对过去的一些置信度值进行加权平均更新当前的阈值。
函数
get_bbox_confidence- 计算边界框在一个整体置信度图上的置信度。
- 映射边界框到特征图坐标上,并从中提取覆盖区域的最高置信度。
类
OSTrack- 初始化:在初始化中,加载模型并为其准备好组件,比如处理器(preprocessor),窗口函数等。
- 方法
initialize:在跟踪的初始化阶段使用,处理初始图像帧以从中提取模板。 - 方法
track:这是主要跟踪逻辑实现,包含以下几个步骤:- 使用
THOR_Wrapper更新短期和长期模板。 - 通过网络推断,获取当前帧上的预测框。
- 将响应图加上汉明窗,以实施空间约束。
- 返回最终的目标边界框及预测结果。
- 使用
- 方法
getCAM2:生成和保存类激活图(CAM)叠加在输入图像上的可视化,用于调试或分析模型内在机制。 - 方法
pltshow:生成用于可视化调试的响应度图(主要用于可视化热映射)。
其他
- 调试功能:通过
debug开关,提供调试和可视化功能。这包括用matplotlib或cv2绘制边界框或热映射。 - 边界框映射:实现将框从特征图坐标转换回原始图像坐标的方法(
map_box_back和map_box_back_batch),以确保跟踪的准确性。
总结
OSTrack的实现提供了一个结合了短期和长期记忆机制的框架,适用于视频跟踪任务。在每个跟踪帧中,它通过与现有的动态模板库进行更新和匹配,确保目标的持续跟踪。调试工具提供了深入了解网络推理过程的能力。 - User
分析在ostrack中使用的thor_wrapper函数的逻辑
- Assistant
在
OSTrack类中,THOR_Wrapper是用来管理短期(Short-Term, ST)和长期(Long-Term, LT)模板的组件。THOR_Wrapper负责在视觉跟踪过程中动态更新和管理这些模板,使得跟踪器在不同时间间隔内适应目标或环境的变化。以下是THOR_Wrapper在OSTrack中的使用逻辑的分析:初始化:
-
THOR_Wrapper在OSTrack的初始化中被实例化。它通过将网络(network)以及短期和长期模板的容量(st_capacity和lt_capacity)、采样间隔(sample_interval)等参数传递进去进行配置。self.thor_wrapper = THOR_Wrapper( net=self.network, st_capacity=params.cfg.TEST.SHORTTERM_LIBRARY_NUMS, lt_capacity=params.cfg.TEST.LONGTERM_LIBRARY_NUMS, sample_interval=params.cfg.TEST.SAMPLE_INTERVAL, lb=params.cfg.TEST.THOR_LOWER_BOUND )
方法
setup:-
在
initialize方法中,THOR_Wrapper的setup被调用来初始化ST和LT模块。 -
setup使用初始模板图像(template.tensors)来填充短期和长期模板模块。这个过程确保在跟踪开始时,ST和LT模块中都有一个模板。self.thor_wrapper.setup(template.tensors)
方法
update:-
update函数在每一帧被track方法调用,用于更新模板库。 -
通过与当前模板的相似度计算和比较,动态选择更新的模板类型(短期或长期)。这对于应对目标的显著变化或背景的变化非常重要。
-
返回的
dynamic_template用于网络的推断过程,使得模型能根据最新的动态环境适应性地进行调整。dynamic_template = self.thor_wrapper.update(info['previous_output']['prediction'])
工作逻辑:
-
短期模块(ST Module):
- 短期模板用于快速变化的目标,通常反映最近的帧特征。
- 在给定的采样间隔内(
sample_interval),会进行更新。 - 更新通过插入新模板来替换最旧的模板,从而保持模板库的新鲜度。
-
长期模块(LT Module):
- 长期模块存储更稳定或更具代表性的模板。
- 经过一系列规则或条件来决定是否替换现有模板,比如计算模板特征的多样性和对目标特征的匹配度。
- 在更新过程中,长期模块会对存储的模板进行筛选,以确保存储的是最有利于长期跟踪稳定性的模板。
作用:
THOR_Wrapper提供的这种模板管理机制使得OSTrack能够在不同场景下保持对目标的鲁棒跟踪。- 通过管理和更新短期和长期模板,它有效地提升了模型对目标变化的适应能力,同时也维持了算法的整体效率。
这种设计使
OSTrack在跟踪任务中具有较强的抗干扰能力和稳定的性能表现,特别是在目标形态或背景频繁改变的情况下。 -
- User
综上,分析一下THOR_Warpper类中涉及到的计算
- Assistant
THOR_Wrapper类的计算和逻辑主要围绕管理和更新短期(ST)和长期(LT)模板模块展开。这些计算根据输入序列中的目标变化动态调整模型,以维持跟踪器的性能。以下是对THOR_Wrapper类中涉及到的主要计算的详细分析:计算步骤:
-
相似度计算:
- 在短期和长期模板更新过程中,最为核心的一个步骤是计算当前帧特征与模板库中每个模板特征之间的相似度。
- 典型地使用皮尔逊相关系数来计算两个特征向量之间的相似性,衡量当前目标与已存储模板的匹配程度。
-
模板更新决策:
- 短期模块更新:
- 在
update方法中,短期模块的模板直接替换最旧的条目,以保持最新的状态信息。这是一个策略上简单但有效的更新策略。
- 在
- 长期模块更新:
- 更复杂,通过计算样本与基地模板之间的相似度,以及其与现有库中模板间的多样性来决定是否替换旧模板。
- 计算 Gram 矩阵,并通过其行列式评估替换前后的多样性。
- 多样性和相似度的计算用于确保库中不仅存储的是当前最重要的特征,还维持了模板的多样性以保障长期效率。
- 短期模块更新:
-
多样性度量(Diversity Measure):
- 通过计算 Gram 矩阵的行列式来衡量当前模板库的多样性。
- 计算多样性涉及到对模板特征的两两相似度计算,再通过规范化处理来估计整体的多样性值。
-
动态阈值调整:
- 在某些环境设置下,如调整长短期库的容量或更新间隔,可能涉及根据过去的置信度计算动态调整置信度阈值的逻辑。
-
平滑和标准化:
- 对一些计算结果(如卷积特征)的标准化处理,以消除计算中由于尺度或其他因素带来的偏差。
- 使用均值、标准差等统计数据来实现这种标准化过程。
目的:
- 鲁棒跟踪:通过 ST 和 LT 的分离处理机制,
THOR_Wrapper能有效适应动态目标和背景变化。 - 高效决策:借助相似度和多样性度量,仅在最适合的情况下更新模板,这样可以减少不必要的计算开销。
- 多样性保持:通过复杂的 LT 模块更新逻辑,维持模板间的多样性,从而增强系统的泛化能力。
这些计算共同服务于动态调整和管理跟踪器的模板库的目标,以提升目标跟踪的鲁棒性和效率。通过这种设计,
OSTrack能够在各种复杂场景下保持追踪的稳定性。 -
- User
import abc import random import numpy as np import torch import torch.nn.functional as F import math
class THOR_Wrapper(): def init(self, net, st_capacity, lt_capacity, sample_interval, lb=0.90, response_threshold=0.8): self.response_threshold = response_threshold self.st_capacity = st_capacity self.lt_capacity = lt_capacity self._curr_type = 'st' self._frame_no = 0 self.interval = sample_interval self.net = net self.lb = lb self.lt_count = 0 self.st_count = 0
def setup(self, rawImage): """ initialize the short-term and long-term module """ self.st_module = ST_Module(template_capacity=self.st_capacity, network=self.net) self.st_module.fill(rawImage) self.lt_module = LT_Module(template_capacity=self.lt_capacity, network=self.net, lb=self.lb) self.lt_module.fill(rawImage) def update(self, rawImage): """ update the short-term and long-term module """ ''' 相似度方法 # 使用相似度来作为判断依据,发现最终都会使用LT ''' if self._frame_no % self.interval == 0: div_scale = self.st_module.update(rawImage) # print('frame no: {}, div_scale: {}'.format(self._frame_no, div_scale)) self.lt_module.update(rawImage, div_scale=div_scale, time=self._frame_no) cur_feature = self.net.backbone.forward_dynamic_features_inference([rawImage]) st_curr_sims = self.st_module.pairwise_similarities(cur_feature) lt_curr_sims = self.lt_module.pairwise_similarities(cur_feature) if torch.max(st_curr_sims) >= torch.max(lt_curr_sims): self._curr_type = 'st' self.st_count += 1 return self.st_module.get_dynamic_template_features() else: self._curr_type = 'lt' self.lt_count += 1 return self.lt_module.get_dynamic_template_features()class TemplateModule(): def init(self, network, template_capacity, feat_len_z=64): self.is_full = False self.template_capacity = template_capacity # 模板数量 self.template_raw_list = [] self.sorted_template_raw_list = [] self._templates_features_stack = None self._templates_features_stack_sim = None self._base_sim = 0 self._gram_matrix = None self.feat_len_z = feat_len_z self.network = network self.timestamp_list = []
def __len__(self): return self.template_capacity def pairwise_similarities(self, T_n, to_cpu=False): """ 卷积法计算相似度 calculate similarity of given template to all templates in memory 将一个(1, N, C)的模板T_n与内存中的所有模板进行比较,返回一个(M,)的相似度向量 assert isinstance(T_n, torch.Tensor) B, N, C = T_n.size() H = W = int(math.sqrt(N)) T_n = T_n.view(1, C, H, W) # nums = int(self._templates_features_stack.size()[1]/self.feat_len_z) # self._templates_features_stack = self._templates_features_stack.view(nums, C, H, W) sims = F.conv2d(T_n, self._templates_features_stack_sim) sims = sims.flatten() return sims """ # 将A的形状从(1, N, C)调整为(N, C) T = T_n.clone().squeeze(0) # 将LIST的形状从(1, N*m, C)调整为(N*m, C) LIST = self._templates_features_stack.clone().squeeze(0) # 初始化sims变量,用于存储每个LIST元素与A的皮尔逊相关性系数 sims = torch.empty((LIST.size(0) // T.size(0),)) # 计算A的均值和标准差 mean_T_n = torch.mean(T, dim=0) std_T_n = torch.std(T, dim=0) # 遍历LIST中的每个元素(假设每个元素大小也是(N, C)) for i in range(0, LIST.size(0), T.size(0)): # 提取LIST中的对应元素 element = LIST[i:i+T.size(0)] # 计算元素的均值和标准差 mean_element = torch.mean(element, dim=0) std_element = torch.std(element, dim=0) # 计算皮尔逊相关性系数 cov = torch.sum((T - mean_T_n) * (element - mean_element), dim=0) pearson_coeff = cov / (std_T_n * std_element) # 将计算结果赋值给sims sims[i // T.size(0)] = torch.mean(pearson_coeff) # 取平均是为了处理多通道C的情况 return sims def _calculate_gram_matrix(self): ''' self._templates_features_stack.shape : (n*64, 384) ''' # 使用PyTorch计算所有模板的成对相似度 nums = int(self._templates_features_stack.size()[1]/self.feat_len_z) dists = [] for idx in range(nums): left = idx*self.feat_len_z right = (idx+1)*self.feat_len_z dists.append(self.pairwise_similarities(self._templates_features_stack[:,left:right])) # dists.append(self.pairwise_similarities(self._templates_features_stack[:,idx*self.feat_len_z:(idx+1)*self.feat_len_z])) # dists = [self.pairwise_similarities(self._templates_features_stack[:,idx*self.feat_len_z:(idx+1)*self.feat_len_z]) for idx in range(nums)] # 使用PyTorch的squeeze函数,并且不需要转换成numpy数组 gram_matrices = [dists[i].squeeze() for i in range(len(dists))] gram_matrices = torch.stack(gram_matrices, dim=0) return gram_matrices def _set_temp(self, temp, idx, time): """ switch out the template at idx """ self.template_raw_list[idx] = temp self.timestamp_list[idx] = time @abc.abstractmethod def update(self, temp): """ check if template should be taken into memory """ raise NotImplementedError("Must be implemented in subclass.") @abc.abstractmethod def fill(self, temp): """ fill all slots in the memory with the given template # 填充函数 """ for _ in range(self.template_capacity): self.template_raw_list.append(temp) self.sorted_template_raw_list = self.template_raw_list self._gen_templates_features_stack() self._gram_matrix = self._calculate_gram_matrix() self._base_sim = self._gram_matrix[0, 0] self.timestamp_list = [0] * self.template_capacity def _gen_templates_features_stack(self): self._templates_features_stack = [] for temp in self.template_raw_list: self._templates_features_stack.append(self.network.backbone.forward_dynamic_features_inference([temp])) self._templates_features_stack = torch.cat(self._templates_features_stack, dim=1) B, N, C = self._templates_features_stack.size() N = N//self.feat_len_z H = W = int(math.sqrt(self.feat_len_z)) self._templates_features_stack_sim = self._templates_features_stack.clone().view(N, C, H, W) def get_dynamic_template_features(self): return self.network.backbone.forward_dynamic_features_inference(self.template_raw_list)[:, -self.feat_len_z:] # return self._templates_features_stack[:, -self.feat_len_z:]class ST_Module(TemplateModule): def init(self, network, template_capacity): super(ST_Module, self).init(network, template_capacity)
@staticmethod def normed_div_measure(matrix): """ calculate the normed diversity measure of matirx, the higher the more diverse """ # 确认输入张量t是方阵 assert matrix.shape[0] == matrix.shape[1], "Input tensor must be square" dim = matrix.shape[0] - 1 triu_no = int(dim * (dim + 1) / 2) # 注意这个 dim=n-1, n为方阵维度,此处计算的是原方阵上三角数目 # 使用PyTorch的triu函数来计算上三角矩阵的和 triu_sum = torch.sum(torch.triu(matrix, 1)) # 计算归一化多样性度量 # measure = 1 - triu_sum / (matirx.max() * triu_no) measure = triu_sum / (matrix[0, 0] * triu_no) # 多样性计算公式, np.triu(t, 1)提取上三角部分。公式为: 上三角元素和 / 首元素*上三角元素数目 return measure def _update_gram_matrix(self, new_template): ''' new_template.shape -> (B, N, C) ''' # 计算当前的距离 curr_sims = self.pairwise_similarities(new_template) # sorted_indices = torch.argsort(curr_sims, descending=True) # # 使用排序索引对self.template_raw_list进行排序 # self.sorted_template_raw_list = [self.template_raw_list[i] for i in sorted_indices] curr_sims = curr_sims.unsqueeze(1) # 使用 PyTorch 的 unsqueeze 方法 # 更新 Gram 矩阵 all_dists_new = torch.cat([ self._gram_matrix, curr_sims], dim=1) # 水平拼接 curr_sims = torch.cat((torch.zeros(1, 1).to(curr_sims.device), curr_sims.T), dim=1) # 水平拼接 all_dists_new = torch.cat([ all_dists_new, curr_sims], dim=0) # 垂直拼接 # 删除索引为 0 的行和列,即最旧的模板 self._gram_matrix = all_dists_new[1:, 1:] def update(self, temp): """ append to the current memory and rebuild canvas return div_scale (diversity of the current memory) """ # update distance matrix and calculate the div scale self.template_raw_list.append(temp) self.template_raw_list.pop(0) self._gen_templates_features_stack() temp = self.network.backbone.forward_dynamic_features_inference([temp]) self._update_gram_matrix(temp) # self._templates_features_stack = self.network.backbone.forward_dynamic_features_inference(self.template_raw_list) return self.normed_div_measure(matrix=self._gram_matrix)class LT_Module(TemplateModule): def init(self, network, template_capacity, lb=0.883665, lb_type='dynamic'): ''' lb : 0.90, 0.75, 0.60, 0.45 '''
super(LT_Module, self).__init__(network, template_capacity) self._lb = lb self._lb_type = lb_type self._filled_idx = 0 # def get_dynamic_template_features(self): # # shuffle_list = self.template_raw_list.copy() # sorted_indices = sorted(range(len(self.timestamp_list)), key=lambda k: self.timestamp_list[k]) # sorted_timestamp_list = [self.template_raw_list[i] for i in sorted_indices] # return self.network.backbone.forward_dynamic_features_inference(sorted_timestamp_list)[:, -self.feat_len_z:] def _throw_away_or_keep(self, curr_sims, self_sim, div_scale): """ determine if we keep the template or not if the template is rejected: return -1 (not better) or -2 (rejected by lower bound) if we keep the template: return idx where to switch """ base_sim = self._base_sim curr_sims = curr_sims.unsqueeze(1) # normalize the gram_matrix, otherwise determinants are huge gram_matrix_norm = self._gram_matrix / base_sim curr_sims_norm = curr_sims / base_sim # check if distance to base template is below lower bound if self._lb_type == 'static': reject = (curr_sims[0] < self._lb * base_sim) elif self._lb_type == 'dynamic': lb = self._lb - (1 - div_scale) lb = torch.clamp(lb, 0.0, 1.0) reject = (curr_sims[0] < lb * base_sim) # print('Reject: ', reject, 'curr_sims: ', curr_sims[0].item(), # 'lb: ', lb.item(), 'base_sim: ', base_sim.item(), 'lb * base_sim: ', (lb * base_sim).item()) elif self._lb_type == 'ensemble': reject = not torch.all(curr_sims_norm > self._lb) else: raise TypeError(f"lower boundary type {self._lb_type} not known.") if reject: return -2 # fill the memory with adjacent frames if they are not # populated with something different than the base frame yet if self._filled_idx < (self.template_capacity - 1): self._filled_idx += 1 throwaway_idx = self._filled_idx # determine if and in which spot the template increases the current gram determinant else: curr_det = torch.det(gram_matrix_norm) # start at 1 so we never throwaway the base template dets = torch.zeros((self.template_capacity - 1,)) for i in range(self.template_capacity - 1): mat = gram_matrix_norm.clone() mat[i + 1, :] = curr_sims_norm.T mat[:, i + 1] = curr_sims_norm.T mat[i + 1, i + 1] = self_sim / base_sim dets[i] = torch.det(mat) # check if any of the new combinations is better than the prev. gram_matrix max_idx = torch.argmax(dets) if curr_det > dets[max_idx]: throwaway_idx = -1 else: throwaway_idx = max_idx + 1 assert throwaway_idx != 0 return throwaway_idx if throwaway_idx != self.template_capacity else -1 # @staticmethod # def save_det(d, p): # if os.path.exists(p): # old_det = np.load(p) # else: # old_det = np.array([]) # np.save(p, np.concatenate([old_det, d.reshape(-1)])) def _update_gram_matrix(self, curr_sims, self_sim, idx): """ update the current gram matrix """ curr_sims = curr_sims.unsqueeze(1) # add the self similarity at throwaway_idx spot curr_sims[idx] = self_sim self._gram_matrix[idx, :] = curr_sims.T self._gram_matrix[:, idx] = curr_sims.T # gram_matrix_norm = self._gram_matrix/self._base_sim # curr_det = np.linalg.det(gram_matrix_norm) # self.save_det(curr_det, 'determinants_dyn.npy') def update(self, temp, div_scale, time): """ decide if the templates is taken into the lt module """ # calculate the "throwaway_idx", the spot that the new template will take temp_features = self.network.backbone.forward_dynamic_features_inference([temp]) curr_templatae = temp_features.clone() B, N, C = curr_templatae.size() H = W = int(math.sqrt(N)) curr_templatae = curr_templatae.view(1, C, H, W) curr_sims = self.pairwise_similarities(temp_features) self_sim = F.conv2d(curr_templatae, curr_templatae).squeeze().item() throwaway_idx = self._throw_away_or_keep(curr_sims=curr_sims, self_sim=self_sim, div_scale=div_scale) # if the idx is -2 or -1, the template is rejected, otherwise we update if throwaway_idx == -2 or throwaway_idx == -1: # print('div_scale: {}, reject: {}'.format(div_scale, 'True')) pass # rejected else: # print('div_scale: {}, reject: {}'.format(div_scale, 'False')) self._set_temp(temp=temp, idx=throwaway_idx, time=time) # sorted_indices = torch.argsort(curr_sims, descending=True) # 使用排序索引对self.template_raw_list进行排序 # self.sorted_template_raw_list = [self.template_raw_list[i] for i in sorted_indices] self._update_gram_matrix(curr_sims=curr_sims, self_sim=self_sim, idx=throwaway_idx) self._gen_templates_features_stack() # self._templates_features_stack = self.network.backbone.forward_dynamic_features_inference(self.sorted_template_raw_list) - Assistant
THOR_Wrapper类实现了一个短期和长期模板动态更新的系统,旨在优化目标跟踪任务,并应对目标或背景的变化。以下是它使用的关键计算步骤和逻辑分析:1. 初始化和设置
- 初始化:
THOR_Wrapper类在初始化中接收一些参数,如短期和长期模板容量、采样间隔等。 - 设置:
setup():初始化短期(ST)和长期(LT)模板模块,各自根据传入的初始图像填充模板空间。
2. 更新过程:
update()方法- 时序更新:使用帧编号和采样间隔控制更新频率。在每个采样间隔时,短期模块会更新相应的模板。
- 计算相似度:
- 通过计算输入特征与当前模板特征的相似性(通过皮尔逊相关系数),决定使用短期模板还是长期模板进行推断。
- 短期和长期模块都有其独立的相似度计算和更新策略。
3. 相似性计算:
pairwise_similarities()- 实现方式:通过计算特征之间的皮尔逊相关系数来衡量输入图像与缓存的模板之间的相似性。
- 用途:用于更新策略决策,即在更新新模板的滚动选择时,帮助选择最近似的模板。
4. 格拉姆矩阵计算:
_calculate_gram_matrix()- 目的:用于量化模板集合的多样性。
- 计算细节:通过计算所有模板的成对相似性,形成一个类似于内积的矩阵,用来衡量和维持多样性。
5. 短期模块(
ST_Module)逻辑-
多样性度量:
normed_div_measure()- 通过格拉姆矩阵的上三角形区域总和来计算多样性。
-
更新策略:
- 在模板列表中加入新模板,并替换最旧的模板。
- 更新相应的格拉姆矩阵以反映最新的模板集合。
6. 长期模块(
LT_Module)逻辑-
更新逻辑:
- 使用
_throw_away_or_keep()方法来决定是否保留新模板。判断依据包括与原始样板的相似度、相对多样性以及是否超出设定的下限。 - 在计算中利用格拉姆矩阵的行列式,并保留可带来相对优势或多样性的更新。
- 使用
-
决策逻辑:
- 通过一系列计算(如相似性计算、填充位置判断)判断是否更新长期模板库,确保模版库始终保持代表性和多样性。
通过这些计算和策略,
THOR_Wrapper类能够在不同情况下动态管理模板库,提升目标跟踪的鲁棒性和稳定性。模板的更新策略和相似度度量确保模型能够快速适应动态变化的场景,同时长期保持对目标的稳定跟踪。 - 初始化: