① import initpaths import matplotlib.pyplot as plt plt.rcParams['figure.figsize'] = [8,…
- User
① import _init_paths import matplotlib.pyplot as plt plt.rcParams['figure.figsize'] = [8, 8] import argparse
from lib.test.analysis.plot_results import plot_results, print_results, print_per_sequence_results from lib.test.evaluation import get_dataset, trackerlist
if name == 'main': parser = argparse.ArgumentParser(description='Run tracker on sequence or dataset.') parser.add_argument('--dataset', type=str, default='coesot', help='coesot or fe108 or visevent or eed.') parser.add_argument('--parameter_name', type=str, default='ceutrack_coesot', help='coesot or fe108 or visevent or eed.') args = parser.parse_args()
trackers = [] dataset_name = args.dataset # coesot fe108 """pytracking""" # trackers.extend(trackerlist('atom', 'default', None, range(0,5), 'ATOM')) """ceutrack""" trackers.extend(trackerlist(name='ceutrack', parameter_name=args.parameter_name, dataset_name=dataset_name, run_ids=None, display_name='CEUTrack256')) dataset = get_dataset(dataset_name) print_results(trackers, dataset, dataset_name, merge_results=True, plot_types=('success', 'norm_prec', 'prec')) 和 ② import importlibimport os from collections import OrderedDict from lib.test.evaluation.environment import env_settings import time import cv2 as cv
from lib.utils.lmdb_utils import decode_img from pathlib import Path import numpy as np import scipy.io as scio import torch.nn.functional as F import torch
def trackerlist(name: str, parameter_name: str, dataset_name: str, run_ids = None, display_name: str = None, result_only=False): """Generate list of trackers. args: name: Name of tracking method. parameter_name: Name of parameter file. run_ids: A single or list of run_ids. display_name: Name to be displayed in the result plots. """ if run_ids is None or isinstance(run_ids, int): run_ids = [run_ids] return [Tracker(name, parameter_name, dataset_name, run_id, display_name, result_only) for run_id in run_ids]
class Tracker: """Wraps the tracker for evaluation and running purposes. args: name: Name of tracking method. parameter_name: Name of parameter file. run_id: The run id. display_name: Name to be displayed in the result plots. """
def __init__(self, name: str, parameter_name: str, dataset_name: str, run_id: int = None, display_name: str = None, result_only=False): assert run_id is None or isinstance(run_id, int) self.name = name self.parameter_name = parameter_name self.dataset_name = dataset_name self.run_id = run_id self.display_name = display_name env = env_settings() if self.run_id is None: self.results_dir = '{}/{}/{}'.format(env.results_path, self.name, self.parameter_name) else: self.results_dir = '{}/{}/{}_{:03d}'.format(env.results_path, self.name, self.parameter_name, self.run_id) if result_only: self.results_dir = '{}/{}'.format(env.results_path, self.name) tracker_module_abspath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'tracker', '%s.py' % self.name)) if os.path.isfile(tracker_module_abspath): tracker_module = importlib.import_module('lib.test.tracker.{}'.format(self.name)) self.tracker_class = tracker_module.get_tracker_class() else: self.tracker_class = None def create_tracker(self, params): tracker = self.tracker_class(params, self.dataset_name) return tracker def run_sequence(self, seq, debug=None): """Run tracker on sequence. args: seq: Sequence to run the tracker on. visualization: Set visualization flag (None means default value specified in the parameters). debug: Set debug level (None means default value specified in the parameters). multiobj_mode: Which mode to use for multiple objects. """ params = self.get_parameters() debug_ = debug if debug is None: debug_ = getattr(params, 'debug', 0) params.debug = debug_ # Get init information init_info = seq.init_info(0) tracker = self.create_tracker(params) output = self._track_sequence(tracker, seq, init_info) return output def _track_sequence(self, tracker, seq, init_info): # Define outputs # Each field in output is a list containing tracker prediction for each frame. # In case of single object tracking mode: # target_bbox[i] is the predicted bounding box for frame i # time[i] is the processing time for frame i # In case of multi object tracking mode: # target_bbox[i] is an OrderedDict, where target_bbox[i][obj_id] is the predicted box for target obj_id in # frame i # time[i] is either the processing time for frame i, or an OrderedDict containing processing times for each # object in frame i output = {'target_bbox': [], 'time': []} if tracker.params.save_all_boxes: output['all_boxes'] = [] output['all_scores'] = [] def _store_outputs(tracker_out: dict, defaults=None): defaults = {} if defaults is None else defaults for key in output.keys(): val = tracker_out.get(key, defaults.get(key, None)) if key in tracker_out or val is not None: output[key].append(val) right_template = False # flag for while i = 0 # template index, # Initialize while not right_template: image = self._read_image(seq.frames[i]) # event_image = self._read_image(seq.event_img_list[i]) # image = cv.addWeighted(image, 1, event_image, 0.2, 0) # event_template event_template = scio.loadmat(seq.event_frames[i]) gt_bbox = seq.ground_truth_rect[i] if (os.path.getsize(seq.event_frames[i]) == 0) or (np.isnan(event_template['features']).any()): event_template = torch.zeros([1024, 19]) right_template = True print(seq.event_frames[i], i, 'template voxel is empty/nan, only based on rgb frames for tracking.') elif gt_bbox[2]*gt_bbox[3] < 1: right_template = False print(seq.event_frames[i], 'idx bbox zero, without any target or too small.') i = i+1 else: event_template = np.concatenate((event_template['coor'], event_template['features']), axis=1) event_template = torch.from_numpy(event_template) right_template = True init_info['init_bbox'] = gt_bbox start_time = time.time() out = tracker.initialize(image, event_template, init_info, idx=i) if out is None: out = {} prev_output = OrderedDict(out) if i != 0: for idx in range(0, i+1): target_box_template = seq.ground_truth_rect[idx] init_info = {'target_bbox': target_box_template, 'time': 0} _store_outputs(out, init_info) else: target_box_template = init_info.get('init_bbox') init_default = {'target_bbox': target_box_template, 'time': time.time() - start_time} _store_outputs(out, init_default) if tracker.params.save_all_boxes: init_default['all_boxes'] = out['all_boxes'] init_default['all_scores'] = out['all_scores'] _store_outputs(out, init_default) for frame_num, (frame_path, frame_event_path) in enumerate(zip(seq.frames[i+1:], seq.event_frames[i+1:]), start=1): # for frame_num, (frame_path, frame_event_path, event_img_path) in enumerate(zip(seq.frames[i+1:], seq.event_frames[i+1:], seq.event_img_list[i+1:]), start=1): image = self._read_image(frame_path) # event_img = self._read_image(event_img_path) # image = cv.addWeighted(image, 1, event_img, 0.2, 0) # event_search if os.path.getsize(frame_event_path) == 0: event_search = torch.zeros([4096, 19]) print(frame_event_path, 'idx frame of search voxel is empty. only based rgb') else: event_search = scio.loadmat(frame_event_path) event_search = np.concatenate((event_search['coor'], event_search['features']), axis=1) event_search = torch.from_numpy(event_search) if np.isnan(event_search).any(): event_search = torch.zeros([4096, 19]) print(frame_event_path, 'idx frame of search voxel exist nan, only based on rgb.') start_time = time.time() info = seq.frame_info(frame_num) info['previous_output'] = prev_output if len(seq.ground_truth_rect) > 1: info['gt_bbox'] = seq.ground_truth_rect[frame_num] out = tracker.track(image, event_search, info) prev_output = OrderedDict(out) _store_outputs(out, {'time': time.time() - start_time}) for key in ['target_bbox', 'all_boxes', 'all_scores']: if key in output and len(output[key]) <= 1: output.pop(key) return output def run_video(self, videofilepath, optional_box=None, debug=None, visdom_info=None, save_results=False): """Run the tracker with the vieofile. args: debug: Debug level. """ params = self.get_parameters() debug_ = debug if debug is None: debug_ = getattr(params, 'debug', 0) params.debug = debug_ params.tracker_name = self.name params.param_name = self.parameter_name # self._init_visdom(visdom_info, debug_) multiobj_mode = getattr(params, 'multiobj_mode', getattr(self.tracker_class, 'multiobj_mode', 'default')) if multiobj_mode == 'default': tracker = self.create_tracker(params) elif multiobj_mode == 'parallel': tracker = MultiObjectWrapper(self.tracker_class, params, self.visdom, fast_load=True) else: raise ValueError('Unknown multi object mode {}'.format(multiobj_mode)) assert os.path.isfile(videofilepath), "Invalid param {}".format(videofilepath) ", videofilepath must be a valid videofile" output_boxes = [] cap = cv.VideoCapture(videofilepath) display_name = 'Display: ' + tracker.params.tracker_name cv.namedWindow(display_name, cv.WINDOW_NORMAL | cv.WINDOW_KEEPRATIO) cv.resizeWindow(display_name, 960, 720) success, frame = cap.read() cv.imshow(display_name, frame) def _build_init_info(box): return {'init_bbox': box} if success is not True: print("Read frame from {} failed.".format(videofilepath)) exit(-1) if optional_box is not None: assert isinstance(optional_box, (list, tuple)) assert len(optional_box) == 4, "valid box's foramt is [x,y,w,h]" tracker.initialize(frame, _build_init_info(optional_box)) output_boxes.append(optional_box) else: while True: # cv.waitKey() frame_disp = frame.copy() cv.putText(frame_disp, 'Select target ROI and press ENTER', (20, 30), cv.FONT_HERSHEY_COMPLEX_SMALL, 1.5, (0, 0, 0), 1) x, y, w, h = cv.selectROI(display_name, frame_disp, fromCenter=False) init_state = [x, y, w, h] tracker.initialize(frame, _build_init_info(init_state)) output_boxes.append(init_state) break while True: ret, frame = cap.read() if frame is None: break frame_disp = frame.copy() # Draw box out = tracker.track(frame) state = [int(s) for s in out['target_bbox']] output_boxes.append(state) cv.rectangle(frame_disp, (state[0], state[1]), (state[2] + state[0], state[3] + state[1]), (0, 255, 0), 5) font_color = (0, 0, 0) cv.putText(frame_disp, 'Tracking!', (20, 30), cv.FONT_HERSHEY_COMPLEX_SMALL, 1, font_color, 1) cv.putText(frame_disp, 'Press r to reset', (20, 55), cv.FONT_HERSHEY_COMPLEX_SMALL, 1, font_color, 1) cv.putText(frame_disp, 'Press q to quit', (20, 80), cv.FONT_HERSHEY_COMPLEX_SMALL, 1, font_color, 1) # Display the resulting frame cv.imshow(display_name, frame_disp) key = cv.waitKey(1) if key == ord('q'): break elif key == ord('r'): ret, frame = cap.read() frame_disp = frame.copy() cv.putText(frame_disp, 'Select target ROI and press ENTER', (20, 30), cv.FONT_HERSHEY_COMPLEX_SMALL, 1.5, (0, 0, 0), 1) cv.imshow(display_name, frame_disp) x, y, w, h = cv.selectROI(display_name, frame_disp, fromCenter=False) init_state = [x, y, w, h] tracker.initialize(frame, _build_init_info(init_state)) output_boxes.append(init_state) # When everything done, release the capture cap.release() cv.destroyAllWindows() if save_results: if not os.path.exists(self.results_dir): os.makedirs(self.results_dir) video_name = Path(videofilepath).stem base_results_path = os.path.join(self.results_dir, 'video_{}'.format(video_name)) tracked_bb = np.array(output_boxes).astype(int) bbox_file = '{}.txt'.format(base_results_path) np.savetxt(bbox_file, tracked_bb, delimiter='\t', fmt='%d') def get_parameters(self): """Get parameters.""" param_module = importlib.import_module('lib.test.parameter.{}'.format(self.name)) params = param_module.parameters(self.parameter_name) return params def _read_image(self, image_file: str): if isinstance(image_file, str): im = cv.imread(image_file) return cv.cvtColor(im, cv.COLOR_BGR2RGB) elif isinstance(image_file, list) and len(image_file) == 2: return decode_img(image_file[0], image_file[1]) else: raise ValueError("type of image_file should be str or list")③ import tikzplotlib import matplotlib import matplotlib.pyplot as plt import os import torch import pickle import json from lib.test.evaluation.environment import env_settings from lib.test.analysis.extract_results import extract_results
def get_plot_draw_styles(): plot_draw_style = [{'color': (1.0, 0.0, 0.0), 'line_style': '-'}, {'color': (0.0, 1.0, 0.0), 'line_style': '-'}, {'color': (0.0, 0.0, 1.0), 'line_style': '-'}, {'color': (0.0, 0.0, 0.0), 'line_style': '-'}, {'color': (1.0, 0.0, 1.0), 'line_style': '-'}, {'color': (0.0, 1.0, 1.0), 'line_style': '-'}, {'color': (0.5, 0.5, 0.5), 'line_style': '-'}, {'color': (136.0 / 255.0, 0.0, 21.0 / 255.0), 'line_style': '-'}, {'color': (1.0, 127.0 / 255.0, 39.0 / 255.0), 'line_style': '-'}, {'color': (0.0, 162.0 / 255.0, 232.0 / 255.0), 'line_style': '-'}, {'color': (0.0, 0.5, 0.0), 'line_style': '-'}, {'color': (1.0, 0.5, 0.2), 'line_style': '-'}, {'color': (0.1, 0.4, 0.0), 'line_style': '-'}, {'color': (0.6, 0.3, 0.9), 'line_style': '-'}, {'color': (0.4, 0.7, 0.1), 'line_style': '-'}, {'color': (0.2, 0.1, 0.7), 'line_style': '-'}, {'color': (0.7, 0.6, 0.2), 'line_style': '-'}]
return plot_draw_styledef check_eval_data_is_valid(eval_data, trackers, dataset): """ Checks if the pre-computed results are valid""" seq_names = [s.name for s in dataset] seq_names_saved = eval_data['sequences']
tracker_names_f = [(t.name, t.parameter_name, t.run_id) for t in trackers] tracker_names_f_saved = [(t['name'], t['param'], t['run_id']) for t in eval_data['trackers']] return seq_names == seq_names_saved and tracker_names_f == tracker_names_f_saveddef merge_multiple_runs(eval_data): new_tracker_names = [] ave_success_rate_plot_overlap_merged = [] ave_success_rate_plot_center_merged = [] ave_success_rate_plot_center_norm_merged = [] avg_overlap_all_merged = []
ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap']) ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center']) ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm']) avg_overlap_all = torch.tensor(eval_data['avg_overlap_all']) trackers = eval_data['trackers'] merged = torch.zeros(len(trackers), dtype=torch.uint8) for i in range(len(trackers)): if merged[i]: continue base_tracker = trackers[i] new_tracker_names.append(base_tracker) match = [t['name'] == base_tracker['name'] and t['param'] == base_tracker['param'] for t in trackers] match = torch.tensor(match) ave_success_rate_plot_overlap_merged.append(ave_success_rate_plot_overlap[:, match, :].mean(1)) ave_success_rate_plot_center_merged.append(ave_success_rate_plot_center[:, match, :].mean(1)) ave_success_rate_plot_center_norm_merged.append(ave_success_rate_plot_center_norm[:, match, :].mean(1)) avg_overlap_all_merged.append(avg_overlap_all[:, match].mean(1)) merged[match] = 1 ave_success_rate_plot_overlap_merged = torch.stack(ave_success_rate_plot_overlap_merged, dim=1) ave_success_rate_plot_center_merged = torch.stack(ave_success_rate_plot_center_merged, dim=1) ave_success_rate_plot_center_norm_merged = torch.stack(ave_success_rate_plot_center_norm_merged, dim=1) avg_overlap_all_merged = torch.stack(avg_overlap_all_merged, dim=1) eval_data['trackers'] = new_tracker_names eval_data['ave_success_rate_plot_overlap'] = ave_success_rate_plot_overlap_merged.tolist() eval_data['ave_success_rate_plot_center'] = ave_success_rate_plot_center_merged.tolist() eval_data['ave_success_rate_plot_center_norm'] = ave_success_rate_plot_center_norm_merged.tolist() eval_data['avg_overlap_all'] = avg_overlap_all_merged.tolist() return eval_datadef get_tracker_display_name(tracker): if tracker['disp_name'] is None: if tracker['run_id'] is None: disp_name = '{}{}'.format(tracker['name'], tracker['param']) else: disp_name = '{}{}_{:03d}'.format(tracker['name'], tracker['param'], tracker['run_id']) else: disp_name = tracker['disp_name']
return disp_namedef plot_draw_save(y, x, scores, trackers, plot_draw_styles, result_plot_path, plot_opts): plt.rcParams['text.usetex']=True plt.rcParams["font.family"] = "Times New Roman" # Plot settings font_size = plot_opts.get('font_size', 20) font_size_axis = plot_opts.get('font_size_axis', 20) line_width = plot_opts.get('line_width', 2) font_size_legend = plot_opts.get('font_size_legend', 20)
plot_type = plot_opts['plot_type'] legend_loc = plot_opts['legend_loc'] xlabel = plot_opts['xlabel'] ylabel = plot_opts['ylabel'] ylabel = "%s"%(ylabel.replace('%','\%')) xlim = plot_opts['xlim'] ylim = plot_opts['ylim'] title = r"$\bf{%s}$" %(plot_opts['title']) matplotlib.rcParams.update({'font.size': font_size}) matplotlib.rcParams.update({'axes.titlesize': font_size_axis}) matplotlib.rcParams.update({'axes.titleweight': 'black'}) matplotlib.rcParams.update({'axes.labelsize': font_size_axis}) fig, ax = plt.subplots() index_sort = scores.argsort(descending=False) plotted_lines = [] legend_text = [] for id, id_sort in enumerate(index_sort): line = ax.plot(x.tolist(), y[id_sort, :].tolist(), linewidth=line_width, color=plot_draw_styles[index_sort.numel() - id - 1]['color'], linestyle=plot_draw_styles[index_sort.numel() - id - 1]['line_style']) plotted_lines.append(line[0]) tracker = trackers[id_sort] disp_name = get_tracker_display_name(tracker) legend_text.append('{} [{:.1f}]'.format(disp_name, scores[id_sort])) try: # add bold to our method for i in range(1,2): legend_text[-i] = r'\textbf{%s}'%(legend_text[-i]) ax.legend(plotted_lines[::-1], legend_text[::-1], loc=legend_loc, fancybox=False, edgecolor='black', fontsize=font_size_legend, framealpha=1.0) except: pass ax.set(xlabel=xlabel, ylabel=ylabel, xlim=xlim, ylim=ylim, title=title) ax.grid(True, linestyle='-.') fig.tight_layout() tikzplotlib.save('{}/{}_plot.tex'.format(result_plot_path, plot_type)) fig.savefig('{}/{}_plot.pdf'.format(result_plot_path, plot_type), dpi=300, format='pdf', transparent=True) plt.draw()def check_and_load_precomputed_results(trackers, dataset, report_name, force_evaluation=False, **kwargs): # Load data settings = env_settings()
# Load pre-computed results result_plot_path = os.path.join(settings.result_plot_path, report_name) eval_data_path = os.path.join(result_plot_path, 'eval_data.pkl') if os.path.isfile(eval_data_path) and not force_evaluation: with open(eval_data_path, 'rb') as fh: eval_data = pickle.load(fh) else: # print('Pre-computed evaluation data not found. Computing results!') eval_data = extract_results(trackers, dataset, report_name, **kwargs) if not check_eval_data_is_valid(eval_data, trackers, dataset): # print('Pre-computed evaluation data invalid. Re-computing results!') eval_data = extract_results(trackers, dataset, report_name, **kwargs) # pass else: # Update display names tracker_names = [{'name': t.name, 'param': t.parameter_name, 'run_id': t.run_id, 'disp_name': t.display_name} for t in trackers] eval_data['trackers'] = tracker_names with open(eval_data_path, 'wb') as fh: pickle.dump(eval_data, fh) return eval_datadef get_auc_curve(ave_success_rate_plot_overlap, valid_sequence): ave_success_rate_plot_overlap = ave_success_rate_plot_overlap[valid_sequence, :, :] auc_curve = ave_success_rate_plot_overlap.mean(0) * 100.0 auc = auc_curve.mean(-1)
return auc_curve, aucdef get_prec_curve(ave_success_rate_plot_center, valid_sequence): ave_success_rate_plot_center = ave_success_rate_plot_center[valid_sequence, :, :] prec_curve = ave_success_rate_plot_center.mean(0) * 100.0 prec_score = prec_curve[:, 20]
return prec_curve, prec_scoredef plot_results(trackers, dataset, report_name, merge_results=False, plot_types=('success'), force_evaluation=False, **kwargs): """ Plot results for the given trackers
args: trackers - List of trackers to evaluate dataset - List of sequences to evaluate report_name - Name of the folder in env_settings.perm_mat_path where the computed results and plots are saved merge_results - If True, multiple random runs for a non-deterministic trackers are averaged plot_types - List of scores to display. Can contain 'success', 'prec' (precision), and 'norm_prec' (normalized precision) """ # Load data settings = env_settings() plot_draw_styles = get_plot_draw_styles() # Load pre-computed results result_plot_path = os.path.join(settings.result_plot_path, report_name) eval_data = check_and_load_precomputed_results(trackers, dataset, report_name, force_evaluation, **kwargs) # Merge results from multiple runs if merge_results: eval_data = merge_multiple_runs(eval_data) tracker_names = eval_data['trackers'] valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool) print('\nPlotting results over {} / {} sequences'.format(valid_sequence.long().sum().item(), valid_sequence.shape[0])) print('\nGenerating plots for: {}'.format(report_name)) # ******************************** Success Plot ************************************** if 'success' in plot_types: ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap']) # Index out valid sequences auc_curve, auc = get_auc_curve(ave_success_rate_plot_overlap, valid_sequence) threshold_set_overlap = torch.tensor(eval_data['threshold_set_overlap']) success_plot_opts = {'plot_type': 'success', 'legend_loc': 'lower left', 'xlabel': 'Overlap threshold', 'ylabel': 'Overlap Precision [%]', 'xlim': (0, 1.0), 'ylim': (0, 88), 'title': 'Success'} plot_draw_save(auc_curve, threshold_set_overlap, auc, tracker_names, plot_draw_styles, result_plot_path, success_plot_opts) # ******************************** Precision Plot ************************************** if 'prec' in plot_types: ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center']) # Index out valid sequences prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center, valid_sequence) threshold_set_center = torch.tensor(eval_data['threshold_set_center']) precision_plot_opts = {'plot_type': 'precision', 'legend_loc': 'lower right', 'xlabel': 'Location error threshold [pixels]', 'ylabel': 'Distance Precision [%]', 'xlim': (0, 50), 'ylim': (0, 100), 'title': 'Precision plot'} plot_draw_save(prec_curve, threshold_set_center, prec_score, tracker_names, plot_draw_styles, result_plot_path, precision_plot_opts) # ******************************** Norm Precision Plot ************************************** if 'norm_prec' in plot_types: ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm']) # Index out valid sequences prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center_norm, valid_sequence) threshold_set_center_norm = torch.tensor(eval_data['threshold_set_center_norm']) norm_precision_plot_opts = {'plot_type': 'norm_precision', 'legend_loc': 'lower right', 'xlabel': 'Location error threshold', 'ylabel': 'Distance Precision [%]', 'xlim': (0, 0.5), 'ylim': (0, 85), 'title': 'Normalized Precision'} plot_draw_save(prec_curve, threshold_set_center_norm, prec_score, tracker_names, plot_draw_styles, result_plot_path, norm_precision_plot_opts) plt.show()def generate_formatted_report(row_labels, scores, table_name=''): name_width = max([len(d) for d in row_labels] + [len(table_name)]) + 5 min_score_width = 10
report_text = '\n{label: <{width}} |'.format(label=table_name, width=name_width) score_widths = [max(min_score_width, len(k) + 3) for k in scores.keys()] for s, s_w in zip(scores.keys(), score_widths): report_text = '{prev} {s: <{width}} |'.format(prev=report_text, s=s, width=s_w) report_text = '{prev}\n'.format(prev=report_text) for trk_id, d_name in enumerate(row_labels): # display name report_text = '{prev}{tracker: <{width}} |'.format(prev=report_text, tracker=d_name, width=name_width) for (score_type, score_value), s_w in zip(scores.items(), score_widths): report_text = '{prev} {score: <{width}} |'.format(prev=report_text, score='{:0.2f}'.format(score_value[trk_id].item()), width=s_w) report_text = '{prev}\n'.format(prev=report_text) return report_textdef print_results(trackers, dataset, report_name, merge_results=False, plot_types=('success'), **kwargs): """ Print the results for the given trackers in a formatted table args: trackers - List of trackers to evaluate dataset - List of sequences to evaluate report_name - Name of the folder in env_settings.perm_mat_path where the computed results and plots are saved merge_results - If True, multiple random runs for a non-deterministic trackers are averaged plot_types - List of scores to display. Can contain 'success' (prints AUC, OP50, and OP75 scores), 'prec' (prints precision score), and 'norm_prec' (prints normalized precision score) """ # Load pre-computed results eval_data = check_and_load_precomputed_results(trackers, dataset, report_name, **kwargs)
# Merge results from multiple runs if merge_results: eval_data = merge_multiple_runs(eval_data) tracker_names = eval_data['trackers'] valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool) print('\nReporting results over {} / {} sequences'.format(valid_sequence.long().sum().item(), valid_sequence.shape[0])) scores = {} # ******************************** Success Plot ************************************** if 'success' in plot_types: threshold_set_overlap = torch.tensor(eval_data['threshold_set_overlap']) ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap']) # Index out valid sequences auc_curve, auc = get_auc_curve(ave_success_rate_plot_overlap, valid_sequence) scores['AUC'] = auc scores['OP50'] = auc_curve[:, threshold_set_overlap == 0.50] scores['OP75'] = auc_curve[:, threshold_set_overlap == 0.75] # ******************************** Precision Plot ************************************** if 'prec' in plot_types: ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center']) # Index out valid sequences prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center, valid_sequence) scores['Precision'] = prec_score # ******************************** Norm Precision Plot ********************************* if 'norm_prec' in plot_types: ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm']) # Index out valid sequences norm_prec_curve, norm_prec_score = get_prec_curve(ave_success_rate_plot_center_norm, valid_sequence) scores['Norm Precision'] = norm_prec_score # Print tracker_disp_names = [get_tracker_display_name(trk) for trk in tracker_names] report_text = generate_formatted_report(tracker_disp_names, scores, table_name=report_name) print(report_text)def plot_got_success(trackers, report_name): """ Plot success plot for GOT-10k dataset using the json reports. Save the json reports from http://got-10k.aitestunion.com/leaderboard in the directory set to env_settings.got_reports_path
The tracker name in the experiment file should be set to the name of the report file for that tracker, e.g. DiMP50_report_2019_09_02_15_44_25 if the report is name DiMP50_report_2019_09_02_15_44_25.json args: trackers - List of trackers to evaluate report_name - Name of the folder in env_settings.perm_mat_path where the computed results and plots are saved """ # Load data settings = env_settings() plot_draw_styles = get_plot_draw_styles() result_plot_path = os.path.join(settings.result_plot_path, report_name) auc_curve = torch.zeros((len(trackers), 101)) scores = torch.zeros(len(trackers)) # Load results tracker_names = [] for trk_id, trk in enumerate(trackers): json_path = '{}/{}.json'.format(settings.got_reports_path, trk.name) if os.path.isfile(json_path): with open(json_path, 'r') as f: eval_data = json.load(f) else: raise Exception('Report not found {}'.format(json_path)) if len(eval_data.keys()) > 1: raise Exception # First field is the tracker name. Index it out eval_data = eval_data[list(eval_data.keys())[0]] if 'succ_curve' in eval_data.keys(): curve = eval_data['succ_curve'] ao = eval_data['ao'] elif 'overall' in eval_data.keys() and 'succ_curve' in eval_data['overall'].keys(): curve = eval_data['overall']['succ_curve'] ao = eval_data['overall']['ao'] else: raise Exception('Invalid JSON file {}'.format(json_path)) auc_curve[trk_id, :] = torch.tensor(curve) * 100.0 scores[trk_id] = ao * 100.0 tracker_names.append({'name': trk.name, 'param': trk.parameter_name, 'run_id': trk.run_id, 'disp_name': trk.display_name}) threshold_set_overlap = torch.arange(0.0, 1.01, 0.01, dtype=torch.float64) success_plot_opts = {'plot_type': 'success', 'legend_loc': 'lower left', 'xlabel': 'Overlap threshold', 'ylabel': 'Overlap Precision [%]', 'xlim': (0, 1.0), 'ylim': (0, 100), 'title': 'Success plot'} plot_draw_save(auc_curve, threshold_set_overlap, scores, tracker_names, plot_draw_styles, result_plot_path, success_plot_opts) plt.show()def print_per_sequence_results(trackers, dataset, report_name, merge_results=False, filter_criteria=None, **kwargs): """ Print per-sequence results for the given trackers. Additionally, the sequences to list can be filtered using the filter criteria.
args: trackers - List of trackers to evaluate dataset - List of sequences to evaluate report_name - Name of the folder in env_settings.perm_mat_path where the computed results and plots are saved merge_results - If True, multiple random runs for a non-deterministic trackers are averaged filter_criteria - Filter sequence results which are reported. Following modes are supported None: No filtering. Display results for all sequences in dataset 'ao_min': Only display sequences for which the minimum average overlap (AO) score over the trackers is less than a threshold filter_criteria['threshold']. This mode can be used to select sequences where at least one tracker performs poorly. 'ao_max': Only display sequences for which the maximum average overlap (AO) score over the trackers is less than a threshold filter_criteria['threshold']. This mode can be used to select sequences all tracker performs poorly. 'delta_ao': Only display sequences for which the performance of different trackers vary by at least filter_criteria['threshold'] in average overlap (AO) score. This mode can be used to select sequences where the behaviour of the trackers greatly differ between each other. """ # Load pre-computed results eval_data = check_and_load_precomputed_results(trackers, dataset, report_name, **kwargs) # Merge results from multiple runs if merge_results: eval_data = merge_multiple_runs(eval_data) tracker_names = eval_data['trackers'] valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool) sequence_names = eval_data['sequences'] avg_overlap_all = torch.tensor(eval_data['avg_overlap_all']) * 100.0 # Filter sequences if filter_criteria is not None: if filter_criteria['mode'] == 'ao_min': min_ao = avg_overlap_all.min(dim=1)[0] valid_sequence = valid_sequence & (min_ao < filter_criteria['threshold']) elif filter_criteria['mode'] == 'ao_max': max_ao = avg_overlap_all.max(dim=1)[0] valid_sequence = valid_sequence & (max_ao < filter_criteria['threshold']) elif filter_criteria['mode'] == 'delta_ao': min_ao = avg_overlap_all.min(dim=1)[0] max_ao = avg_overlap_all.max(dim=1)[0] valid_sequence = valid_sequence & ((max_ao - min_ao) > filter_criteria['threshold']) else: raise Exception avg_overlap_all = avg_overlap_all[valid_sequence, :] sequence_names = [s + ' (ID={})'.format(i) for i, (s, v) in enumerate(zip(sequence_names, valid_sequence.tolist())) if v] tracker_disp_names = [get_tracker_display_name(trk) for trk in tracker_names] scores_per_tracker = {k: avg_overlap_all[:, i] for i, k in enumerate(tracker_disp_names)} report_text = generate_formatted_report(sequence_names, scores_per_tracker) print(report_text)def print_results_per_video(trackers, dataset, report_name, merge_results=False, plot_types=('success'), per_video=False, **kwargs): """ Print the results for the given trackers in a formatted table args: trackers - List of trackers to evaluate dataset - List of sequences to evaluate report_name - Name of the folder in env_settings.perm_mat_path where the computed results and plots are saved merge_results - If True, multiple random runs for a non-deterministic trackers are averaged plot_types - List of scores to display. Can contain 'success' (prints AUC, OP50, and OP75 scores), 'prec' (prints precision score), and 'norm_prec' (prints normalized precision score) """ # Load pre-computed results eval_data = check_and_load_precomputed_results(trackers, dataset, report_name, **kwargs)
# Merge results from multiple runs if merge_results: eval_data = merge_multiple_runs(eval_data) seq_lens = len(eval_data['sequences']) eval_datas = [{} for _ in range(seq_lens)] if per_video: for key, value in eval_data.items(): if len(value) == seq_lens: for i in range(seq_lens): eval_datas[i][key] = [value[i]] else: for i in range(seq_lens): eval_datas[i][key] = value tracker_names = eval_data['trackers'] valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool) print('\nReporting results over {} / {} sequences'.format(valid_sequence.long().sum().item(), valid_sequence.shape[0])) scores = {} # ******************************** Success Plot ************************************** if 'success' in plot_types: threshold_set_overlap = torch.tensor(eval_data['threshold_set_overlap']) ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap']) # Index out valid sequences auc_curve, auc = get_auc_curve(ave_success_rate_plot_overlap, valid_sequence) scores['AUC'] = auc scores['OP50'] = auc_curve[:, threshold_set_overlap == 0.50] scores['OP75'] = auc_curve[:, threshold_set_overlap == 0.75] # ******************************** Precision Plot ************************************** if 'prec' in plot_types: ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center']) # Index out valid sequences prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center, valid_sequence) scores['Precision'] = prec_score # ******************************** Norm Precision Plot ********************************* if 'norm_prec' in plot_types: ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm']) # Index out valid sequences norm_prec_curve, norm_prec_score = get_prec_curve(ave_success_rate_plot_center_norm, valid_sequence) scores['Norm Precision'] = norm_prec_score # Print tracker_disp_names = [get_tracker_display_name(trk) for trk in tracker_names] report_text = generate_formatted_report(tracker_disp_names, scores, table_name=report_name) print(report_text) if per_video: for i in range(seq_lens): eval_data = eval_datas[i] print('\n{} sequences'.format(eval_data['sequences'][0])) scores = {} valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool) # ******************************** Success Plot ************************************** if 'success' in plot_types: threshold_set_overlap = torch.tensor(eval_data['threshold_set_overlap']) ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap']) # Index out valid sequences auc_curve, auc = get_auc_curve(ave_success_rate_plot_overlap, valid_sequence) scores['AUC'] = auc scores['OP50'] = auc_curve[:, threshold_set_overlap == 0.50] scores['OP75'] = auc_curve[:, threshold_set_overlap == 0.75] # ******************************** Precision Plot ************************************** if 'prec' in plot_types: ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center']) # Index out valid sequences prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center, valid_sequence) scores['Precision'] = prec_score # ******************************** Norm Precision Plot ********************************* if 'norm_prec' in plot_types: ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm']) # Index out valid sequences norm_prec_curve, norm_prec_score = get_prec_curve(ave_success_rate_plot_center_norm, valid_sequence) scores['Norm Precision'] = norm_prec_score # Print tracker_disp_names = [get_tracker_display_name(trk) for trk in tracker_names] report_text = generate_formatted_report(tracker_disp_names, scores, table_name=report_name) print(report_text),当前运行命令:python tracking/vis_results.py --dataset coesot --parameter_name ceutrack_coesot,那么现在我们期望同时评估两个跟踪器,ceutrack和atom,那么应该如何调整,直接将需要调整的部分标出来。
- Assistant
要在同时评估两个跟踪器(
ceutrack和atom),需要调整以下部分代码:1. 调整主程序中的跟踪器列表
if __name__ == '__main__': parser = argparse.ArgumentParser(description='Run tracker on sequence or dataset.') parser.add_argument('--dataset', type=str, default='coesot', help='coesot or fe108 or visevent or eed.') parser.add_argument('--parameter_name', type=str, default='ceutrack_coesot', help='coesot or fe108 or visevent or eed.') args = parser.parse_args() trackers = [] dataset_name = args.dataset # coesot fe108 """pytracking""" trackers.extend(trackerlist(name='atom', parameter_name='default', dataset_name=dataset_name, run_ids=None, display_name='ATOM')) """ceutrack""" trackers.extend(trackerlist(name='ceutrack', parameter_name=args.parameter_name, dataset_name=dataset_name, run_ids=None, display_name='CEUTrack256')) dataset = get_dataset(dataset_name) print_results(trackers, dataset, dataset_name, merge_results=True, plot_types=('success', 'norm_prec', 'prec'))2. 调整
trackerlist的函数导入不需要调整这个部分,因为已经包含在原有代码中,可以用于创建多种类型的跟踪器。
以上修改之后,即可评估两个跟踪器
ceutrack和atom。
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.