init commit of samurai

lib/test/analysis/extract_results.py

@@ -0,0 +1,226 @@
+import os
+import os.path as osp
+import sys
+import numpy as np
+from lib.test.utils.load_text import load_text
+import torch
+import pickle
+from tqdm import tqdm
+
+env_path = os.path.join(os.path.dirname(__file__), '../../..')
+if env_path not in sys.path:
+    sys.path.append(env_path)
+
+from lib.test.evaluation.environment import env_settings
+
+
+def calc_err_center(pred_bb, anno_bb, normalized=False):
+    pred_center = pred_bb[:, :2] + 0.5 * (pred_bb[:, 2:] - 1.0)
+    anno_center = anno_bb[:, :2] + 0.5 * (anno_bb[:, 2:] - 1.0)
+
+    if normalized:
+        pred_center = pred_center / anno_bb[:, 2:]
+        anno_center = anno_center / anno_bb[:, 2:]
+
+    err_center = ((pred_center - anno_center)**2).sum(1).sqrt()
+    return err_center
+
+
+def calc_iou_overlap(pred_bb, anno_bb):
+    tl = torch.max(pred_bb[:, :2], anno_bb[:, :2])
+    br = torch.min(pred_bb[:, :2] + pred_bb[:, 2:] - 1.0, anno_bb[:, :2] + anno_bb[:, 2:] - 1.0)
+    sz = (br - tl + 1.0).clamp(0)
+
+    # Area
+    intersection = sz.prod(dim=1)
+    union = pred_bb[:, 2:].prod(dim=1) + anno_bb[:, 2:].prod(dim=1) - intersection
+
+    return intersection / union
+
+
+def calc_seq_err_robust(pred_bb, anno_bb, dataset, target_visible=None):
+    pred_bb = pred_bb.clone()
+
+    # Check if invalid values are present
+    if torch.isnan(pred_bb).any() or (pred_bb[:, 2:] < 0.0).any():
+        raise Exception('Error: Invalid results')
+
+    if torch.isnan(anno_bb).any():
+        if dataset == 'uav':
+            pass
+        else:
+            raise Exception('Warning: NaNs in annotation')
+
+    if (pred_bb[:, 2:] == 0.0).any():
+        for i in range(1, pred_bb.shape[0]):
+            if i >= anno_bb.shape[0]:
+                continue
+            if (pred_bb[i, 2:] == 0.0).any() and not torch.isnan(anno_bb[i, :]).any():
+                pred_bb[i, :] = pred_bb[i-1, :]
+
+    if pred_bb.shape[0] != anno_bb.shape[0]:
+        if dataset == 'lasot':
+            if pred_bb.shape[0] > anno_bb.shape[0]:
+                # For monkey-17, there is a mismatch for some trackers.
+                pred_bb = pred_bb[:anno_bb.shape[0], :]
+            else:
+                raise Exception('Mis-match in tracker prediction and GT lengths')
+        else:
+            # print('Warning: Mis-match in tracker prediction and GT lengths')
+            if pred_bb.shape[0] > anno_bb.shape[0]:
+                pred_bb = pred_bb[:anno_bb.shape[0], :]
+            else:
+                pad = torch.zeros((anno_bb.shape[0] - pred_bb.shape[0], 4)).type_as(pred_bb)
+                pred_bb = torch.cat((pred_bb, pad), dim=0)
+
+    pred_bb[0, :] = anno_bb[0, :]
+
+    if target_visible is not None:
+        target_visible = target_visible.bool()
+        valid = ((anno_bb[:, 2:] > 0.0).sum(1) == 2) & target_visible
+    else:
+        valid = ((anno_bb[:, 2:] > 0.0).sum(1) == 2)
+
+    err_center = calc_err_center(pred_bb, anno_bb)
+    err_center_normalized = calc_err_center(pred_bb, anno_bb, normalized=True)
+    err_overlap = calc_iou_overlap(pred_bb, anno_bb)
+
+    # handle invalid anno cases
+    if dataset in ['uav']:
+        err_center[~valid] = -1.0
+    else:
+        err_center[~valid] = float("Inf")
+    err_center_normalized[~valid] = -1.0
+    err_overlap[~valid] = -1.0
+
+    if dataset == 'lasot':
+        err_center_normalized[~target_visible] = float("Inf")
+        err_center[~target_visible] = float("Inf")
+
+    if torch.isnan(err_overlap).any():
+        raise Exception('Nans in calculated overlap')
+    return err_overlap, err_center, err_center_normalized, valid
+
+
+def extract_results(trackers, dataset, report_name, skip_missing_seq=False, plot_bin_gap=0.05,
+                    exclude_invalid_frames=False):
+    settings = env_settings()
+    eps = 1e-16
+
+    result_plot_path = os.path.join(settings.result_plot_path, report_name)
+
+    if not os.path.exists(result_plot_path):
+        os.makedirs(result_plot_path)
+
+    threshold_set_overlap = torch.arange(0.0, 1.0 + plot_bin_gap, plot_bin_gap, dtype=torch.float64)
+    threshold_set_center = torch.arange(0, 51, dtype=torch.float64)
+    threshold_set_center_norm = torch.arange(0, 51, dtype=torch.float64) / 100.0
+
+    avg_overlap_all = torch.zeros((len(dataset), len(trackers)), dtype=torch.float64)
+    ave_success_rate_plot_overlap = torch.zeros((len(dataset), len(trackers), threshold_set_overlap.numel()),
+                                                dtype=torch.float32)
+    ave_success_rate_plot_center = torch.zeros((len(dataset), len(trackers), threshold_set_center.numel()),
+                                               dtype=torch.float32)
+    ave_success_rate_plot_center_norm = torch.zeros((len(dataset), len(trackers), threshold_set_center.numel()),
+                                                    dtype=torch.float32)
+
+    from collections import defaultdict
+    # default dict of default dict of list
+    
+
+    valid_sequence = torch.ones(len(dataset), dtype=torch.uint8)
+
+    for seq_id, seq in enumerate(tqdm(dataset)):
+        frame_success_rate_plot_overlap = defaultdict(lambda: defaultdict(list))    
+        frame_success_rate_plot_center = defaultdict(lambda: defaultdict(list))
+        frame_success_rate_plot_center_norm = defaultdict(lambda: defaultdict(list))
+        # Load anno
+        anno_bb = torch.tensor(seq.ground_truth_rect)
+        target_visible = torch.tensor(seq.target_visible, dtype=torch.uint8) if seq.target_visible is not None else None
+        for trk_id, trk in enumerate(trackers):
+            # Load results
+            base_results_path = '{}/{}'.format(trk.results_dir, seq.name)
+            results_path = '{}.txt'.format(base_results_path)
+
+            if os.path.isfile(results_path):
+                pred_bb = torch.tensor(load_text(str(results_path), delimiter=('\t', ','), dtype=np.float64))
+            else:
+                if skip_missing_seq:
+                    valid_sequence[seq_id] = 0
+                    break
+                else:
+                    raise Exception('Result not found. {}'.format(results_path))
+
+            # Calculate measures
+            err_overlap, err_center, err_center_normalized, valid_frame = calc_seq_err_robust(
+                pred_bb, anno_bb, seq.dataset, target_visible)
+
+            avg_overlap_all[seq_id, trk_id] = err_overlap[valid_frame].mean()
+
+            if exclude_invalid_frames:
+                seq_length = valid_frame.long().sum()
+            else:
+                seq_length = anno_bb.shape[0]
+
+            if seq_length <= 0:
+                raise Exception('Seq length zero')
+
+            ave_success_rate_plot_overlap[seq_id, trk_id, :] = (err_overlap.view(-1, 1) > threshold_set_overlap.view(1, -1)).sum(0).float() / seq_length
+            ave_success_rate_plot_center[seq_id, trk_id, :] = (err_center.view(-1, 1) <= threshold_set_center.view(1, -1)).sum(0).float() / seq_length
+            ave_success_rate_plot_center_norm[seq_id, trk_id, :] = (err_center_normalized.view(-1, 1) <= threshold_set_center_norm.view(1, -1)).sum(0).float() / seq_length
+
+            # for frame_id in range(seq_length):
+            #     frame_success_rate_plot_overlap[trk_id][frame_id].append((err_overlap[frame_id]).item())
+            #     frame_success_rate_plot_center[trk_id][frame_id].append((err_center[frame_id]).item())
+            #     frame_success_rate_plot_center_norm[trk_id][frame_id].append((err_center_normalized[frame_id] < 0.2).item())
+
+        # output_folder = "../cvpr2025/per_frame_success_rate"
+        # os.makedirs(output_folder, exist_ok=True)
+        # with open(osp.join(output_folder, f"{seq.name}.txt"), 'w') as f:
+        #     for frame_id in range(seq_length):
+        #         suc_score = frame_success_rate_plot_overlap[trk_id][frame_id][0]
+        #         f.write(f"{suc_score}\n")
+
+    # # plot the average success rate, center normalized for each tracker 
+    # # y axis: success rate
+    # # x axis: frame number
+    # # different color for each tracker
+    # # save the plot as a figure
+    # import matplotlib.pyplot as plt
+    # plt.figure(figsize=(10, 6))
+    # for trk_id, trk in enumerate(trackers):
+    #     list_to_plot = [np.mean(frame_success_rate_plot_overlap[trk_id][frame_id]) for frame_id in range(2000)]
+    #     # smooth the curve; window size = 10
+    #     smooth_list_to_plot = np.convolve(list_to_plot, np.ones((10,))/10, mode='valid')
+    #     # the smooth curve and non smooth curve have the same label
+    #     plt.plot(smooth_list_to_plot, label=trk.display_name, alpha=1)
+    # plt.xlabel('Frame Number')
+    # plt.ylabel('Success Rate')
+    # plt.title('Average Success Rate Over Frames')
+    # plt.legend()
+    # plt.grid(True)
+    # plt.savefig('average_success_rate_plot_overlap.png')
+    # plt.close()
+
+
+    print('\n\nComputed results over {} / {} sequences'.format(valid_sequence.long().sum().item(), valid_sequence.shape[0]))
+
+    # Prepare dictionary for saving data
+    seq_names = [s.name for s in dataset]
+    tracker_names = [{'name': t.name, 'param': t.parameter_name, 'run_id': t.run_id, 'disp_name': t.display_name}
+                     for t in trackers]
+
+    eval_data = {'sequences': seq_names, 'trackers': tracker_names,
+                 'valid_sequence': valid_sequence.tolist(),
+                 'ave_success_rate_plot_overlap': ave_success_rate_plot_overlap.tolist(),
+                 'ave_success_rate_plot_center': ave_success_rate_plot_center.tolist(),
+                 'ave_success_rate_plot_center_norm': ave_success_rate_plot_center_norm.tolist(),
+                 'avg_overlap_all': avg_overlap_all.tolist(),
+                 'threshold_set_overlap': threshold_set_overlap.tolist(),
+                 'threshold_set_center': threshold_set_center.tolist(),
+                 'threshold_set_center_norm': threshold_set_center_norm.tolist()}
+
+    with open(result_plot_path + '/eval_data.pkl', 'wb') as fh:
+        pickle.dump(eval_data, fh)
+
+    return eval_data

lib/test/analysis/plot_results.py

@@ -0,0 +1,796 @@
+import tikzplotlib
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+import os.path as osp
+import torch
+import pickle
+import json
+from lib.test.evaluation.environment import env_settings
+from lib.test.analysis.extract_results import extract_results
+
+
+def get_plot_draw_styles():
+    plot_draw_style = [
+                    #    {'color': (1.0, 0.0, 0.0), 'line_style': '-'},
+                    #    {'color': (0.0, 1.0, 0.0), 'line_style': '-'},
+                       {'color': (0.0, 1.0, 0.0), 'line_style': '-'},
+                       {'color': (0.0, 0.0, 0.0), 'line_style': '-'},
+                       {'color': (1.0, 0.0, 1.0), 'line_style': '-'},
+                       {'color': (0.0, 1.0, 1.0), 'line_style': '-'},
+                       {'color': (0.5, 0.5, 0.5), 'line_style': '-'},
+                       {'color': (136.0 / 255.0, 0.0, 21.0 / 255.0), 'line_style': '-'},
+                       {'color': (1.0, 127.0 / 255.0, 39.0 / 255.0), 'line_style': '-'},
+                       {'color': (0.0, 162.0 / 255.0, 232.0 / 255.0), 'line_style': '-'},
+                       {'color': (0.0, 0.5, 0.0), 'line_style': '-'},
+                       {'color': (1.0, 0.5, 0.2), 'line_style': '-'},
+                       {'color': (0.1, 0.4, 0.0), 'line_style': '-'},
+                       {'color': (0.6, 0.3, 0.9), 'line_style': '-'},
+                       {'color': (0.4, 0.7, 0.1), 'line_style': '-'},
+                       {'color': (0.2, 0.1, 0.7), 'line_style': '-'},
+                       {'color': (0.7, 0.6, 0.2), 'line_style': '-'}]
+
+    return plot_draw_style
+
+
+def check_eval_data_is_valid(eval_data, trackers, dataset):
+    """ Checks if the pre-computed results are valid"""
+    seq_names = [s.name for s in dataset]
+    seq_names_saved = eval_data['sequences']
+
+    tracker_names_f = [(t.name, t.parameter_name, t.run_id) for t in trackers]
+    tracker_names_f_saved = [(t['name'], t['param'], t['run_id']) for t in eval_data['trackers']]
+
+    return seq_names == seq_names_saved and tracker_names_f == tracker_names_f_saved
+
+
+def merge_multiple_runs(eval_data):
+    new_tracker_names = []
+    ave_success_rate_plot_overlap_merged = []
+    ave_success_rate_plot_center_merged = []
+    ave_success_rate_plot_center_norm_merged = []
+    avg_overlap_all_merged = []
+
+    ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap'])
+    ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center'])
+    ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm'])
+    avg_overlap_all = torch.tensor(eval_data['avg_overlap_all'])
+
+    trackers = eval_data['trackers']
+    merged = torch.zeros(len(trackers), dtype=torch.uint8)
+    for i in range(len(trackers)):
+        if merged[i]:
+            continue
+        base_tracker = trackers[i]
+        new_tracker_names.append(base_tracker)
+
+        match = [t['name'] == base_tracker['name'] and t['param'] == base_tracker['param'] for t in trackers]
+        match = torch.tensor(match)
+
+        ave_success_rate_plot_overlap_merged.append(ave_success_rate_plot_overlap[:, match, :].mean(1))
+        ave_success_rate_plot_center_merged.append(ave_success_rate_plot_center[:, match, :].mean(1))
+        ave_success_rate_plot_center_norm_merged.append(ave_success_rate_plot_center_norm[:, match, :].mean(1))
+        avg_overlap_all_merged.append(avg_overlap_all[:, match].mean(1))
+
+        merged[match] = 1
+
+    ave_success_rate_plot_overlap_merged = torch.stack(ave_success_rate_plot_overlap_merged, dim=1)
+    ave_success_rate_plot_center_merged = torch.stack(ave_success_rate_plot_center_merged, dim=1)
+    ave_success_rate_plot_center_norm_merged = torch.stack(ave_success_rate_plot_center_norm_merged, dim=1)
+    avg_overlap_all_merged = torch.stack(avg_overlap_all_merged, dim=1)
+
+    eval_data['trackers'] = new_tracker_names
+    eval_data['ave_success_rate_plot_overlap'] = ave_success_rate_plot_overlap_merged.tolist()
+    eval_data['ave_success_rate_plot_center'] = ave_success_rate_plot_center_merged.tolist()
+    eval_data['ave_success_rate_plot_center_norm'] = ave_success_rate_plot_center_norm_merged.tolist()
+    eval_data['avg_overlap_all'] = avg_overlap_all_merged.tolist()
+
+    return eval_data
+
+
+def get_tracker_display_name(tracker):
+    if tracker['disp_name'] is None:
+        if tracker['run_id'] is None:
+            disp_name = '{}_{}'.format(tracker['name'], tracker['param'])
+        else:
+            disp_name = '{}_{}_{:03d}'.format(tracker['name'], tracker['param'],
+                                              tracker['run_id'])
+    else:
+        disp_name = tracker['disp_name']
+
+    return  disp_name
+
+
+def plot_draw_save(y, x, scores, trackers, plot_draw_styles, result_plot_path, plot_opts):
+    plt.rcParams['text.usetex']=True
+    plt.rcParams["font.family"] = "Times New Roman"
+    # Plot settings
+    font_size = plot_opts.get('font_size', 25)
+    font_size_axis = plot_opts.get('font_size_axis', 20)
+    line_width = plot_opts.get('line_width', 2)
+    font_size_legend = plot_opts.get('font_size_legend', 15)
+
+    plot_type = plot_opts['plot_type']
+    legend_loc = plot_opts['legend_loc']
+    if 'attr' in plot_opts:
+        attr = plot_opts['attr']
+    else:
+        attr = None
+
+    xlabel = plot_opts['xlabel']
+    ylabel = plot_opts['ylabel']
+    ylabel = "%s"%(ylabel.replace('%','\%'))
+    xlim = plot_opts['xlim']
+    ylim = plot_opts['ylim']
+
+    title = r"\textbf{%s}" %(plot_opts['title'])
+    print
+
+    matplotlib.rcParams.update({'font.size': font_size})
+    matplotlib.rcParams.update({'axes.titlesize': font_size_axis})
+    matplotlib.rcParams.update({'axes.titleweight': 'black'})
+    matplotlib.rcParams.update({'axes.labelsize': font_size_axis})
+
+    fig, ax = plt.subplots()
+
+    index_sort = scores.argsort(descending=False)
+
+    plotted_lines = []
+    legend_text = []
+
+    for id, id_sort in enumerate(index_sort):
+        if trackers[id_sort]['disp_name'].startswith('SAMURAI'):
+            alpha = 1.0
+            line_style = '-'
+            if trackers[id_sort]['disp_name'] == 'SAMURAI-L':
+                color = (1.0, 0.0, 0.0)
+            elif trackers[id_sort]['disp_name'] == 'SAMURAI-B':
+                color = (0.0, 0.0, 1.0)
+        elif trackers[id_sort]['disp_name'].startswith('SAM2.1'):
+            alpha = 0.8
+            line_style = '--'
+            if trackers[id_sort]['disp_name'] == 'SAM2.1-L':
+                color = (1.0, 0.0, 0.0)
+            elif trackers[id_sort]['disp_name'] == 'SAM2.1-B':
+                color = (0.0, 0.0, 1.0)
+        else:
+            alpha = 0.5
+            color = plot_draw_styles[index_sort.numel() - id - 1]['color']
+            line_style = ":"
+        line = ax.plot(x.tolist(), y[id_sort, :].tolist(),
+                       linewidth=line_width,
+                       color=color,
+                       linestyle=line_style,
+                       alpha=alpha)
+
+        plotted_lines.append(line[0])
+
+        tracker = trackers[id_sort]
+        disp_name = get_tracker_display_name(tracker)
+
+        legend_text.append('{} [{:.1f}]'.format(disp_name, scores[id_sort]))
+
+    try:
+        # add bold to top method
+        # for i in range(1,2):
+        #     legend_text[-i] = r'\textbf{%s}'%(legend_text[-i])
+
+        for id, id_sort in enumerate(index_sort):
+            if trackers[id_sort]['disp_name'].startswith('SAMTrack'):
+                legend_text[id] = r'\textbf{%s}'%(legend_text[id])
+
+        ax.legend(plotted_lines[::-1], legend_text[::-1], loc=legend_loc, fancybox=False, edgecolor='black',
+                  fontsize=font_size_legend, framealpha=1.0)
+    except:
+        pass
+
+    ax.set(xlabel=xlabel,
+           ylabel=ylabel,
+           xlim=xlim, ylim=ylim,
+           title=title)
+
+    ax.grid(True, linestyle='-.')
+    fig.tight_layout()
+
+    def tikzplotlib_fix_ncols(obj):
+        """
+        workaround for matplotlib 3.6 renamed legend's _ncol to _ncols, which breaks tikzplotlib
+        """
+        if hasattr(obj, "_ncols"):
+            obj._ncol = obj._ncols
+        for child in obj.get_children():
+            tikzplotlib_fix_ncols(child)
+        
+    tikzplotlib_fix_ncols(fig)
+
+    # tikzplotlib.save('{}/{}_plot.tex'.format(result_plot_path, plot_type))
+    if attr is not None:
+        fig.savefig('{}/{}_{}_plot.pdf'.format(result_plot_path, plot_type, attr), dpi=300, format='pdf', transparent=True)
+    else:
+        fig.savefig('{}/{}_plot.pdf'.format(result_plot_path, plot_type), dpi=300, format='pdf', transparent=True)
+    plt.draw()
+
+
+def check_and_load_precomputed_results(trackers, dataset, report_name, force_evaluation=False, **kwargs):
+    # Load data
+    settings = env_settings()
+
+    # Load pre-computed results
+    result_plot_path = os.path.join(settings.result_plot_path, report_name)
+    eval_data_path = os.path.join(result_plot_path, 'eval_data.pkl')
+
+    if os.path.isfile(eval_data_path) and not force_evaluation:
+        with open(eval_data_path, 'rb') as fh:
+            eval_data = pickle.load(fh)
+    else:
+        # print('Pre-computed evaluation data not found. Computing results!')
+        eval_data = extract_results(trackers, dataset, report_name, **kwargs)
+
+    if not check_eval_data_is_valid(eval_data, trackers, dataset):
+        # print('Pre-computed evaluation data invalid. Re-computing results!')
+        eval_data = extract_results(trackers, dataset, report_name, **kwargs)
+        # pass
+    else:
+        # Update display names
+        tracker_names = [{'name': t.name, 'param': t.parameter_name, 'run_id': t.run_id, 'disp_name': t.display_name}
+                         for t in trackers]
+        eval_data['trackers'] = tracker_names
+    with open(eval_data_path, 'wb') as fh:
+        pickle.dump(eval_data, fh)
+    return eval_data
+
+
+def get_auc_curve(ave_success_rate_plot_overlap, valid_sequence):
+    ave_success_rate_plot_overlap = ave_success_rate_plot_overlap[valid_sequence, :, :]
+    auc_curve = ave_success_rate_plot_overlap.mean(0) * 100.0
+    auc = auc_curve.mean(-1)
+
+    return auc_curve, auc
+
+
+def get_prec_curve(ave_success_rate_plot_center, valid_sequence):
+    ave_success_rate_plot_center = ave_success_rate_plot_center[valid_sequence, :, :]
+    prec_curve = ave_success_rate_plot_center.mean(0) * 100.0
+    prec_score = prec_curve[:, 20]
+
+    return prec_curve, prec_score
+
+def plot_per_attribute_results(trackers, dataset, report_name, merge_results=False,
+                                plot_types=('success'), **kwargs):
+    # Load data
+    settings = env_settings()
+
+    plot_draw_styles = get_plot_draw_styles()
+
+    # Load pre-computed results
+    result_plot_path = os.path.join(settings.result_plot_path, report_name)
+    eval_data = check_and_load_precomputed_results(trackers, dataset, report_name, **kwargs)
+
+    tracker_names = eval_data['trackers']
+    valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool)
+
+    attr_folder = 'data/LaSOT/att'
+
+    attr_list = ['Illumination Variation', 'Partial Occlusion', 'Deformation', 'Motion Blur', 'Camera Motion', 'Rotation', 'Background Clutter', 'Viewpoint Change', 'Scale Variation', 'Full Occlusion', 'Fast Motion', 'Out-of-View', 'Low Resolution', 'Aspect Ration Change']
+    attr_list = ['IV', 'POC', 'DEF', 'MB', 'CM', 'ROT', 'BC', 'VC', 'SV', 'FOC', 'FM', 'OV', 'LR', 'ARC']
+
+    # Iterate over the sequence and construct a valid_sequence for each attribute
+    valid_sequence_attr = {}
+    for attr in attr_list:
+        valid_sequence_attr[attr] = torch.zeros(valid_sequence.shape[0], dtype=torch.bool)
+    for seq_id, seq_obj in enumerate(dataset):
+        seq_name = seq_obj.name
+        attr_txt = osp.join(attr_folder, f'{seq_name}.txt')
+        if osp.exists(attr_txt):
+            # read the attribute file into a list of True and False
+            # the attribute file looks like this: 0,0,0,0,0,1,0,1,1,0,0,0,0,0
+            attr_anno = np.loadtxt(attr_txt, dtype=int, delimiter=',')
+            # broadcast the valid_sequence to the attribute list
+            for attr_id, attr in enumerate(attr_list):
+                valid_sequence_attr[attr][seq_id] = attr_anno[attr_id]
+        else:
+            raise Exception(f'Attribute file not found for sequence {seq_name}')
+
+    tracker_names = eval_data['trackers']
+
+    if report_name == 'LaSOT-ext':
+        ylim_success = (0, 95)
+        ylim_precision = (0, 85)
+        ylim_norm_precision = (0, 85)
+        report_name = "LaSOT_{ext}"
+    else:
+        ylim_success = (0, 95)
+        ylim_precision = (0, 100)
+        ylim_norm_precision = (0, 88)
+    
+    threshold_set_overlap = torch.tensor(eval_data['threshold_set_overlap'])
+    ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap'])
+    ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center'])
+    ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm'])
+    for attr in attr_list:
+        scores = {}
+
+        print(f'{attr}: {valid_sequence_attr[attr].sum().item()}')
+        valid_sequence_attr[attr] = valid_sequence_attr[attr] & valid_sequence
+        
+        auc_curve, auc = get_auc_curve(ave_success_rate_plot_overlap, valid_sequence_attr[attr])
+        scores['AUC'] = auc
+
+        prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center, valid_sequence_attr[attr])
+        scores['Precision'] = prec_score
+
+        norm_prec_curve, norm_prec_score = get_prec_curve(ave_success_rate_plot_center_norm, valid_sequence_attr[attr])
+        scores['Norm Precision'] = norm_prec_score
+
+        tracker_disp_names = [get_tracker_display_name(trk) for trk in tracker_names]
+        report_text = generate_formatted_report(tracker_disp_names, scores, table_name=attr)
+        print(report_text)
+
+        success_plot_opts = {'plot_type': 'success', 'legend_loc': 'lower left', 'xlabel': 'Overlap threshold', 'attr': attr,
+                             'ylabel': 'Overlap Precision [%]', 'xlim': (0, 1.0), 'ylim': ylim_success, 'title': f'Success\ of\ {attr}\ ({report_name})'}
+        plot_draw_save(auc_curve, threshold_set_overlap, auc, tracker_names, plot_draw_styles, result_plot_path, success_plot_opts)
+
+    
+
+
+def plot_results(trackers, dataset, report_name, merge_results=False,
+                 plot_types=('success'), force_evaluation=False, **kwargs):
+    """
+    Plot results for the given trackers
+
+    args:
+        trackers - List of trackers to evaluate
+        dataset - List of sequences to evaluate
+        report_name - Name of the folder in env_settings.perm_mat_path where the computed results and plots are saved
+        merge_results - If True, multiple random runs for a non-deterministic trackers are averaged
+        plot_types - List of scores to display. Can contain 'success',
+                    'prec' (precision), and 'norm_prec' (normalized precision)
+    """
+    # Load data
+    settings = env_settings()
+
+    plot_draw_styles = get_plot_draw_styles()
+
+    # Load pre-computed results
+    result_plot_path = os.path.join(settings.result_plot_path, report_name)
+    eval_data = check_and_load_precomputed_results(trackers, dataset, report_name, force_evaluation, **kwargs)
+
+    # Merge results from multiple runs
+    if merge_results:
+        eval_data = merge_multiple_runs(eval_data)
+
+    tracker_names = eval_data['trackers']
+
+    valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool)
+
+    print('\nPlotting results over {} / {} sequences'.format(valid_sequence.long().sum().item(), valid_sequence.shape[0]))
+
+    print('\nGenerating plots for: {}'.format(report_name))
+
+    print(report_name)
+    if report_name == 'LaSOT':
+        ylim_success = (0, 95)
+        ylim_precision = (0, 95)
+        ylim_norm_precision = (0, 95)
+    elif report_name == 'LaSOT-ext':
+        ylim_success = (0, 85)
+        ylim_precision = (0, 85)
+        ylim_norm_precision = (0, 85)
+    else:
+        ylim_success = (0, 85)
+        ylim_precision = (0, 85)
+        ylim_norm_precision = (0, 85)
+
+    # ********************************  Success Plot **************************************
+    if 'success' in plot_types:
+        ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap'])
+
+        # Index out valid sequences
+        auc_curve, auc = get_auc_curve(ave_success_rate_plot_overlap, valid_sequence)
+        threshold_set_overlap = torch.tensor(eval_data['threshold_set_overlap'])
+
+        success_plot_opts = {'plot_type': 'success', 'legend_loc': 'lower left', 'xlabel': 'Overlap threshold',
+                             'ylabel': 'Overlap Precision [%]', 'xlim': (0, 1.0), 'ylim': ylim_success, 'title': f'Success\ ({report_name})'}
+        plot_draw_save(auc_curve, threshold_set_overlap, auc, tracker_names, plot_draw_styles, result_plot_path, success_plot_opts)
+
+    # ********************************  Precision Plot **************************************
+    if 'prec' in plot_types:
+        ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center'])
+
+        # Index out valid sequences
+        prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center, valid_sequence)
+        threshold_set_center = torch.tensor(eval_data['threshold_set_center'])
+
+        precision_plot_opts = {'plot_type': 'precision', 'legend_loc': 'lower right',
+                               'xlabel': 'Location error threshold [pixels]', 'ylabel': 'Distance Precision [%]',
+                               'xlim': (0, 50), 'ylim': ylim_precision, 'title': f'Precision\ ({report_name})'}
+        plot_draw_save(prec_curve, threshold_set_center, prec_score, tracker_names, plot_draw_styles, result_plot_path,
+                       precision_plot_opts)
+
+    # ********************************  Norm Precision Plot **************************************
+    if 'norm_prec' in plot_types:
+        ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm'])
+
+        # Index out valid sequences
+        prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center_norm, valid_sequence)
+        threshold_set_center_norm = torch.tensor(eval_data['threshold_set_center_norm'])
+
+        norm_precision_plot_opts = {'plot_type': 'norm_precision', 'legend_loc': 'lower right',
+                                    'xlabel': 'Location error threshold', 'ylabel': 'Distance Precision [%]',
+                                    'xlim': (0, 0.5), 'ylim': ylim_norm_precision, 'title': f'Normalized\ Precision\ ({report_name})'}
+        plot_draw_save(prec_curve, threshold_set_center_norm, prec_score, tracker_names, plot_draw_styles, result_plot_path,
+                       norm_precision_plot_opts)
+
+    plt.show()
+
+
+def generate_formatted_report(row_labels, scores, table_name=''):
+    name_width = max([len(d) for d in row_labels] + [len(table_name)]) + 5
+    min_score_width = 10
+
+    report_text = '\n{label: <{width}} |'.format(label=table_name, width=name_width)
+
+    score_widths = [max(min_score_width, len(k) + 3) for k in scores.keys()]
+
+    for s, s_w in zip(scores.keys(), score_widths):
+        report_text = '{prev} {s: <{width}} |'.format(prev=report_text, s=s, width=s_w)
+
+    report_text = '{prev}\n'.format(prev=report_text)
+
+    for trk_id, d_name in enumerate(row_labels):
+        # display name
+        report_text = '{prev}{tracker: <{width}} |'.format(prev=report_text, tracker=d_name,
+                                                           width=name_width)
+        for (score_type, score_value), s_w in zip(scores.items(), score_widths):
+            report_text = '{prev} {score: <{width}} |'.format(prev=report_text,
+                                                              score='{:0.2f}'.format(score_value[trk_id].item()),
+                                                              width=s_w)
+        report_text = '{prev}\n'.format(prev=report_text)
+
+    return report_text
+
+def print_per_attribute_results(trackers, dataset, report_name, merge_results=False,
+                                plot_types=('success'), **kwargs):
+    # Load pre-computed results
+    eval_data = check_and_load_precomputed_results(trackers, dataset, report_name, **kwargs)
+
+    tracker_names = eval_data['trackers']
+    valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool)
+
+    attr_folder = 'data/LaSOT/att'
+
+    attr_list = ['Illumination Variation', 'Partial Occlusion', 'Deformation', 'Motion Blur', 'Camera Motion', 'Rotation', 'Background Clutter', 'Viewpoint Change', 'Scale Variation', 'Full Occlusion', 'Fast Motion', 'Out-of-View', 'Low Resolution', 'Aspect Ration Change']
+    attr_list = ['IV', 'POC', 'DEF', 'MB', 'CM', 'ROT', 'BC', 'VC', 'SV', 'FOC', 'FM', 'OV', 'LR', 'ARC']
+
+    # Iterate over the sequence and construct a valid_sequence for each attribute
+    valid_sequence_attr = {}
+    for attr in attr_list:
+        valid_sequence_attr[attr] = torch.zeros(valid_sequence.shape[0], dtype=torch.bool)
+    for seq_id, seq_obj in enumerate(dataset):
+        seq_name = seq_obj.name
+        attr_txt = osp.join(attr_folder, f'{seq_name}.txt')
+        if osp.exists(attr_txt):
+            # read the attribute file into a list of True and False
+            # the attribute file looks like this: 0,0,0,0,0,1,0,1,1,0,0,0,0,0
+            attr_anno = np.loadtxt(attr_txt, dtype=int, delimiter=',')
+            # broadcast the valid_sequence to the attribute list
+            for attr_id, attr in enumerate(attr_list):
+                valid_sequence_attr[attr][seq_id] = attr_anno[attr_id]
+        else:
+            raise Exception(f'Attribute file not found for sequence {seq_name}')
+
+    tracker_names = eval_data['trackers']
+    
+    threshold_set_overlap = torch.tensor(eval_data['threshold_set_overlap'])
+    ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap'])
+    ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center'])
+    ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm'])
+    for attr in attr_list:
+        scores = {}
+
+        print(f'{attr}: {valid_sequence_attr[attr].sum().item()}')
+        valid_sequence_attr[attr] = valid_sequence_attr[attr] & valid_sequence
+        
+        auc_curve, auc = get_auc_curve(ave_success_rate_plot_overlap, valid_sequence_attr[attr])
+        scores['AUC'] = auc
+
+        prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center, valid_sequence_attr[attr])
+        scores['Precision'] = prec_score
+
+        norm_prec_curve, norm_prec_score = get_prec_curve(ave_success_rate_plot_center_norm, valid_sequence_attr[attr])
+        scores['Norm Precision'] = norm_prec_score
+
+        tracker_disp_names = [get_tracker_display_name(trk) for trk in tracker_names]
+        report_text = generate_formatted_report(tracker_disp_names, scores, table_name=attr)
+        print(report_text)
+
+
+def print_results(trackers, dataset, report_name, merge_results=False,
+                  plot_types=('success'), **kwargs):
+    """ Print the results for the given trackers in a formatted table
+    args:
+        trackers - List of trackers to evaluate
+        dataset - List of sequences to evaluate
+        report_name - Name of the folder in env_settings.perm_mat_path where the computed results and plots are saved
+        merge_results - If True, multiple random runs for a non-deterministic trackers are averaged
+        plot_types - List of scores to display. Can contain 'success' (prints AUC, OP50, and OP75 scores),
+                    'prec' (prints precision score), and 'norm_prec' (prints normalized precision score)
+    """
+    # Load pre-computed results
+    eval_data = check_and_load_precomputed_results(trackers, dataset, report_name, **kwargs)
+
+    # Merge results from multiple runs
+    if merge_results:
+        eval_data = merge_multiple_runs(eval_data)
+
+    tracker_names = eval_data['trackers']
+    valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool)
+
+    print('\nReporting results over {} / {} sequences'.format(valid_sequence.long().sum().item(), valid_sequence.shape[0]))
+
+    scores = {}
+
+    # ********************************  Success Plot **************************************
+    if 'success' in plot_types:
+        threshold_set_overlap = torch.tensor(eval_data['threshold_set_overlap'])
+        ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap'])
+
+        # Index out valid sequences
+        auc_curve, auc = get_auc_curve(ave_success_rate_plot_overlap, valid_sequence)
+        scores['AUC'] = auc
+        scores['OP50'] = auc_curve[:, threshold_set_overlap == 0.50]
+        scores['OP75'] = auc_curve[:, threshold_set_overlap == 0.75]
+
+    # ********************************  Precision Plot **************************************
+    if 'prec' in plot_types:
+        ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center'])
+
+        # Index out valid sequences
+        prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center, valid_sequence)
+        scores['Precision'] = prec_score
+
+    # ********************************  Norm Precision Plot *********************************
+    if 'norm_prec' in plot_types:
+        ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm'])
+
+        # Index out valid sequences
+        norm_prec_curve, norm_prec_score = get_prec_curve(ave_success_rate_plot_center_norm, valid_sequence)
+        scores['Norm Precision'] = norm_prec_score
+
+    # Print
+    tracker_disp_names = [get_tracker_display_name(trk) for trk in tracker_names]
+    report_text = generate_formatted_report(tracker_disp_names, scores, table_name=report_name)
+    print(report_text)
+
+
+def plot_got_success(trackers, report_name):
+    """ Plot success plot for GOT-10k dataset using the json reports.
+    Save the json reports from http://got-10k.aitestunion.com/leaderboard in the directory set to
+    env_settings.got_reports_path
+
+    The tracker name in the experiment file should be set to the name of the report file for that tracker,
+    e.g. DiMP50_report_2019_09_02_15_44_25 if the report is name DiMP50_report_2019_09_02_15_44_25.json
+
+    args:
+        trackers - List of trackers to evaluate
+        report_name - Name of the folder in env_settings.perm_mat_path where the computed results and plots are saved
+    """
+    # Load data
+    settings = env_settings()
+    plot_draw_styles = get_plot_draw_styles()
+
+    result_plot_path = os.path.join(settings.result_plot_path, report_name)
+
+    auc_curve = torch.zeros((len(trackers), 101))
+    scores = torch.zeros(len(trackers))
+
+    # Load results
+    tracker_names = []
+    for trk_id, trk in enumerate(trackers):
+        json_path = '{}/{}.json'.format(settings.got_reports_path, trk.name)
+
+        if os.path.isfile(json_path):
+            with open(json_path, 'r') as f:
+                eval_data = json.load(f)
+        else:
+            raise Exception('Report not found {}'.format(json_path))
+
+        if len(eval_data.keys()) > 1:
+            raise Exception
+
+        # First field is the tracker name. Index it out
+        eval_data = eval_data[list(eval_data.keys())[0]]
+        if 'succ_curve' in eval_data.keys():
+            curve = eval_data['succ_curve']
+            ao = eval_data['ao']
+        elif 'overall' in eval_data.keys() and 'succ_curve' in eval_data['overall'].keys():
+            curve = eval_data['overall']['succ_curve']
+            ao = eval_data['overall']['ao']
+        else:
+            raise Exception('Invalid JSON file {}'.format(json_path))
+
+        auc_curve[trk_id, :] = torch.tensor(curve) * 100.0
+        scores[trk_id] = ao * 100.0
+
+        tracker_names.append({'name': trk.name, 'param': trk.parameter_name, 'run_id': trk.run_id,
+                              'disp_name': trk.display_name})
+
+    threshold_set_overlap = torch.arange(0.0, 1.01, 0.01, dtype=torch.float64)
+
+    success_plot_opts = {'plot_type': 'success', 'legend_loc': 'lower left', 'xlabel': 'Overlap threshold',
+                         'ylabel': 'Overlap Precision [%]', 'xlim': (0, 1.0), 'ylim': (0, 100), 'title': 'Success plot'}
+    plot_draw_save(auc_curve, threshold_set_overlap, scores, tracker_names, plot_draw_styles, result_plot_path,
+                   success_plot_opts)
+    plt.show()
+
+
+def print_per_sequence_results(trackers, dataset, report_name, merge_results=False,
+                               filter_criteria=None, **kwargs):
+    """ Print per-sequence results for the given trackers. Additionally, the sequences to list can be filtered using
+    the filter criteria.
+
+    args:
+        trackers - List of trackers to evaluate
+        dataset - List of sequences to evaluate
+        report_name - Name of the folder in env_settings.perm_mat_path where the computed results and plots are saved
+        merge_results - If True, multiple random runs for a non-deterministic trackers are averaged
+        filter_criteria - Filter sequence results which are reported. Following modes are supported
+                        None: No filtering. Display results for all sequences in dataset
+                        'ao_min': Only display sequences for which the minimum average overlap (AO) score over the
+                                  trackers is less than a threshold filter_criteria['threshold']. This mode can
+                                  be used to select sequences where at least one tracker performs poorly.
+                        'ao_max': Only display sequences for which the maximum average overlap (AO) score over the
+                                  trackers is less than a threshold filter_criteria['threshold']. This mode can
+                                  be used to select sequences all tracker performs poorly.
+                        'delta_ao': Only display sequences for which the performance of different trackers vary by at
+                                    least filter_criteria['threshold'] in average overlap (AO) score. This mode can
+                                    be used to select sequences where the behaviour of the trackers greatly differ
+                                    between each other.
+    """
+    # Load pre-computed results
+    eval_data = check_and_load_precomputed_results(trackers, dataset, report_name, **kwargs)
+
+    # Merge results from multiple runs
+    if merge_results:
+        eval_data = merge_multiple_runs(eval_data)
+
+    tracker_names = eval_data['trackers']
+    valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool)
+    sequence_names = eval_data['sequences']
+    avg_overlap_all = torch.tensor(eval_data['avg_overlap_all']) * 100.0
+
+    # Filter sequences
+    if filter_criteria is not None:
+        if filter_criteria['mode'] == 'ao_min':
+            min_ao = avg_overlap_all.min(dim=1)[0]
+            valid_sequence = valid_sequence & (min_ao < filter_criteria['threshold'])
+        elif filter_criteria['mode'] == 'ao_max':
+            max_ao = avg_overlap_all.max(dim=1)[0]
+            valid_sequence = valid_sequence & (max_ao < filter_criteria['threshold'])
+        elif filter_criteria['mode'] == 'delta_ao':
+            min_ao = avg_overlap_all.min(dim=1)[0]
+            max_ao = avg_overlap_all.max(dim=1)[0]
+            valid_sequence = valid_sequence & ((max_ao - min_ao) > filter_criteria['threshold'])
+        else:
+            raise Exception
+
+    avg_overlap_all = avg_overlap_all[valid_sequence, :]
+    sequence_names = [s + ' (ID={})'.format(i) for i, (s, v) in enumerate(zip(sequence_names, valid_sequence.tolist())) if v]
+
+    tracker_disp_names = [get_tracker_display_name(trk) for trk in tracker_names]
+
+    scores_per_tracker = {k: avg_overlap_all[:, i] for i, k in enumerate(tracker_disp_names)}
+    report_text = generate_formatted_report(sequence_names, scores_per_tracker)
+
+    print(report_text)
+
+
+def print_results_per_video(trackers, dataset, report_name, merge_results=False,
+                  plot_types=('success'), per_video=False, **kwargs):
+    """ Print the results for the given trackers in a formatted table
+    args:
+        trackers - List of trackers to evaluate
+        dataset - List of sequences to evaluate
+        report_name - Name of the folder in env_settings.perm_mat_path where the computed results and plots are saved
+        merge_results - If True, multiple random runs for a non-deterministic trackers are averaged
+        plot_types - List of scores to display. Can contain 'success' (prints AUC, OP50, and OP75 scores),
+                    'prec' (prints precision score), and 'norm_prec' (prints normalized precision score)
+    """
+    # Load pre-computed results
+    eval_data = check_and_load_precomputed_results(trackers, dataset, report_name, **kwargs)
+
+    # Merge results from multiple runs
+    if merge_results:
+        eval_data = merge_multiple_runs(eval_data)
+
+    seq_lens = len(eval_data['sequences'])
+    eval_datas = [{} for _ in range(seq_lens)]
+    if per_video:
+        for key, value in eval_data.items():
+            if len(value) == seq_lens:
+                for i in range(seq_lens):
+                    eval_datas[i][key] = [value[i]]
+            else:
+                for i in range(seq_lens):
+                    eval_datas[i][key] = value
+
+    tracker_names = eval_data['trackers']
+    valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool)
+
+    print('\nReporting results over {} / {} sequences'.format(valid_sequence.long().sum().item(), valid_sequence.shape[0]))
+
+    scores = {}
+
+    # ********************************  Success Plot **************************************
+    if 'success' in plot_types:
+        threshold_set_overlap = torch.tensor(eval_data['threshold_set_overlap'])
+        ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap'])
+
+        # Index out valid sequences
+        auc_curve, auc = get_auc_curve(ave_success_rate_plot_overlap, valid_sequence)
+        scores['AUC'] = auc
+        scores['OP50'] = auc_curve[:, threshold_set_overlap == 0.50]
+        scores['OP75'] = auc_curve[:, threshold_set_overlap == 0.75]
+
+    # ********************************  Precision Plot **************************************
+    if 'prec' in plot_types:
+        ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center'])
+
+        # Index out valid sequences
+        prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center, valid_sequence)
+        scores['Precision'] = prec_score
+
+    # ********************************  Norm Precision Plot *********************************
+    if 'norm_prec' in plot_types:
+        ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm'])
+
+        # Index out valid sequences
+        norm_prec_curve, norm_prec_score = get_prec_curve(ave_success_rate_plot_center_norm, valid_sequence)
+        scores['Norm Precision'] = norm_prec_score
+
+    # Print
+    tracker_disp_names = [get_tracker_display_name(trk) for trk in tracker_names]
+    report_text = generate_formatted_report(tracker_disp_names, scores, table_name=report_name)
+    print(report_text)
+
+    if per_video:
+        for i in range(seq_lens):
+            eval_data = eval_datas[i]
+
+            print('\n{} sequences'.format(eval_data['sequences'][0]))
+
+            scores = {}
+            valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool)
+
+            # ********************************  Success Plot **************************************
+            if 'success' in plot_types:
+                threshold_set_overlap = torch.tensor(eval_data['threshold_set_overlap'])
+                ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap'])
+
+                # Index out valid sequences
+                auc_curve, auc = get_auc_curve(ave_success_rate_plot_overlap, valid_sequence)
+                scores['AUC'] = auc
+                scores['OP50'] = auc_curve[:, threshold_set_overlap == 0.50]
+                scores['OP75'] = auc_curve[:, threshold_set_overlap == 0.75]
+
+            # ********************************  Precision Plot **************************************
+            if 'prec' in plot_types:
+                ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center'])
+
+                # Index out valid sequences
+                prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center, valid_sequence)
+                scores['Precision'] = prec_score
+
+            # ********************************  Norm Precision Plot *********************************
+            if 'norm_prec' in plot_types:
+                ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm'])
+
+                # Index out valid sequences
+                norm_prec_curve, norm_prec_score = get_prec_curve(ave_success_rate_plot_center_norm, valid_sequence)
+                scores['Norm Precision'] = norm_prec_score
+
+            # Print
+            tracker_disp_names = [get_tracker_display_name(trk) for trk in tracker_names]
+            report_text = generate_formatted_report(tracker_disp_names, scores, table_name=report_name)
+            print(report_text)

lib/test/evaluation/__init__.py

@@ -0,0 +1,4 @@
+from .data import Sequence
+from .tracker import Tracker, trackerlist
+from .datasets import get_dataset
+from .environment import create_default_local_file_ITP_test

lib/test/evaluation/data.py

@@ -0,0 +1,169 @@
+import numpy as np
+from lib.test.evaluation.environment import env_settings
+from lib.train.data.image_loader import imread_indexed
+from collections import OrderedDict
+
+
+class BaseDataset:
+    """Base class for all datasets."""
+    def __init__(self):
+        self.env_settings = env_settings()
+
+    def __len__(self):
+        """Overload this function in your dataset. This should return number of sequences in the dataset."""
+        raise NotImplementedError
+
+    def get_sequence_list(self):
+        """Overload this in your dataset. Should return the list of sequences in the dataset."""
+        raise NotImplementedError
+
+
+class Sequence:
+    """Class for the sequence in an evaluation."""
+    def __init__(self, name, frames, dataset, ground_truth_rect, ground_truth_seg=None, init_data=None,
+                 object_class=None, target_visible=None, object_ids=None, multiobj_mode=False):
+        self.name = name
+        self.frames = frames
+        self.dataset = dataset
+        self.ground_truth_rect = ground_truth_rect
+        self.ground_truth_seg = ground_truth_seg
+        self.object_class = object_class
+        self.target_visible = target_visible
+        self.object_ids = object_ids
+        self.multiobj_mode = multiobj_mode
+        self.init_data = self._construct_init_data(init_data)
+        self._ensure_start_frame()
+
+    def _ensure_start_frame(self):
+        # Ensure start frame is 0
+        start_frame = min(list(self.init_data.keys()))
+        if start_frame > 0:
+            self.frames = self.frames[start_frame:]
+            if self.ground_truth_rect is not None:
+                if isinstance(self.ground_truth_rect, (dict, OrderedDict)):
+                    for obj_id, gt in self.ground_truth_rect.items():
+                        self.ground_truth_rect[obj_id] = gt[start_frame:,:]
+                else:
+                    self.ground_truth_rect = self.ground_truth_rect[start_frame:,:]
+            if self.ground_truth_seg is not None:
+                self.ground_truth_seg = self.ground_truth_seg[start_frame:]
+                assert len(self.frames) == len(self.ground_truth_seg)
+
+            if self.target_visible is not None:
+                self.target_visible = self.target_visible[start_frame:]
+            self.init_data = {frame-start_frame: val for frame, val in self.init_data.items()}
+
+    def _construct_init_data(self, init_data):
+        if init_data is not None:
+            if not self.multiobj_mode:
+                assert self.object_ids is None or len(self.object_ids) == 1
+                for frame, init_val in init_data.items():
+                    if 'bbox' in init_val and isinstance(init_val['bbox'], (dict, OrderedDict)):
+                        init_val['bbox'] = init_val['bbox'][self.object_ids[0]]
+            # convert to list
+            for frame, init_val in init_data.items():
+                if 'bbox' in init_val:
+                    if isinstance(init_val['bbox'], (dict, OrderedDict)):
+                        init_val['bbox'] = OrderedDict({obj_id: list(init) for obj_id, init in init_val['bbox'].items()})
+                    else:
+                        init_val['bbox'] = list(init_val['bbox'])
+        else:
+            init_data = {0: dict()}     # Assume start from frame 0
+
+            if self.object_ids is not None:
+                init_data[0]['object_ids'] = self.object_ids
+
+            if self.ground_truth_rect is not None:
+                if self.multiobj_mode:
+                    assert isinstance(self.ground_truth_rect, (dict, OrderedDict))
+                    init_data[0]['bbox'] = OrderedDict({obj_id: list(gt[0,:]) for obj_id, gt in self.ground_truth_rect.items()})
+                else:
+                    assert self.object_ids is None or len(self.object_ids) == 1
+                    if isinstance(self.ground_truth_rect, (dict, OrderedDict)):
+                        init_data[0]['bbox'] = list(self.ground_truth_rect[self.object_ids[0]][0, :])
+                    else:
+                        init_data[0]['bbox'] = list(self.ground_truth_rect[0,:])
+
+            if self.ground_truth_seg is not None:
+                init_data[0]['mask'] = self.ground_truth_seg[0]
+
+        return init_data
+
+    def init_info(self):
+        info = self.frame_info(frame_num=0)
+        return info
+
+    def frame_info(self, frame_num):
+        info = self.object_init_data(frame_num=frame_num)
+        return info
+
+    def init_bbox(self, frame_num=0):
+        return self.object_init_data(frame_num=frame_num).get('init_bbox')
+
+    def init_mask(self, frame_num=0):
+        return self.object_init_data(frame_num=frame_num).get('init_mask')
+
+    def get_info(self, keys, frame_num=None):
+        info = dict()
+        for k in keys:
+            val = self.get(k, frame_num=frame_num)
+            if val is not None:
+                info[k] = val
+        return info
+
+    def object_init_data(self, frame_num=None) -> dict:
+        if frame_num is None:
+            frame_num = 0
+        if frame_num not in self.init_data:
+            return dict()
+
+        init_data = dict()
+        for key, val in self.init_data[frame_num].items():
+            if val is None:
+                continue
+            init_data['init_'+key] = val
+
+        if 'init_mask' in init_data and init_data['init_mask'] is not None:
+            anno = imread_indexed(init_data['init_mask'])
+            if not self.multiobj_mode and self.object_ids is not None:
+                assert len(self.object_ids) == 1
+                anno = (anno == int(self.object_ids[0])).astype(np.uint8)
+            init_data['init_mask'] = anno
+
+        if self.object_ids is not None:
+            init_data['object_ids'] = self.object_ids
+            init_data['sequence_object_ids'] = self.object_ids
+
+        return init_data
+
+    def target_class(self, frame_num=None):
+        return self.object_class
+
+    def get(self, name, frame_num=None):
+        return getattr(self, name)(frame_num)
+
+    def __repr__(self):
+        return "{self.__class__.__name__} {self.name}, length={len} frames".format(self=self, len=len(self.frames))
+
+
+
+class SequenceList(list):
+    """List of sequences. Supports the addition operator to concatenate sequence lists."""
+    def __getitem__(self, item):
+        if isinstance(item, str):
+            for seq in self:
+                if seq.name == item:
+                    return seq
+            raise IndexError('Sequence name not in the dataset.')
+        elif isinstance(item, int):
+            return super(SequenceList, self).__getitem__(item)
+        elif isinstance(item, (tuple, list)):
+            return SequenceList([super(SequenceList, self).__getitem__(i) for i in item])
+        else:
+            return SequenceList(super(SequenceList, self).__getitem__(item))
+
+    def __add__(self, other):
+        return SequenceList(super(SequenceList, self).__add__(other))
+
+    def copy(self):
+        return SequenceList(super(SequenceList, self).copy())

lib/test/evaluation/datasets.py

@@ -0,0 +1,48 @@
+from collections import namedtuple
+import importlib
+from lib.test.evaluation.data import SequenceList
+
+DatasetInfo = namedtuple('DatasetInfo', ['module', 'class_name', 'kwargs'])
+
+pt = "lib.test.evaluation.%sdataset"  # Useful abbreviations to reduce the clutter
+
+dataset_dict = dict(
+    otb=DatasetInfo(module=pt % "otb", class_name="OTBDataset", kwargs=dict()),
+    nfs=DatasetInfo(module=pt % "nfs", class_name="NFSDataset", kwargs=dict()),
+    uav=DatasetInfo(module=pt % "uav", class_name="UAVDataset", kwargs=dict()),
+    tc128=DatasetInfo(module=pt % "tc128", class_name="TC128Dataset", kwargs=dict()),
+    tc128ce=DatasetInfo(module=pt % "tc128ce", class_name="TC128CEDataset", kwargs=dict()),
+    trackingnet=DatasetInfo(module=pt % "trackingnet", class_name="TrackingNetDataset", kwargs=dict()),
+    got10k_test=DatasetInfo(module=pt % "got10k", class_name="GOT10KDataset", kwargs=dict(split='test')),
+    got10k_val=DatasetInfo(module=pt % "got10k", class_name="GOT10KDataset", kwargs=dict(split='val')),
+    got10k_ltrval=DatasetInfo(module=pt % "got10k", class_name="GOT10KDataset", kwargs=dict(split='ltrval')),
+    lasot=DatasetInfo(module=pt % "lasot", class_name="LaSOTDataset", kwargs=dict()),
+    lasot_lmdb=DatasetInfo(module=pt % "lasot_lmdb", class_name="LaSOTlmdbDataset", kwargs=dict()),
+
+    vot18=DatasetInfo(module=pt % "vot", class_name="VOTDataset", kwargs=dict()),
+    vot22=DatasetInfo(module=pt % "vot", class_name="VOTDataset", kwargs=dict(year=22)),
+    itb=DatasetInfo(module=pt % "itb", class_name="ITBDataset", kwargs=dict()),
+    tnl2k=DatasetInfo(module=pt % "tnl2k", class_name="TNL2kDataset", kwargs=dict()),
+    lasot_extension_subset=DatasetInfo(module=pt % "lasotextensionsubset", class_name="LaSOTExtensionSubsetDataset",
+                                       kwargs=dict()),
+)
+
+
+def load_dataset(name: str):
+    """ Import and load a single dataset."""
+    name = name.lower()
+    dset_info = dataset_dict.get(name)
+    if dset_info is None:
+        raise ValueError('Unknown dataset \'%s\'' % name)
+
+    m = importlib.import_module(dset_info.module)
+    dataset = getattr(m, dset_info.class_name)(**dset_info.kwargs)  # Call the constructor
+    return dataset.get_sequence_list()
+
+
+def get_dataset(*args):
+    """ Get a single or set of datasets."""
+    dset = SequenceList()
+    for name in args:
+        dset.extend(load_dataset(name))
+    return dset

lib/test/evaluation/environment.py

@@ -0,0 +1,124 @@
+import importlib
+import os
+
+
+class EnvSettings:
+    def __init__(self):
+        test_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+
+        self.results_path = '{}/results/'.format(test_path)
+        self.segmentation_path = '{}/segmentation_results/'.format(test_path)
+        self.network_path = '{}/networks/'.format(test_path)
+        self.result_plot_path = '{}/result_plots/'.format(test_path)
+        self.otb_path = ''
+        self.nfs_path = ''
+        self.uav_path = ''
+        self.tpl_path = ''
+        self.vot_path = ''
+        self.got10k_path = ''
+        self.lasot_path = ''
+        self.trackingnet_path = ''
+        self.davis_dir = ''
+        self.youtubevos_dir = ''
+
+        self.got_packed_results_path = ''
+        self.got_reports_path = ''
+        self.tn_packed_results_path = ''
+
+
+def create_default_local_file():
+    comment = {'results_path': 'Where to store tracking results',
+               'network_path': 'Where tracking networks are stored.'}
+
+    path = os.path.join(os.path.dirname(__file__), 'local.py')
+    with open(path, 'w') as f:
+        settings = EnvSettings()
+
+        f.write('from test.evaluation.environment import EnvSettings\n\n')
+        f.write('def local_env_settings():\n')
+        f.write('    settings = EnvSettings()\n\n')
+        f.write('    # Set your local paths here.\n\n')
+
+        for attr in dir(settings):
+            comment_str = None
+            if attr in comment:
+                comment_str = comment[attr]
+            attr_val = getattr(settings, attr)
+            if not attr.startswith('__') and not callable(attr_val):
+                if comment_str is None:
+                    f.write('    settings.{} = \'{}\'\n'.format(attr, attr_val))
+                else:
+                    f.write('    settings.{} = \'{}\'    # {}\n'.format(attr, attr_val, comment_str))
+        f.write('\n    return settings\n\n')
+
+
+class EnvSettings_ITP:
+    def __init__(self, workspace_dir, data_dir, save_dir):
+        self.prj_dir = workspace_dir
+        self.save_dir = save_dir
+        self.results_path = os.path.join(save_dir, 'test/tracking_results')
+        self.segmentation_path = os.path.join(save_dir, 'test/segmentation_results')
+        self.network_path = os.path.join(save_dir, 'test/networks')
+        self.result_plot_path = os.path.join(save_dir, 'test/result_plots')
+        self.otb_path = os.path.join(data_dir, 'otb')
+        self.nfs_path = os.path.join(data_dir, 'nfs')
+        self.uav_path = os.path.join(data_dir, 'uav')
+        self.tc128_path = os.path.join(data_dir, 'TC128')
+        self.tpl_path = ''
+        self.vot_path = os.path.join(data_dir, 'VOT2019')
+        self.got10k_path = os.path.join(data_dir, 'got10k')
+        self.got10k_lmdb_path = os.path.join(data_dir, 'got10k_lmdb')
+        self.lasot_path = os.path.join(data_dir, 'lasot')
+        self.lasot_lmdb_path = os.path.join(data_dir, 'lasot_lmdb')
+        self.trackingnet_path = os.path.join(data_dir, 'trackingnet')
+        self.vot18_path = os.path.join(data_dir, 'vot2018')
+        self.vot22_path = os.path.join(data_dir, 'vot2022')
+        self.itb_path = os.path.join(data_dir, 'itb')
+        self.tnl2k_path = os.path.join(data_dir, 'tnl2k')
+        self.lasot_extension_subset_path_path = os.path.join(data_dir, 'lasot_extension_subset')
+        self.davis_dir = ''
+        self.youtubevos_dir = ''
+
+        self.got_packed_results_path = ''
+        self.got_reports_path = ''
+        self.tn_packed_results_path = ''
+
+
+def create_default_local_file_ITP_test(workspace_dir, data_dir, save_dir):
+    comment = {'results_path': 'Where to store tracking results',
+               'network_path': 'Where tracking networks are stored.'}
+
+    path = os.path.join(os.path.dirname(__file__), 'local.py')
+    with open(path, 'w') as f:
+        settings = EnvSettings_ITP(workspace_dir, data_dir, save_dir)
+
+        f.write('from lib.test.evaluation.environment import EnvSettings\n\n')
+        f.write('def local_env_settings():\n')
+        f.write('    settings = EnvSettings()\n\n')
+        f.write('    # Set your local paths here.\n\n')
+
+        for attr in dir(settings):
+            comment_str = None
+            if attr in comment:
+                comment_str = comment[attr]
+            attr_val = getattr(settings, attr)
+            if not attr.startswith('__') and not callable(attr_val):
+                if comment_str is None:
+                    f.write('    settings.{} = \'{}\'\n'.format(attr, attr_val))
+                else:
+                    f.write('    settings.{} = \'{}\'    # {}\n'.format(attr, attr_val, comment_str))
+        f.write('\n    return settings\n\n')
+
+
+def env_settings():
+    env_module_name = 'lib.test.evaluation.local'
+    try:
+        env_module = importlib.import_module(env_module_name)
+        return env_module.local_env_settings()
+    except:
+        env_file = os.path.join(os.path.dirname(__file__), 'local.py')
+
+        # Create a default file
+        create_default_local_file()
+        raise RuntimeError('YOU HAVE NOT SETUP YOUR local.py!!!\n Go to "{}" and set all the paths you need. '
+                           'Then try to run again.'.format(env_file))

lib/test/evaluation/got10kdataset.py

@@ -0,0 +1,56 @@
+import numpy as np
+from lib.test.evaluation.data import Sequence, BaseDataset, SequenceList
+from lib.test.utils.load_text import load_text
+import os
+
+
+class GOT10KDataset(BaseDataset):
+    """ GOT-10k dataset.
+
+    Publication:
+        GOT-10k: A Large High-Diversity Benchmark for Generic Object Tracking in the Wild
+        Lianghua Huang, Xin Zhao, and Kaiqi Huang
+        arXiv:1810.11981, 2018
+        https://arxiv.org/pdf/1810.11981.pdf
+
+    Download dataset from http://got-10k.aitestunion.com/downloads
+    """
+    def __init__(self, split):
+        super().__init__()
+        # Split can be test, val, or ltrval (a validation split consisting of videos from the official train set)
+        if split == 'test' or split == 'val':
+            self.base_path = os.path.join(self.env_settings.got10k_path, split)
+        else:
+            self.base_path = os.path.join(self.env_settings.got10k_path, 'train')
+
+        self.sequence_list = self._get_sequence_list(split)
+        self.split = split
+
+    def get_sequence_list(self):
+        return SequenceList([self._construct_sequence(s) for s in self.sequence_list])
+
+    def _construct_sequence(self, sequence_name):
+        anno_path = '{}/{}/groundtruth.txt'.format(self.base_path, sequence_name)
+
+        ground_truth_rect = load_text(str(anno_path), delimiter=',', dtype=np.float64)
+
+        frames_path = '{}/{}'.format(self.base_path, sequence_name)
+        frame_list = [frame for frame in os.listdir(frames_path) if frame.endswith(".jpg")]
+        frame_list.sort(key=lambda f: int(f[:-4]))
+        frames_list = [os.path.join(frames_path, frame) for frame in frame_list]
+
+        return Sequence(sequence_name, frames_list, 'got10k', ground_truth_rect.reshape(-1, 4))
+
+    def __len__(self):
+        return len(self.sequence_list)
+
+    def _get_sequence_list(self, split):
+        with open('{}/list.txt'.format(self.base_path)) as f:
+            sequence_list = f.read().splitlines()
+
+        if split == 'ltrval':
+            with open('{}/got10k_val_split.txt'.format(self.env_settings.dataspec_path)) as f:
+                seq_ids = f.read().splitlines()
+
+            sequence_list = [sequence_list[int(x)] for x in seq_ids]
+        return sequence_list

lib/test/evaluation/itbdataset.py

@@ -0,0 +1,75 @@
+import numpy as np
+from lib.test.evaluation.data import Sequence, BaseDataset, SequenceList
+from lib.test.utils.load_text import load_text
+import os
+
+
+class ITBDataset(BaseDataset):
+    """ NUS-PRO dataset
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.base_path = self.env_settings.itb_path
+        self.sequence_info_list = self._get_sequence_info_list(self.base_path)
+
+    def get_sequence_list(self):
+        return SequenceList([self._construct_sequence(s) for s in self.sequence_info_list])
+
+    def _construct_sequence(self, sequence_info):
+        sequence_path = sequence_info['path']
+        nz = sequence_info['nz']
+        ext = sequence_info['ext']
+        start_frame = sequence_info['startFrame']
+        end_frame = sequence_info['endFrame']
+
+        init_omit = 0
+        if 'initOmit' in sequence_info:
+            init_omit = sequence_info['initOmit']
+
+        frames = ['{base_path}/{sequence_path}/{frame:0{nz}}.{ext}'.format(base_path=self.base_path,
+                                                                           sequence_path=sequence_path, frame=frame_num,
+                                                                           nz=nz, ext=ext) for frame_num in
+                  range(start_frame + init_omit, end_frame + 1)]
+
+        anno_path = '{}/{}'.format(self.base_path, sequence_info['anno_path'])
+
+        # NOTE: NUS has some weird annos which panda cannot handle
+        ground_truth_rect = load_text(str(anno_path), delimiter=(',', None), dtype=np.float64, backend='numpy')
+        return Sequence(sequence_info['name'], frames, 'otb', ground_truth_rect[init_omit:, :],
+                        object_class=sequence_info['object_class'])
+
+    def __len__(self):
+        return len(self.sequence_info_list)
+
+    def get_fileNames(self, rootdir):
+        fs = []
+        fs_all = []
+        for root, dirs, files in os.walk(rootdir, topdown=True):
+            files.sort()
+            files.sort(key=len)
+            if files is not None:
+                for name in files:
+                    _, ending = os.path.splitext(name)
+                    if ending == ".jpg":
+                        _, root_ = os.path.split(root)
+                        fs.append(os.path.join(root_, name))
+                        fs_all.append(os.path.join(root, name))
+
+        return fs_all, fs
+
+    def _get_sequence_info_list(self, base_path):
+        sequence_info_list = []
+        for scene in os.listdir(base_path):
+            if '.' in scene:
+                continue
+            videos = os.listdir(os.path.join(base_path, scene))
+            for video in videos:
+                _, fs = self.get_fileNames(os.path.join(base_path, scene, video))
+                video_tmp = {"name": video, "path": scene + '/' + video, "startFrame": 1, "endFrame": len(fs),
+                             "nz": len(fs[0].split('/')[-1].split('.')[0]), "ext": "jpg",
+                             "anno_path": scene + '/' + video + "/groundtruth.txt",
+                             "object_class": "unknown"}
+                sequence_info_list.append(video_tmp)
+
+        return sequence_info_list  # sequence_info_list_50 #

lib/test/evaluation/lasot_lmdbdataset.py

@@ -0,0 +1,345 @@
+from lib.test.evaluation.data import Sequence, BaseDataset, SequenceList
+from lib.utils.lmdb_utils import *
+
+'''2021.1.27 LaSOT dataset using lmdb data'''
+
+
+class LaSOTlmdbDataset(BaseDataset):
+    """
+    LaSOT test set consisting of 280 videos (see Protocol-II in the LaSOT paper)
+
+    Publication:
+        LaSOT: A High-quality Benchmark for Large-scale Single Object Tracking
+        Heng Fan, Liting Lin, Fan Yang, Peng Chu, Ge Deng, Sijia Yu, Hexin Bai, Yong Xu, Chunyuan Liao and Haibin Ling
+        CVPR, 2019
+        https://arxiv.org/pdf/1809.07845.pdf
+
+    Download the dataset from https://cis.temple.edu/lasot/download.html
+    """
+    def __init__(self):
+        super().__init__()
+        self.base_path = self.env_settings.lasot_lmdb_path
+        self.sequence_list = self._get_sequence_list()
+        self.clean_list = self.clean_seq_list()
+
+    def clean_seq_list(self):
+        clean_lst = []
+        for i in range(len(self.sequence_list)):
+            cls, _ = self.sequence_list[i].split('-')
+            clean_lst.append(cls)
+        return clean_lst
+
+    def get_sequence_list(self):
+        return SequenceList([self._construct_sequence(s) for s in self.sequence_list])
+
+    def _construct_sequence(self, sequence_name):
+        class_name = sequence_name.split('-')[0]
+        anno_path = str('{}/{}/groundtruth.txt'.format(class_name, sequence_name))
+        # decode the groundtruth
+        gt_str_list = decode_str(self.base_path, anno_path).split('\n')[:-1]  # the last line is empty
+        gt_list = [list(map(float, line.split(','))) for line in gt_str_list]
+        ground_truth_rect = np.array(gt_list).astype(np.float64)
+        # decode occlusion file
+        occlusion_label_path = str('{}/{}/full_occlusion.txt'.format(class_name, sequence_name))
+        occ_list = list(map(int, decode_str(self.base_path, occlusion_label_path).split(',')))
+        full_occlusion = np.array(occ_list).astype(np.float64)
+        # decode out of view file
+        out_of_view_label_path = str('{}/{}/out_of_view.txt'.format(class_name, sequence_name))
+        out_of_view_list = list(map(int, decode_str(self.base_path, out_of_view_label_path).split(',')))
+        out_of_view = np.array(out_of_view_list).astype(np.float64)
+
+        target_visible = np.logical_and(full_occlusion == 0, out_of_view == 0)
+
+        frames_path = '{}/{}/img'.format(class_name, sequence_name)
+
+        frames_list = [[self.base_path, '{}/{:08d}.jpg'.format(frames_path, frame_number)] for frame_number in range(1, ground_truth_rect.shape[0] + 1)]
+
+        target_class = class_name
+        return Sequence(sequence_name, frames_list, 'lasot', ground_truth_rect.reshape(-1, 4),
+                        object_class=target_class, target_visible=target_visible)
+
+    def __len__(self):
+        return len(self.sequence_list)
+
+    def _get_sequence_list(self):
+        sequence_list = ['airplane-1',
+                         'airplane-9',
+                         'airplane-13',
+                         'airplane-15',
+                         'basketball-1',
+                         'basketball-6',
+                         'basketball-7',
+                         'basketball-11',
+                         'bear-2',
+                         'bear-4',
+                         'bear-6',
+                         'bear-17',
+                         'bicycle-2',
+                         'bicycle-7',
+                         'bicycle-9',
+                         'bicycle-18',
+                         'bird-2',
+                         'bird-3',
+                         'bird-15',
+                         'bird-17',
+                         'boat-3',
+                         'boat-4',
+                         'boat-12',
+                         'boat-17',
+                         'book-3',
+                         'book-10',
+                         'book-11',
+                         'book-19',
+                         'bottle-1',
+                         'bottle-12',
+                         'bottle-14',
+                         'bottle-18',
+                         'bus-2',
+                         'bus-5',
+                         'bus-17',
+                         'bus-19',
+                         'car-2',
+                         'car-6',
+                         'car-9',
+                         'car-17',
+                         'cat-1',
+                         'cat-3',
+                         'cat-18',
+                         'cat-20',
+                         'cattle-2',
+                         'cattle-7',
+                         'cattle-12',
+                         'cattle-13',
+                         'spider-14',
+                         'spider-16',
+                         'spider-18',
+                         'spider-20',
+                         'coin-3',
+                         'coin-6',
+                         'coin-7',
+                         'coin-18',
+                         'crab-3',
+                         'crab-6',
+                         'crab-12',
+                         'crab-18',
+                         'surfboard-12',
+                         'surfboard-4',
+                         'surfboard-5',
+                         'surfboard-8',
+                         'cup-1',
+                         'cup-4',
+                         'cup-7',
+                         'cup-17',
+                         'deer-4',
+                         'deer-8',
+                         'deer-10',
+                         'deer-14',
+                         'dog-1',
+                         'dog-7',
+                         'dog-15',
+                         'dog-19',
+                         'guitar-3',
+                         'guitar-8',
+                         'guitar-10',
+                         'guitar-16',
+                         'person-1',
+                         'person-5',
+                         'person-10',
+                         'person-12',
+                         'pig-2',
+                         'pig-10',
+                         'pig-13',
+                         'pig-18',
+                         'rubicCube-1',
+                         'rubicCube-6',
+                         'rubicCube-14',
+                         'rubicCube-19',
+                         'swing-10',
+                         'swing-14',
+                         'swing-17',
+                         'swing-20',
+                         'drone-13',
+                         'drone-15',
+                         'drone-2',
+                         'drone-7',
+                         'pool-12',
+                         'pool-15',
+                         'pool-3',
+                         'pool-7',
+                         'rabbit-10',
+                         'rabbit-13',
+                         'rabbit-17',
+                         'rabbit-19',
+                         'racing-10',
+                         'racing-15',
+                         'racing-16',
+                         'racing-20',
+                         'robot-1',
+                         'robot-19',
+                         'robot-5',
+                         'robot-8',
+                         'sepia-13',
+                         'sepia-16',
+                         'sepia-6',
+                         'sepia-8',
+                         'sheep-3',
+                         'sheep-5',
+                         'sheep-7',
+                         'sheep-9',
+                         'skateboard-16',
+                         'skateboard-19',
+                         'skateboard-3',
+                         'skateboard-8',
+                         'tank-14',
+                         'tank-16',
+                         'tank-6',
+                         'tank-9',
+                         'tiger-12',
+                         'tiger-18',
+                         'tiger-4',
+                         'tiger-6',
+                         'train-1',
+                         'train-11',
+                         'train-20',
+                         'train-7',
+                         'truck-16',
+                         'truck-3',
+                         'truck-6',
+                         'truck-7',
+                         'turtle-16',
+                         'turtle-5',
+                         'turtle-8',
+                         'turtle-9',
+                         'umbrella-17',
+                         'umbrella-19',
+                         'umbrella-2',
+                         'umbrella-9',
+                         'yoyo-15',
+                         'yoyo-17',
+                         'yoyo-19',
+                         'yoyo-7',
+                         'zebra-10',
+                         'zebra-14',
+                         'zebra-16',
+                         'zebra-17',
+                         'elephant-1',
+                         'elephant-12',
+                         'elephant-16',
+                         'elephant-18',
+                         'goldfish-3',
+                         'goldfish-7',
+                         'goldfish-8',
+                         'goldfish-10',
+                         'hat-1',
+                         'hat-2',
+                         'hat-5',
+                         'hat-18',
+                         'kite-4',
+                         'kite-6',
+                         'kite-10',
+                         'kite-15',
+                         'motorcycle-1',
+                         'motorcycle-3',
+                         'motorcycle-9',
+                         'motorcycle-18',
+                         'mouse-1',
+                         'mouse-8',
+                         'mouse-9',
+                         'mouse-17',
+                         'flag-3',
+                         'flag-9',
+                         'flag-5',
+                         'flag-2',
+                         'frog-3',
+                         'frog-4',
+                         'frog-20',
+                         'frog-9',
+                         'gametarget-1',
+                         'gametarget-2',
+                         'gametarget-7',
+                         'gametarget-13',
+                         'hand-2',
+                         'hand-3',
+                         'hand-9',
+                         'hand-16',
+                         'helmet-5',
+                         'helmet-11',
+                         'helmet-19',
+                         'helmet-13',
+                         'licenseplate-6',
+                         'licenseplate-12',
+                         'licenseplate-13',
+                         'licenseplate-15',
+                         'electricfan-1',
+                         'electricfan-10',
+                         'electricfan-18',
+                         'electricfan-20',
+                         'chameleon-3',
+                         'chameleon-6',
+                         'chameleon-11',
+                         'chameleon-20',
+                         'crocodile-3',
+                         'crocodile-4',
+                         'crocodile-10',
+                         'crocodile-14',
+                         'gecko-1',
+                         'gecko-5',
+                         'gecko-16',
+                         'gecko-19',
+                         'fox-2',
+                         'fox-3',
+                         'fox-5',
+                         'fox-20',
+                         'giraffe-2',
+                         'giraffe-10',
+                         'giraffe-13',
+                         'giraffe-15',
+                         'gorilla-4',
+                         'gorilla-6',
+                         'gorilla-9',
+                         'gorilla-13',
+                         'hippo-1',
+                         'hippo-7',
+                         'hippo-9',
+                         'hippo-20',
+                         'horse-1',
+                         'horse-4',
+                         'horse-12',
+                         'horse-15',
+                         'kangaroo-2',
+                         'kangaroo-5',
+                         'kangaroo-11',
+                         'kangaroo-14',
+                         'leopard-1',
+                         'leopard-7',
+                         'leopard-16',
+                         'leopard-20',
+                         'lion-1',
+                         'lion-5',
+                         'lion-12',
+                         'lion-20',
+                         'lizard-1',
+                         'lizard-3',
+                         'lizard-6',
+                         'lizard-13',
+                         'microphone-2',
+                         'microphone-6',
+                         'microphone-14',
+                         'microphone-16',
+                         'monkey-3',
+                         'monkey-4',
+                         'monkey-9',
+                         'monkey-17',
+                         'shark-2',
+                         'shark-3',
+                         'shark-5',
+                         'shark-6',
+                         'squirrel-8',
+                         'squirrel-11',
+                         'squirrel-13',
+                         'squirrel-19',
+                         'volleyball-1',
+                         'volleyball-13',
+                         'volleyball-18',
+                         'volleyball-19']
+        return sequence_list

lib/test/evaluation/lasotdataset.py

@@ -0,0 +1,342 @@
+import numpy as np
+from lib.test.evaluation.data import Sequence, BaseDataset, SequenceList
+from lib.test.utils.load_text import load_text
+
+
+class LaSOTDataset(BaseDataset):
+    """
+    LaSOT test set consisting of 280 videos (see Protocol-II in the LaSOT paper)
+
+    Publication:
+        LaSOT: A High-quality Benchmark for Large-scale Single Object Tracking
+        Heng Fan, Liting Lin, Fan Yang, Peng Chu, Ge Deng, Sijia Yu, Hexin Bai, Yong Xu, Chunyuan Liao and Haibin Ling
+        CVPR, 2019
+        https://arxiv.org/pdf/1809.07845.pdf
+
+    Download the dataset from https://cis.temple.edu/lasot/download.html
+    """
+    def __init__(self):
+        super().__init__()
+        self.base_path = self.env_settings.lasot_path
+        self.sequence_list = self._get_sequence_list()
+        self.clean_list = self.clean_seq_list()
+
+    def clean_seq_list(self):
+        clean_lst = []
+        for i in range(len(self.sequence_list)):
+            cls, _ = self.sequence_list[i].split('-')
+            clean_lst.append(cls)
+        return  clean_lst
+
+    def get_sequence_list(self):
+        return SequenceList([self._construct_sequence(s) for s in self.sequence_list])
+
+    def _construct_sequence(self, sequence_name):
+        class_name = sequence_name.split('-')[0]
+        anno_path = '{}/{}/{}/groundtruth.txt'.format(self.base_path, class_name, sequence_name)
+
+        ground_truth_rect = load_text(str(anno_path), delimiter=',', dtype=np.float64)
+
+        occlusion_label_path = '{}/{}/{}/full_occlusion.txt'.format(self.base_path, class_name, sequence_name)
+
+        # NOTE: pandas backed seems super super slow for loading occlusion/oov masks
+        full_occlusion = load_text(str(occlusion_label_path), delimiter=',', dtype=np.float64, backend='numpy')
+
+        out_of_view_label_path = '{}/{}/{}/out_of_view.txt'.format(self.base_path, class_name, sequence_name)
+        out_of_view = load_text(str(out_of_view_label_path), delimiter=',', dtype=np.float64, backend='numpy')
+
+        target_visible = np.logical_and(full_occlusion == 0, out_of_view == 0)
+
+        frames_path = '{}/{}/{}/img'.format(self.base_path, class_name, sequence_name)
+
+        frames_list = ['{}/{:08d}.jpg'.format(frames_path, frame_number) for frame_number in range(1, ground_truth_rect.shape[0] + 1)]
+
+        target_class = class_name
+        return Sequence(sequence_name, frames_list, 'lasot', ground_truth_rect.reshape(-1, 4),
+                        object_class=target_class, target_visible=target_visible)
+
+    def __len__(self):
+        return len(self.sequence_list)
+
+    def _get_sequence_list(self):
+        sequence_list = ['airplane-1',
+                         'airplane-9',
+                         'airplane-13',
+                         'airplane-15',
+                         'basketball-1',
+                         'basketball-6',
+                         'basketball-7',
+                         'basketball-11',
+                         'bear-2',
+                         'bear-4',
+                         'bear-6',
+                         'bear-17',
+                         'bicycle-2',
+                         'bicycle-7',
+                         'bicycle-9',
+                         'bicycle-18',
+                         'bird-2',
+                         'bird-3',
+                         'bird-15',
+                         'bird-17',
+                         'boat-3',
+                         'boat-4',
+                         'boat-12',
+                         'boat-17',
+                         'book-3',
+                         'book-10',
+                         'book-11',
+                         'book-19',
+                         'bottle-1',
+                         'bottle-12',
+                         'bottle-14',
+                         'bottle-18',
+                         'bus-2',
+                         'bus-5',
+                         'bus-17',
+                         'bus-19',
+                         'car-2',
+                         'car-6',
+                         'car-9',
+                         'car-17',
+                         'cat-1',
+                         'cat-3',
+                         'cat-18',
+                         'cat-20',
+                         'cattle-2',
+                         'cattle-7',
+                         'cattle-12',
+                         'cattle-13',
+                         'spider-14',
+                         'spider-16',
+                         'spider-18',
+                         'spider-20',
+                         'coin-3',
+                         'coin-6',
+                         'coin-7',
+                         'coin-18',
+                         'crab-3',
+                         'crab-6',
+                         'crab-12',
+                         'crab-18',
+                         'surfboard-12',
+                         'surfboard-4',
+                         'surfboard-5',
+                         'surfboard-8',
+                         'cup-1',
+                         'cup-4',
+                         'cup-7',
+                         'cup-17',
+                         'deer-4',
+                         'deer-8',
+                         'deer-10',
+                         'deer-14',
+                         'dog-1',
+                         'dog-7',
+                         'dog-15',
+                         'dog-19',
+                         'guitar-3',
+                         'guitar-8',
+                         'guitar-10',
+                         'guitar-16',
+                         'person-1',
+                         'person-5',
+                         'person-10',
+                         'person-12',
+                         'pig-2',
+                         'pig-10',
+                         'pig-13',
+                         'pig-18',
+                         'rubicCube-1',
+                         'rubicCube-6',
+                         'rubicCube-14',
+                         'rubicCube-19',
+                         'swing-10',
+                         'swing-14',
+                         'swing-17',
+                         'swing-20',
+                         'drone-13',
+                         'drone-15',
+                         'drone-2',
+                         'drone-7',
+                         'pool-12',
+                         'pool-15',
+                         'pool-3',
+                         'pool-7',
+                         'rabbit-10',
+                         'rabbit-13',
+                         'rabbit-17',
+                         'rabbit-19',
+                         'racing-10',
+                         'racing-15',
+                         'racing-16',
+                         'racing-20',
+                         'robot-1',
+                         'robot-19',
+                         'robot-5',
+                         'robot-8',
+                         'sepia-13',
+                         'sepia-16',
+                         'sepia-6',
+                         'sepia-8',
+                         'sheep-3',
+                         'sheep-5',
+                         'sheep-7',
+                         'sheep-9',
+                         'skateboard-16',
+                         'skateboard-19',
+                         'skateboard-3',
+                         'skateboard-8',
+                         'tank-14',
+                         'tank-16',
+                         'tank-6',
+                         'tank-9',
+                         'tiger-12',
+                         'tiger-18',
+                         'tiger-4',
+                         'tiger-6',
+                         'train-1',
+                         'train-11',
+                         'train-20',
+                         'train-7',
+                         'truck-16',
+                         'truck-3',
+                         'truck-6',
+                         'truck-7',
+                         'turtle-16',
+                         'turtle-5',
+                         'turtle-8',
+                         'turtle-9',
+                         'umbrella-17',
+                         'umbrella-19',
+                         'umbrella-2',
+                         'umbrella-9',
+                         'yoyo-15',
+                         'yoyo-17',
+                         'yoyo-19',
+                         'yoyo-7',
+                         'zebra-10',
+                         'zebra-14',
+                         'zebra-16',
+                         'zebra-17',
+                         'elephant-1',
+                         'elephant-12',
+                         'elephant-16',
+                         'elephant-18',
+                         'goldfish-3',
+                         'goldfish-7',
+                         'goldfish-8',
+                         'goldfish-10',
+                         'hat-1',
+                         'hat-2',
+                         'hat-5',
+                         'hat-18',
+                         'kite-4',
+                         'kite-6',
+                         'kite-10',
+                         'kite-15',
+                         'motorcycle-1',
+                         'motorcycle-3',
+                         'motorcycle-9',
+                         'motorcycle-18',
+                         'mouse-1',
+                         'mouse-8',
+                         'mouse-9',
+                         'mouse-17',
+                         'flag-3',
+                         'flag-9',
+                         'flag-5',
+                         'flag-2',
+                         'frog-3',
+                         'frog-4',
+                         'frog-20',
+                         'frog-9',
+                         'gametarget-1',
+                         'gametarget-2',
+                         'gametarget-7',
+                         'gametarget-13',
+                         'hand-2',
+                         'hand-3',
+                         'hand-9',
+                         'hand-16',
+                         'helmet-5',
+                         'helmet-11',
+                         'helmet-19',
+                         'helmet-13',
+                         'licenseplate-6',
+                         'licenseplate-12',
+                         'licenseplate-13',
+                         'licenseplate-15',
+                         'electricfan-1',
+                         'electricfan-10',
+                         'electricfan-18',
+                         'electricfan-20',
+                         'chameleon-3',
+                         'chameleon-6',
+                         'chameleon-11',
+                         'chameleon-20',
+                         'crocodile-3',
+                         'crocodile-4',
+                         'crocodile-10',
+                         'crocodile-14',
+                         'gecko-1',
+                         'gecko-5',
+                         'gecko-16',
+                         'gecko-19',
+                         'fox-2',
+                         'fox-3',
+                         'fox-5',
+                         'fox-20',
+                         'giraffe-2',
+                         'giraffe-10',
+                         'giraffe-13',
+                         'giraffe-15',
+                         'gorilla-4',
+                         'gorilla-6',
+                         'gorilla-9',
+                         'gorilla-13',
+                         'hippo-1',
+                         'hippo-7',
+                         'hippo-9',
+                         'hippo-20',
+                         'horse-1',
+                         'horse-4',
+                         'horse-12',
+                         'horse-15',
+                         'kangaroo-2',
+                         'kangaroo-5',
+                         'kangaroo-11',
+                         'kangaroo-14',
+                         'leopard-1',
+                         'leopard-7',
+                         'leopard-16',
+                         'leopard-20',
+                         'lion-1',
+                         'lion-5',
+                         'lion-12',
+                         'lion-20',
+                         'lizard-1',
+                         'lizard-3',
+                         'lizard-6',
+                         'lizard-13',
+                         'microphone-2',
+                         'microphone-6',
+                         'microphone-14',
+                         'microphone-16',
+                         'monkey-3',
+                         'monkey-4',
+                         'monkey-9',
+                         'monkey-17',
+                         'shark-2',
+                         'shark-3',
+                         'shark-5',
+                         'shark-6',
+                         'squirrel-8',
+                         'squirrel-11',
+                         'squirrel-13',
+                         'squirrel-19',
+                         'volleyball-1',
+                         'volleyball-13',
+                         'volleyball-18',
+                         'volleyball-19']
+        return sequence_list

lib/test/evaluation/lasotextensionsubsetdataset.py

@@ -0,0 +1,211 @@
+import numpy as np
+from lib.test.evaluation.data import Sequence, BaseDataset, SequenceList
+from lib.test.utils.load_text import load_text
+
+
+class LaSOTExtensionSubsetDataset(BaseDataset):
+    """
+    LaSOT test set consisting of 280 videos (see Protocol-II in the LaSOT paper)
+    Publication:
+        LaSOT: A High-quality Large-scale Single Object Tracking Benchmark
+        Heng Fan, Hexin Bai, Liting Lin, Fan Yang, Peng Chu, Ge Deng, Sijia Yu, Harshit, Mingzhen Huang, Juehuan Liu,
+        Yong Xu, Chunyuan Liao, Lin Yuan, Haibin Ling
+        IJCV, 2020
+        https://arxiv.org/pdf/2009.03465.pdf
+    Download the dataset from http://vision.cs.stonybrook.edu/~lasot/download.html
+    """
+    def __init__(self):
+        super().__init__()
+        self.base_path = self.env_settings.lasot_extension_subset_path
+        self.sequence_list = self._get_sequence_list()
+        self.clean_list = self.clean_seq_list()
+
+    def clean_seq_list(self):
+        clean_lst = []
+        for i in range(len(self.sequence_list)):
+            cls, _ = self.sequence_list[i].split('-')
+            clean_lst.append(cls)
+        return  clean_lst
+
+    def get_sequence_list(self):
+        return SequenceList([self._construct_sequence(s) for s in self.sequence_list])
+
+    def _construct_sequence(self, sequence_name):
+        class_name = sequence_name.split('-')[0]
+        anno_path = '{}/{}/{}/groundtruth.txt'.format(self.base_path, class_name, sequence_name)
+
+        ground_truth_rect = load_text(str(anno_path), delimiter=',', dtype=np.float64)
+
+        occlusion_label_path = '{}/{}/{}/full_occlusion.txt'.format(self.base_path, class_name, sequence_name)
+
+        # NOTE: pandas backed seems super super slow for loading occlusion/oov masks
+        full_occlusion = load_text(str(occlusion_label_path), delimiter=',', dtype=np.float64, backend='numpy')
+
+        out_of_view_label_path = '{}/{}/{}/out_of_view.txt'.format(self.base_path, class_name, sequence_name)
+        out_of_view = load_text(str(out_of_view_label_path), delimiter=',', dtype=np.float64, backend='numpy')
+
+        target_visible = np.logical_and(full_occlusion == 0, out_of_view == 0)
+
+        frames_path = '{}/{}/{}/img'.format(self.base_path, class_name, sequence_name)
+
+        frames_list = ['{}/{:08d}.jpg'.format(frames_path, frame_number) for frame_number in range(1, ground_truth_rect.shape[0] + 1)]
+
+        target_class = class_name
+        return Sequence(sequence_name, frames_list, 'lasot_extension_subset', ground_truth_rect.reshape(-1, 4),
+                        object_class=target_class, target_visible=target_visible)
+
+    def __len__(self):
+        return len(self.sequence_list)
+
+    def _get_sequence_list(self):
+        sequence_list = ['atv-1',
+                         'atv-2',
+                         'atv-3',
+                         'atv-4',
+                         'atv-5',
+                         'atv-6',
+                         'atv-7',
+                         'atv-8',
+                         'atv-9',
+                         'atv-10',
+                         'badminton-1',
+                         'badminton-2',
+                         'badminton-3',
+                         'badminton-4',
+                         'badminton-5',
+                         'badminton-6',
+                         'badminton-7',
+                         'badminton-8',
+                         'badminton-9',
+                         'badminton-10',
+                         'cosplay-1',
+                         'cosplay-10',
+                         'cosplay-2',
+                         'cosplay-3',
+                         'cosplay-4',
+                         'cosplay-5',
+                         'cosplay-6',
+                         'cosplay-7',
+                         'cosplay-8',
+                         'cosplay-9',
+                         'dancingshoe-1',
+                         'dancingshoe-2',
+                         'dancingshoe-3',
+                         'dancingshoe-4',
+                         'dancingshoe-5',
+                         'dancingshoe-6',
+                         'dancingshoe-7',
+                         'dancingshoe-8',
+                         'dancingshoe-9',
+                         'dancingshoe-10',
+                         'footbag-1',
+                         'footbag-2',
+                         'footbag-3',
+                         'footbag-4',
+                         'footbag-5',
+                         'footbag-6',
+                         'footbag-7',
+                         'footbag-8',
+                         'footbag-9',
+                         'footbag-10',
+                         'frisbee-1',
+                         'frisbee-2',
+                         'frisbee-3',
+                         'frisbee-4',
+                         'frisbee-5',
+                         'frisbee-6',
+                         'frisbee-7',
+                         'frisbee-8',
+                         'frisbee-9',
+                         'frisbee-10',
+                         'jianzi-1',
+                         'jianzi-2',
+                         'jianzi-3',
+                         'jianzi-4',
+                         'jianzi-5',
+                         'jianzi-6',
+                         'jianzi-7',
+                         'jianzi-8',
+                         'jianzi-9',
+                         'jianzi-10',
+                         'lantern-1',
+                         'lantern-2',
+                         'lantern-3',
+                         'lantern-4',
+                         'lantern-5',
+                         'lantern-6',
+                         'lantern-7',
+                         'lantern-8',
+                         'lantern-9',
+                         'lantern-10',
+                         'misc-1',
+                         'misc-2',
+                         'misc-3',
+                         'misc-4',
+                         'misc-5',
+                         'misc-6',
+                         'misc-7',
+                         'misc-8',
+                         'misc-9',
+                         'misc-10',
+                         'opossum-1',
+                         'opossum-2',
+                         'opossum-3',
+                         'opossum-4',
+                         'opossum-5',
+                         'opossum-6',
+                         'opossum-7',
+                         'opossum-8',
+                         'opossum-9',
+                         'opossum-10',
+                         'paddle-1',
+                         'paddle-2',
+                         'paddle-3',
+                         'paddle-4',
+                         'paddle-5',
+                         'paddle-6',
+                         'paddle-7',
+                         'paddle-8',
+                         'paddle-9',
+                         'paddle-10',
+                         'raccoon-1',
+                         'raccoon-2',
+                         'raccoon-3',
+                         'raccoon-4',
+                         'raccoon-5',
+                         'raccoon-6',
+                         'raccoon-7',
+                         'raccoon-8',
+                         'raccoon-9',
+                         'raccoon-10',
+                         'rhino-1',
+                         'rhino-2',
+                         'rhino-3',
+                         'rhino-4',
+                         'rhino-5',
+                         'rhino-6',
+                         'rhino-7',
+                         'rhino-8',
+                         'rhino-9',
+                         'rhino-10',
+                         'skatingshoe-1',
+                         'skatingshoe-2',
+                         'skatingshoe-3',
+                         'skatingshoe-4',
+                         'skatingshoe-5',
+                         'skatingshoe-6',
+                         'skatingshoe-7',
+                         'skatingshoe-8',
+                         'skatingshoe-9',
+                         'skatingshoe-10',
+                         'wingsuit-1',
+                         'wingsuit-2',
+                         'wingsuit-3',
+                         'wingsuit-4',
+                         'wingsuit-5',
+                         'wingsuit-6',
+                         'wingsuit-7',
+                         'wingsuit-8',
+                         'wingsuit-9',
+                         'wingsuit-10']
+        return sequence_list

lib/test/evaluation/local.py

@@ -0,0 +1,38 @@
+from lib.test.evaluation.environment import EnvSettings
+
+def local_env_settings():
+    settings = EnvSettings()
+
+    # Set your local paths here.
+
+    settings.lasot_path = '/home/cycyang/code/vot-sam/data/LaSOT'
+    settings.lasot_extension_subset_path = '/home/cycyang/code/vot-sam/data/LaSOT-ext'
+    settings.nfs_path = '/home/cycyang/code/vot-sam/data/NFS'
+    settings.otb_path = '/home/cycyang/code/vot-sam/data/otb'
+    settings.uav_path = '//home/cycyang/code/vot-sam/data/uav'
+    settings.results_path = '/home/cycyang/code/vot-sam/raw_results'
+    settings.result_plot_path = '/home/cycyang/code/vot-sam/evaluation_results'
+    settings.save_dir = '/home/cycyang/code/vot-sam/evaluation_results'
+
+    settings.davis_dir = ''
+    settings.got10k_lmdb_path = '/home/baiyifan/code/OSTrack/data/got10k_lmdb'
+    settings.got10k_path = '/home/baiyifan/GOT-10k'
+    settings.got_packed_results_path = ''
+    settings.got_reports_path = ''
+    settings.itb_path = '/home/baiyifan/code/OSTrack/data/itb'
+    settings.lasot_lmdb_path = '/home/baiyifan/code/OSTrack/data/lasot_lmdb'
+    settings.network_path =  '/ssddata/baiyifan/artrack_256_full_re/'   # Where tracking networks are stored.
+    settings.prj_dir = '/home/baiyifan/code/2d_autoregressive/bins_mask'
+    settings.segmentation_path = '/data1/os/test/segmentation_results'
+    settings.tc128_path = '/home/baiyifan/code/OSTrack/data/TC128'
+    settings.tn_packed_results_path = ''
+    settings.tnl2k_path = '/home/baiyifan/code/OSTrack/data/tnl2k'
+    settings.tpl_path = ''
+    settings.trackingnet_path = '/ssddata/TrackingNet/all_zip'
+    settings.vot18_path = '/home/baiyifan/code/OSTrack/data/vot2018'
+    settings.vot22_path = '/home/baiyifan/code/OSTrack/data/vot2022'
+    settings.vot_path = '/home/baiyifan/code/OSTrack/data/VOT2019'
+    settings.youtubevos_dir = ''
+
+    return settings
+

lib/test/evaluation/nfsdataset.py

@@ -0,0 +1,153 @@
+import numpy as np
+from lib.test.evaluation.data import Sequence, BaseDataset, SequenceList
+from lib.test.utils.load_text import load_text
+
+
+class NFSDataset(BaseDataset):
+    """ NFS dataset.
+    Publication:
+        Need for Speed: A Benchmark for Higher Frame Rate Object Tracking
+        H. Kiani Galoogahi, A. Fagg, C. Huang, D. Ramanan, and S.Lucey
+        ICCV, 2017
+        http://openaccess.thecvf.com/content_ICCV_2017/papers/Galoogahi_Need_for_Speed_ICCV_2017_paper.pdf
+    Download the dataset from http://ci2cv.net/nfs/index.html
+    """
+    def __init__(self):
+        super().__init__()
+        self.base_path = self.env_settings.nfs_path
+        self.sequence_info_list = self._get_sequence_info_list()
+
+    def get_sequence_list(self):
+        return SequenceList([self._construct_sequence(s) for s in self.sequence_info_list])
+
+    def _construct_sequence(self, sequence_info):
+        sequence_path = sequence_info['path']
+        nz = sequence_info['nz']
+        ext = sequence_info['ext']
+        start_frame = sequence_info['startFrame']
+        end_frame = sequence_info['endFrame']
+
+        init_omit = 0
+        if 'initOmit' in sequence_info:
+            init_omit = sequence_info['initOmit']
+
+        frames = ['{base_path}/{sequence_path}/{frame:0{nz}}.{ext}'.format(base_path=self.base_path, 
+        sequence_path=sequence_path, frame=frame_num, nz=nz, ext=ext) for frame_num in range(start_frame+init_omit, end_frame+1)]
+
+        # anno_path = '{}/{}'.format(self.base_path, sequence_info['anno_path'])
+        anno_path = f"{self.base_path}/{sequence_info['name'][4:]}/30/groundtruth.txt"
+
+        # ground_truth_rect = load_text(str(anno_path), delimiter='\t', dtype=np.float64)
+        ground_truth_rect = load_text(str(anno_path), delimiter=',', dtype=np.float64)
+
+        return Sequence(sequence_info['name'][4:], frames, 'nfs', ground_truth_rect[init_omit:,:],
+                        object_class=sequence_info['object_class'])
+
+    def __len__(self):
+        return len(self.sequence_info_list)
+
+    def _get_sequence_info_list(self):
+        sequence_info_list = [
+            {"name": "nfs_Gymnastics", "path": "sequences/Gymnastics", "startFrame": 1, "endFrame": 368, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_Gymnastics.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_MachLoop_jet", "path": "sequences/MachLoop_jet", "startFrame": 1, "endFrame": 99, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_MachLoop_jet.txt", "object_class": "aircraft", 'occlusion': False},
+            {"name": "nfs_Skiing_red", "path": "sequences/Skiing_red", "startFrame": 1, "endFrame": 69, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_Skiing_red.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_Skydiving", "path": "sequences/Skydiving", "startFrame": 1, "endFrame": 196, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_Skydiving.txt", "object_class": "person", 'occlusion': True},
+            {"name": "nfs_airboard_1", "path": "sequences/airboard_1", "startFrame": 1, "endFrame": 425, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_airboard_1.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_airplane_landing", "path": "sequences/airplane_landing", "startFrame": 1, "endFrame": 81, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_airplane_landing.txt", "object_class": "aircraft", 'occlusion': False},
+            {"name": "nfs_airtable_3", "path": "sequences/airtable_3", "startFrame": 1, "endFrame": 482, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_airtable_3.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_basketball_1", "path": "sequences/basketball_1", "startFrame": 1, "endFrame": 282, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_basketball_1.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_basketball_2", "path": "sequences/basketball_2", "startFrame": 1, "endFrame": 102, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_basketball_2.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_basketball_3", "path": "sequences/basketball_3", "startFrame": 1, "endFrame": 421, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_basketball_3.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_basketball_6", "path": "sequences/basketball_6", "startFrame": 1, "endFrame": 224, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_basketball_6.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_basketball_7", "path": "sequences/basketball_7", "startFrame": 1, "endFrame": 240, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_basketball_7.txt", "object_class": "person", 'occlusion': True},
+            {"name": "nfs_basketball_player", "path": "sequences/basketball_player", "startFrame": 1, "endFrame": 369, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_basketball_player.txt", "object_class": "person", 'occlusion': True},
+            {"name": "nfs_basketball_player_2", "path": "sequences/basketball_player_2", "startFrame": 1, "endFrame": 437, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_basketball_player_2.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_beach_flipback_person", "path": "sequences/beach_flipback_person", "startFrame": 1, "endFrame": 61, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_beach_flipback_person.txt", "object_class": "person head", 'occlusion': False},
+            {"name": "nfs_bee", "path": "sequences/bee", "startFrame": 1, "endFrame": 45, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_bee.txt", "object_class": "insect", 'occlusion': False},
+            {"name": "nfs_biker_acrobat", "path": "sequences/biker_acrobat", "startFrame": 1, "endFrame": 128, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_biker_acrobat.txt", "object_class": "bicycle", 'occlusion': False},
+            {"name": "nfs_biker_all_1", "path": "sequences/biker_all_1", "startFrame": 1, "endFrame": 113, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_biker_all_1.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_biker_head_2", "path": "sequences/biker_head_2", "startFrame": 1, "endFrame": 132, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_biker_head_2.txt", "object_class": "person head", 'occlusion': False},
+            {"name": "nfs_biker_head_3", "path": "sequences/biker_head_3", "startFrame": 1, "endFrame": 254, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_biker_head_3.txt", "object_class": "person head", 'occlusion': False},
+            {"name": "nfs_biker_upper_body", "path": "sequences/biker_upper_body", "startFrame": 1, "endFrame": 194, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_biker_upper_body.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_biker_whole_body", "path": "sequences/biker_whole_body", "startFrame": 1, "endFrame": 572, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_biker_whole_body.txt", "object_class": "person", 'occlusion': True},
+            {"name": "nfs_billiard_2", "path": "sequences/billiard_2", "startFrame": 1, "endFrame": 604, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_billiard_2.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_billiard_3", "path": "sequences/billiard_3", "startFrame": 1, "endFrame": 698, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_billiard_3.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_billiard_6", "path": "sequences/billiard_6", "startFrame": 1, "endFrame": 771, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_billiard_6.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_billiard_7", "path": "sequences/billiard_7", "startFrame": 1, "endFrame": 724, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_billiard_7.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_billiard_8", "path": "sequences/billiard_8", "startFrame": 1, "endFrame": 778, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_billiard_8.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_bird_2", "path": "sequences/bird_2", "startFrame": 1, "endFrame": 476, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_bird_2.txt", "object_class": "bird", 'occlusion': False},
+            {"name": "nfs_book", "path": "sequences/book", "startFrame": 1, "endFrame": 288, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_book.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_bottle", "path": "sequences/bottle", "startFrame": 1, "endFrame": 2103, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_bottle.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_bowling_1", "path": "sequences/bowling_1", "startFrame": 1, "endFrame": 303, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_bowling_1.txt", "object_class": "ball", 'occlusion': True},
+            {"name": "nfs_bowling_2", "path": "sequences/bowling_2", "startFrame": 1, "endFrame": 710, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_bowling_2.txt", "object_class": "ball", 'occlusion': True},
+            {"name": "nfs_bowling_3", "path": "sequences/bowling_3", "startFrame": 1, "endFrame": 271, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_bowling_3.txt", "object_class": "ball", 'occlusion': True},
+            {"name": "nfs_bowling_6", "path": "sequences/bowling_6", "startFrame": 1, "endFrame": 260, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_bowling_6.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_bowling_ball", "path": "sequences/bowling_ball", "startFrame": 1, "endFrame": 275, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_bowling_ball.txt", "object_class": "ball", 'occlusion': True},
+            {"name": "nfs_bunny", "path": "sequences/bunny", "startFrame": 1, "endFrame": 705, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_bunny.txt", "object_class": "mammal", 'occlusion': False},
+            {"name": "nfs_car", "path": "sequences/car", "startFrame": 1, "endFrame": 2020, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_car.txt", "object_class": "car", 'occlusion': True},
+            {"name": "nfs_car_camaro", "path": "sequences/car_camaro", "startFrame": 1, "endFrame": 36, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_car_camaro.txt", "object_class": "car", 'occlusion': False},
+            {"name": "nfs_car_drifting", "path": "sequences/car_drifting", "startFrame": 1, "endFrame": 173, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_car_drifting.txt", "object_class": "car", 'occlusion': False},
+            {"name": "nfs_car_jumping", "path": "sequences/car_jumping", "startFrame": 1, "endFrame": 22, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_car_jumping.txt", "object_class": "car", 'occlusion': False},
+            {"name": "nfs_car_rc_rolling", "path": "sequences/car_rc_rolling", "startFrame": 1, "endFrame": 62, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_car_rc_rolling.txt", "object_class": "car", 'occlusion': False},
+            {"name": "nfs_car_rc_rotating", "path": "sequences/car_rc_rotating", "startFrame": 1, "endFrame": 80, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_car_rc_rotating.txt", "object_class": "car", 'occlusion': False},
+            {"name": "nfs_car_side", "path": "sequences/car_side", "startFrame": 1, "endFrame": 108, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_car_side.txt", "object_class": "car", 'occlusion': False},
+            {"name": "nfs_car_white", "path": "sequences/car_white", "startFrame": 1, "endFrame": 2063, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_car_white.txt", "object_class": "car", 'occlusion': False},
+            {"name": "nfs_cheetah", "path": "sequences/cheetah", "startFrame": 1, "endFrame": 167, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_cheetah.txt", "object_class": "mammal", 'occlusion': True},
+            {"name": "nfs_cup", "path": "sequences/cup", "startFrame": 1, "endFrame": 1281, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_cup.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_cup_2", "path": "sequences/cup_2", "startFrame": 1, "endFrame": 182, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_cup_2.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_dog", "path": "sequences/dog", "startFrame": 1, "endFrame": 1030, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_dog.txt", "object_class": "dog", 'occlusion': True},
+            {"name": "nfs_dog_1", "path": "sequences/dog_1", "startFrame": 1, "endFrame": 168, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_dog_1.txt", "object_class": "dog", 'occlusion': False},
+            # {"name": "nfs_dog_2", "path": "sequences/dog_2", "startFrame": 1, "endFrame": 594, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_dog_2.txt", "object_class": "dog", 'occlusion': True},
+            {"name": "nfs_dog_3", "path": "sequences/dog_3", "startFrame": 1, "endFrame": 200, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_dog_3.txt", "object_class": "dog", 'occlusion': False},
+            {"name": "nfs_dogs", "path": "sequences/dogs", "startFrame": 1, "endFrame": 198, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_dogs.txt", "object_class": "dog", 'occlusion': True},
+            {"name": "nfs_dollar", "path": "sequences/dollar", "startFrame": 1, "endFrame": 1426, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_dollar.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_drone", "path": "sequences/drone", "startFrame": 1, "endFrame": 70, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_drone.txt", "object_class": "aircraft", 'occlusion': False},
+            {"name": "nfs_ducks_lake", "path": "sequences/ducks_lake", "startFrame": 1, "endFrame": 107, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_ducks_lake.txt", "object_class": "bird", 'occlusion': False},
+            {"name": "nfs_exit", "path": "sequences/exit", "startFrame": 1, "endFrame": 359, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_exit.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_first", "path": "sequences/first", "startFrame": 1, "endFrame": 435, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_first.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_flower", "path": "sequences/flower", "startFrame": 1, "endFrame": 448, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_flower.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_footbal_skill", "path": "sequences/footbal_skill", "startFrame": 1, "endFrame": 131, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_footbal_skill.txt", "object_class": "ball", 'occlusion': True},
+            {"name": "nfs_helicopter", "path": "sequences/helicopter", "startFrame": 1, "endFrame": 310, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_helicopter.txt", "object_class": "aircraft", 'occlusion': False},
+            {"name": "nfs_horse_jumping", "path": "sequences/horse_jumping", "startFrame": 1, "endFrame": 117, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_horse_jumping.txt", "object_class": "horse", 'occlusion': True},
+            {"name": "nfs_horse_running", "path": "sequences/horse_running", "startFrame": 1, "endFrame": 139, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_horse_running.txt", "object_class": "horse", 'occlusion': False},
+            {"name": "nfs_iceskating_6", "path": "sequences/iceskating_6", "startFrame": 1, "endFrame": 603, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_iceskating_6.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_jellyfish_5", "path": "sequences/jellyfish_5", "startFrame": 1, "endFrame": 746, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_jellyfish_5.txt", "object_class": "invertebrate", 'occlusion': False},
+            {"name": "nfs_kid_swing", "path": "sequences/kid_swing", "startFrame": 1, "endFrame": 169, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_kid_swing.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_motorcross", "path": "sequences/motorcross", "startFrame": 1, "endFrame": 39, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_motorcross.txt", "object_class": "vehicle", 'occlusion': True},
+            {"name": "nfs_motorcross_kawasaki", "path": "sequences/motorcross_kawasaki", "startFrame": 1, "endFrame": 65, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_motorcross_kawasaki.txt", "object_class": "vehicle", 'occlusion': False},
+            {"name": "nfs_parkour", "path": "sequences/parkour", "startFrame": 1, "endFrame": 58, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_parkour.txt", "object_class": "person head", 'occlusion': False},
+            {"name": "nfs_person_scooter", "path": "sequences/person_scooter", "startFrame": 1, "endFrame": 413, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_person_scooter.txt", "object_class": "person", 'occlusion': True},
+            {"name": "nfs_pingpong_2", "path": "sequences/pingpong_2", "startFrame": 1, "endFrame": 1277, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_pingpong_2.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_pingpong_7", "path": "sequences/pingpong_7", "startFrame": 1, "endFrame": 1290, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_pingpong_7.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_pingpong_8", "path": "sequences/pingpong_8", "startFrame": 1, "endFrame": 296, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_pingpong_8.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_purse", "path": "sequences/purse", "startFrame": 1, "endFrame": 968, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_purse.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_rubber", "path": "sequences/rubber", "startFrame": 1, "endFrame": 1328, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_rubber.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_running", "path": "sequences/running", "startFrame": 1, "endFrame": 677, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_running.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_running_100_m", "path": "sequences/running_100_m", "startFrame": 1, "endFrame": 313, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_running_100_m.txt", "object_class": "person", 'occlusion': True},
+            {"name": "nfs_running_100_m_2", "path": "sequences/running_100_m_2", "startFrame": 1, "endFrame": 337, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_running_100_m_2.txt", "object_class": "person", 'occlusion': True},
+            {"name": "nfs_running_2", "path": "sequences/running_2", "startFrame": 1, "endFrame": 363, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_running_2.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_shuffleboard_1", "path": "sequences/shuffleboard_1", "startFrame": 1, "endFrame": 42, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_shuffleboard_1.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_shuffleboard_2", "path": "sequences/shuffleboard_2", "startFrame": 1, "endFrame": 41, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_shuffleboard_2.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_shuffleboard_4", "path": "sequences/shuffleboard_4", "startFrame": 1, "endFrame": 62, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_shuffleboard_4.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_shuffleboard_5", "path": "sequences/shuffleboard_5", "startFrame": 1, "endFrame": 32, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_shuffleboard_5.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_shuffleboard_6", "path": "sequences/shuffleboard_6", "startFrame": 1, "endFrame": 52, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_shuffleboard_6.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_shuffletable_2", "path": "sequences/shuffletable_2", "startFrame": 1, "endFrame": 372, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_shuffletable_2.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_shuffletable_3", "path": "sequences/shuffletable_3", "startFrame": 1, "endFrame": 368, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_shuffletable_3.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_shuffletable_4", "path": "sequences/shuffletable_4", "startFrame": 1, "endFrame": 101, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_shuffletable_4.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_ski_long", "path": "sequences/ski_long", "startFrame": 1, "endFrame": 274, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_ski_long.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_soccer_ball", "path": "sequences/soccer_ball", "startFrame": 1, "endFrame": 163, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_soccer_ball.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_soccer_ball_2", "path": "sequences/soccer_ball_2", "startFrame": 1, "endFrame": 1934, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_soccer_ball_2.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_soccer_ball_3", "path": "sequences/soccer_ball_3", "startFrame": 1, "endFrame": 1381, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_soccer_ball_3.txt", "object_class": "ball", 'occlusion': False},
+            {"name": "nfs_soccer_player_2", "path": "sequences/soccer_player_2", "startFrame": 1, "endFrame": 475, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_soccer_player_2.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_soccer_player_3", "path": "sequences/soccer_player_3", "startFrame": 1, "endFrame": 319, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_soccer_player_3.txt", "object_class": "person", 'occlusion': True},
+            {"name": "nfs_stop_sign", "path": "sequences/stop_sign", "startFrame": 1, "endFrame": 302, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_stop_sign.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_suv", "path": "sequences/suv", "startFrame": 1, "endFrame": 2584, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_suv.txt", "object_class": "car", 'occlusion': False},
+            {"name": "nfs_tiger", "path": "sequences/tiger", "startFrame": 1, "endFrame": 1556, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_tiger.txt", "object_class": "mammal", 'occlusion': False},
+            {"name": "nfs_walking", "path": "sequences/walking", "startFrame": 1, "endFrame": 555, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_walking.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_walking_3", "path": "sequences/walking_3", "startFrame": 1, "endFrame": 1427, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_walking_3.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_water_ski_2", "path": "sequences/water_ski_2", "startFrame": 1, "endFrame": 47, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_water_ski_2.txt", "object_class": "person", 'occlusion': False},
+            {"name": "nfs_yoyo", "path": "sequences/yoyo", "startFrame": 1, "endFrame": 67, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_yoyo.txt", "object_class": "other", 'occlusion': False},
+            {"name": "nfs_zebra_fish", "path": "sequences/zebra_fish", "startFrame": 1, "endFrame": 671, "nz": 5, "ext": "jpg", "anno_path": "anno/nfs_zebra_fish.txt", "object_class": "fish", 'occlusion': False},
+        ]
+
+        return sequence_info_list

lib/test/evaluation/otbdataset.py

@@ -0,0 +1,259 @@
+import numpy as np
+from lib.test.evaluation.data import Sequence, BaseDataset, SequenceList
+from lib.test.utils.load_text import load_text
+
+
+class OTBDataset(BaseDataset):
+    """ OTB-2015 dataset
+    Publication:
+        Object Tracking Benchmark
+        Wu, Yi, Jongwoo Lim, and Ming-hsuan Yan
+        TPAMI, 2015
+        http://faculty.ucmerced.edu/mhyang/papers/pami15_tracking_benchmark.pdf
+    Download the dataset from http://cvlab.hanyang.ac.kr/tracker_benchmark/index.html
+    """
+    def __init__(self):
+        super().__init__()
+        self.base_path = self.env_settings.otb_path
+        self.sequence_info_list = self._get_sequence_info_list()
+
+    def get_sequence_list(self):
+        return SequenceList([self._construct_sequence(s) for s in self.sequence_info_list])
+
+    def _construct_sequence(self, sequence_info):
+        sequence_path = sequence_info['path']
+        nz = sequence_info['nz']
+        ext = sequence_info['ext']
+        start_frame = sequence_info['startFrame']
+        end_frame = sequence_info['endFrame']
+
+        init_omit = 0
+        if 'initOmit' in sequence_info:
+            init_omit = sequence_info['initOmit']
+
+        frames = ['{base_path}/{sequence_path}/{frame:0{nz}}.{ext}'.format(base_path=self.base_path, 
+        sequence_path=sequence_path, frame=frame_num, nz=nz, ext=ext) for frame_num in range(start_frame+init_omit, end_frame+1)]
+
+        # anno_path = '{}/{}'.format(self.base_path, sequence_info['anno_path'])
+        anno_path = '{}/{}/groundtruth.txt'.format(self.base_path, sequence_info['name'])
+
+        # NOTE: OTB has some weird annos which panda cannot handle
+        ground_truth_rect = load_text(str(anno_path), delimiter=(',', None), dtype=np.float64, backend='numpy')
+
+        return Sequence(sequence_info['name'], frames, 'otb', ground_truth_rect[init_omit:,:],
+                        object_class=sequence_info['object_class'])
+
+    def __len__(self):
+        return len(self.sequence_info_list)
+
+    def _get_sequence_info_list(self):
+        sequence_info_list = [
+            {"name": "Basketball", "path": "Basketball/img", "startFrame": 1, "endFrame": 725, "nz": 4, "ext": "jpg", "anno_path": "Basketball/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Biker", "path": "Biker/img", "startFrame": 1, "endFrame": 142, "nz": 4, "ext": "jpg", "anno_path": "Biker/groundtruth_rect.txt",
+             "object_class": "person head"},
+            {"name": "Bird1", "path": "Bird1/img", "startFrame": 1, "endFrame": 408, "nz": 4, "ext": "jpg", "anno_path": "Bird1/groundtruth_rect.txt",
+             "object_class": "bird"},
+            {"name": "Bird2", "path": "Bird2/img", "startFrame": 1, "endFrame": 99, "nz": 4, "ext": "jpg", "anno_path": "Bird2/groundtruth_rect.txt",
+             "object_class": "bird"},
+            {"name": "BlurBody", "path": "BlurBody/img", "startFrame": 1, "endFrame": 334, "nz": 4, "ext": "jpg", "anno_path": "BlurBody/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "BlurCar1", "path": "BlurCar1/img", "startFrame": 247, "endFrame": 988, "nz": 4, "ext": "jpg", "anno_path": "BlurCar1/groundtruth_rect.txt",
+             "object_class": "car"},
+            {"name": "BlurCar2", "path": "BlurCar2/img", "startFrame": 1, "endFrame": 585, "nz": 4, "ext": "jpg", "anno_path": "BlurCar2/groundtruth_rect.txt",
+             "object_class": "car"},
+            {"name": "BlurCar3", "path": "BlurCar3/img", "startFrame": 3, "endFrame": 359, "nz": 4, "ext": "jpg", "anno_path": "BlurCar3/groundtruth_rect.txt",
+             "object_class": "car"},
+            {"name": "BlurCar4", "path": "BlurCar4/img", "startFrame": 18, "endFrame": 397, "nz": 4, "ext": "jpg", "anno_path": "BlurCar4/groundtruth_rect.txt",
+             "object_class": "car"},
+            {"name": "BlurFace", "path": "BlurFace/img", "startFrame": 1, "endFrame": 493, "nz": 4, "ext": "jpg", "anno_path": "BlurFace/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "BlurOwl", "path": "BlurOwl/img", "startFrame": 1, "endFrame": 631, "nz": 4, "ext": "jpg", "anno_path": "BlurOwl/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Board", "path": "Board/img", "startFrame": 1, "endFrame": 698, "nz": 5, "ext": "jpg", "anno_path": "Board/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Bolt", "path": "Bolt/img", "startFrame": 1, "endFrame": 350, "nz": 4, "ext": "jpg", "anno_path": "Bolt/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Bolt2", "path": "Bolt2/img", "startFrame": 1, "endFrame": 293, "nz": 4, "ext": "jpg", "anno_path": "Bolt2/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Box", "path": "Box/img", "startFrame": 1, "endFrame": 1161, "nz": 4, "ext": "jpg", "anno_path": "Box/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Boy", "path": "Boy/img", "startFrame": 1, "endFrame": 602, "nz": 4, "ext": "jpg", "anno_path": "Boy/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Car1", "path": "Car1/img", "startFrame": 1, "endFrame": 1020, "nz": 4, "ext": "jpg", "anno_path": "Car1/groundtruth_rect.txt",
+             "object_class": "car"},
+            {"name": "Car2", "path": "Car2/img", "startFrame": 1, "endFrame": 913, "nz": 4, "ext": "jpg", "anno_path": "Car2/groundtruth_rect.txt",
+             "object_class": "car"},
+            {"name": "Car24", "path": "Car24/img", "startFrame": 1, "endFrame": 3059, "nz": 4, "ext": "jpg", "anno_path": "Car24/groundtruth_rect.txt",
+             "object_class": "car"},
+            {"name": "Car4", "path": "Car4/img", "startFrame": 1, "endFrame": 659, "nz": 4, "ext": "jpg", "anno_path": "Car4/groundtruth_rect.txt",
+             "object_class": "car"},
+            {"name": "CarDark", "path": "CarDark/img", "startFrame": 1, "endFrame": 393, "nz": 4, "ext": "jpg", "anno_path": "CarDark/groundtruth_rect.txt",
+             "object_class": "car"},
+            {"name": "CarScale", "path": "CarScale/img", "startFrame": 1, "endFrame": 252, "nz": 4, "ext": "jpg", "anno_path": "CarScale/groundtruth_rect.txt",
+             "object_class": "car"},
+            {"name": "ClifBar", "path": "ClifBar/img", "startFrame": 1, "endFrame": 472, "nz": 4, "ext": "jpg", "anno_path": "ClifBar/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Coke", "path": "Coke/img", "startFrame": 1, "endFrame": 291, "nz": 4, "ext": "jpg", "anno_path": "Coke/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Couple", "path": "Couple/img", "startFrame": 1, "endFrame": 140, "nz": 4, "ext": "jpg", "anno_path": "Couple/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Coupon", "path": "Coupon/img", "startFrame": 1, "endFrame": 327, "nz": 4, "ext": "jpg", "anno_path": "Coupon/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Crossing", "path": "Crossing/img", "startFrame": 1, "endFrame": 120, "nz": 4, "ext": "jpg", "anno_path": "Crossing/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Crowds", "path": "Crowds/img", "startFrame": 1, "endFrame": 347, "nz": 4, "ext": "jpg", "anno_path": "Crowds/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Dancer", "path": "Dancer/img", "startFrame": 1, "endFrame": 225, "nz": 4, "ext": "jpg", "anno_path": "Dancer/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Dancer2", "path": "Dancer2/img", "startFrame": 1, "endFrame": 150, "nz": 4, "ext": "jpg", "anno_path": "Dancer2/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "David", "path": "David/img", "startFrame": 300, "endFrame": 770, "nz": 4, "ext": "jpg", "anno_path": "David/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "David2", "path": "David2/img", "startFrame": 1, "endFrame": 537, "nz": 4, "ext": "jpg", "anno_path": "David2/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "David3", "path": "David3/img", "startFrame": 1, "endFrame": 252, "nz": 4, "ext": "jpg", "anno_path": "David3/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Deer", "path": "Deer/img", "startFrame": 1, "endFrame": 71, "nz": 4, "ext": "jpg", "anno_path": "Deer/groundtruth_rect.txt",
+             "object_class": "mammal"},
+            {"name": "Diving", "path": "Diving/img", "startFrame": 1, "endFrame": 215, "nz": 4, "ext": "jpg", "anno_path": "Diving/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Dog", "path": "Dog/img", "startFrame": 1, "endFrame": 127, "nz": 4, "ext": "jpg", "anno_path": "Dog/groundtruth_rect.txt",
+             "object_class": "dog"},
+            {"name": "Dog1", "path": "Dog1/img", "startFrame": 1, "endFrame": 1350, "nz": 4, "ext": "jpg", "anno_path": "Dog1/groundtruth_rect.txt",
+             "object_class": "dog"},
+            {"name": "Doll", "path": "Doll/img", "startFrame": 1, "endFrame": 3872, "nz": 4, "ext": "jpg", "anno_path": "Doll/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "DragonBaby", "path": "DragonBaby/img", "startFrame": 1, "endFrame": 113, "nz": 4, "ext": "jpg", "anno_path": "DragonBaby/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Dudek", "path": "Dudek/img", "startFrame": 1, "endFrame": 1145, "nz": 4, "ext": "jpg", "anno_path": "Dudek/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "FaceOcc1", "path": "FaceOcc1/img", "startFrame": 1, "endFrame": 892, "nz": 4, "ext": "jpg", "anno_path": "FaceOcc1/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "FaceOcc2", "path": "FaceOcc2/img", "startFrame": 1, "endFrame": 812, "nz": 4, "ext": "jpg", "anno_path": "FaceOcc2/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Fish", "path": "Fish/img", "startFrame": 1, "endFrame": 476, "nz": 4, "ext": "jpg", "anno_path": "Fish/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "FleetFace", "path": "FleetFace/img", "startFrame": 1, "endFrame": 707, "nz": 4, "ext": "jpg", "anno_path": "FleetFace/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Football", "path": "Football/img", "startFrame": 1, "endFrame": 362, "nz": 4, "ext": "jpg", "anno_path": "Football/groundtruth_rect.txt",
+             "object_class": "person head"},
+            {"name": "Football1", "path": "Football1/img", "startFrame": 1, "endFrame": 74, "nz": 4, "ext": "jpg", "anno_path": "Football1/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Freeman1", "path": "Freeman1/img", "startFrame": 1, "endFrame": 326, "nz": 4, "ext": "jpg", "anno_path": "Freeman1/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Freeman3", "path": "Freeman3/img", "startFrame": 1, "endFrame": 460, "nz": 4, "ext": "jpg", "anno_path": "Freeman3/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Freeman4", "path": "Freeman4/img", "startFrame": 1, "endFrame": 283, "nz": 4, "ext": "jpg", "anno_path": "Freeman4/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Girl", "path": "Girl/img", "startFrame": 1, "endFrame": 500, "nz": 4, "ext": "jpg", "anno_path": "Girl/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Girl2", "path": "Girl2/img", "startFrame": 1, "endFrame": 1500, "nz": 4, "ext": "jpg", "anno_path": "Girl2/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Gym", "path": "Gym/img", "startFrame": 1, "endFrame": 767, "nz": 4, "ext": "jpg", "anno_path": "Gym/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Human2", "path": "Human2/img", "startFrame": 1, "endFrame": 1128, "nz": 4, "ext": "jpg", "anno_path": "Human2/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Human3", "path": "Human3/img", "startFrame": 1, "endFrame": 1698, "nz": 4, "ext": "jpg", "anno_path": "Human3/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Human4_2", "path": "Human4/img", "startFrame": 1, "endFrame": 667, "nz": 4, "ext": "jpg", "anno_path": "Human4/groundtruth_rect.2.txt",
+             "object_class": "person"},
+            {"name": "Human4", "path": "Human4/img", "startFrame": 1, "endFrame": 667, "nz": 4, "ext": "jpg", "anno_path": "Human4/groundtruth_rect.2.txt",
+             "object_class": "person"},
+            {"name": "Human5", "path": "Human5/img", "startFrame": 1, "endFrame": 713, "nz": 4, "ext": "jpg", "anno_path": "Human5/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Human6", "path": "Human6/img", "startFrame": 1, "endFrame": 792, "nz": 4, "ext": "jpg", "anno_path": "Human6/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Human7", "path": "Human7/img", "startFrame": 1, "endFrame": 250, "nz": 4, "ext": "jpg", "anno_path": "Human7/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Human8", "path": "Human8/img", "startFrame": 1, "endFrame": 128, "nz": 4, "ext": "jpg", "anno_path": "Human8/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Human9", "path": "Human9/img", "startFrame": 1, "endFrame": 305, "nz": 4, "ext": "jpg", "anno_path": "Human9/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Ironman", "path": "Ironman/img", "startFrame": 1, "endFrame": 166, "nz": 4, "ext": "jpg", "anno_path": "Ironman/groundtruth_rect.txt",
+             "object_class": "person head"},
+            {"name": "Jogging", "path": "Jogging/img", "startFrame": 1, "endFrame": 307, "nz": 4, "ext": "jpg", "anno_path": "Jogging/groundtruth_rect.1.txt",
+             "object_class": "person"},
+            # {"name": "Jogging_1", "path": "Jogging/img", "startFrame": 1, "endFrame": 307, "nz": 4, "ext": "jpg", "anno_path": "Jogging/groundtruth_rect.1.txt",
+            #  "object_class": "person"},
+            # {"name": "Jogging_2", "path": "Jogging/img", "startFrame": 1, "endFrame": 307, "nz": 4, "ext": "jpg", "anno_path": "Jogging/groundtruth_rect.2.txt",
+            #  "object_class": "person"},
+            {"name": "Jump", "path": "Jump/img", "startFrame": 1, "endFrame": 122, "nz": 4, "ext": "jpg", "anno_path": "Jump/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Jumping", "path": "Jumping/img", "startFrame": 1, "endFrame": 313, "nz": 4, "ext": "jpg", "anno_path": "Jumping/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "KiteSurf", "path": "KiteSurf/img", "startFrame": 1, "endFrame": 84, "nz": 4, "ext": "jpg", "anno_path": "KiteSurf/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Lemming", "path": "Lemming/img", "startFrame": 1, "endFrame": 1336, "nz": 4, "ext": "jpg", "anno_path": "Lemming/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Liquor", "path": "Liquor/img", "startFrame": 1, "endFrame": 1741, "nz": 4, "ext": "jpg", "anno_path": "Liquor/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Man", "path": "Man/img", "startFrame": 1, "endFrame": 134, "nz": 4, "ext": "jpg", "anno_path": "Man/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Matrix", "path": "Matrix/img", "startFrame": 1, "endFrame": 100, "nz": 4, "ext": "jpg", "anno_path": "Matrix/groundtruth_rect.txt",
+             "object_class": "person head"},
+            {"name": "Mhyang", "path": "Mhyang/img", "startFrame": 1, "endFrame": 1490, "nz": 4, "ext": "jpg", "anno_path": "Mhyang/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "MotorRolling", "path": "MotorRolling/img", "startFrame": 1, "endFrame": 164, "nz": 4, "ext": "jpg", "anno_path": "MotorRolling/groundtruth_rect.txt",
+             "object_class": "vehicle"},
+            {"name": "MountainBike", "path": "MountainBike/img", "startFrame": 1, "endFrame": 228, "nz": 4, "ext": "jpg", "anno_path": "MountainBike/groundtruth_rect.txt",
+             "object_class": "bicycle"},
+            {"name": "Panda", "path": "Panda/img", "startFrame": 1, "endFrame": 1000, "nz": 4, "ext": "jpg", "anno_path": "Panda/groundtruth_rect.txt",
+             "object_class": "mammal"},
+            {"name": "RedTeam", "path": "RedTeam/img", "startFrame": 1, "endFrame": 1918, "nz": 4, "ext": "jpg", "anno_path": "RedTeam/groundtruth_rect.txt",
+             "object_class": "vehicle"},
+            {"name": "Rubik", "path": "Rubik/img", "startFrame": 1, "endFrame": 1997, "nz": 4, "ext": "jpg", "anno_path": "Rubik/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Shaking", "path": "Shaking/img", "startFrame": 1, "endFrame": 365, "nz": 4, "ext": "jpg", "anno_path": "Shaking/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Singer1", "path": "Singer1/img", "startFrame": 1, "endFrame": 351, "nz": 4, "ext": "jpg", "anno_path": "Singer1/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Singer2", "path": "Singer2/img", "startFrame": 1, "endFrame": 366, "nz": 4, "ext": "jpg", "anno_path": "Singer2/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Skater", "path": "Skater/img", "startFrame": 1, "endFrame": 160, "nz": 4, "ext": "jpg", "anno_path": "Skater/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Skater2", "path": "Skater2/img", "startFrame": 1, "endFrame": 435, "nz": 4, "ext": "jpg", "anno_path": "Skater2/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Skating1", "path": "Skating1/img", "startFrame": 1, "endFrame": 400, "nz": 4, "ext": "jpg", "anno_path": "Skating1/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Skating2", "path": "Skating2/img", "startFrame": 1, "endFrame": 473, "nz": 4, "ext": "jpg", "anno_path": "Skating2/groundtruth_rect.1.txt",
+             "object_class": "person"},
+            {"name": "Skating2_1", "path": "Skating2/img", "startFrame": 1, "endFrame": 473, "nz": 4, "ext": "jpg", "anno_path": "Skating2/groundtruth_rect.1.txt",
+             "object_class": "person"},
+            {"name": "Skating2_2", "path": "Skating2/img", "startFrame": 1, "endFrame": 473, "nz": 4, "ext": "jpg", "anno_path": "Skating2/groundtruth_rect.2.txt",
+             "object_class": "person"},
+            {"name": "Skiing", "path": "Skiing/img", "startFrame": 1, "endFrame": 81, "nz": 4, "ext": "jpg", "anno_path": "Skiing/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Soccer", "path": "Soccer/img", "startFrame": 1, "endFrame": 392, "nz": 4, "ext": "jpg", "anno_path": "Soccer/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Subway", "path": "Subway/img", "startFrame": 1, "endFrame": 175, "nz": 4, "ext": "jpg", "anno_path": "Subway/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Surfer", "path": "Surfer/img", "startFrame": 1, "endFrame": 376, "nz": 4, "ext": "jpg", "anno_path": "Surfer/groundtruth_rect.txt",
+             "object_class": "person head"},
+            {"name": "Suv", "path": "Suv/img", "startFrame": 1, "endFrame": 945, "nz": 4, "ext": "jpg", "anno_path": "Suv/groundtruth_rect.txt",
+             "object_class": "car"},
+            {"name": "Sylvester", "path": "Sylvester/img", "startFrame": 1, "endFrame": 1345, "nz": 4, "ext": "jpg", "anno_path": "Sylvester/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Tiger1", "path": "Tiger1/img", "startFrame": 1, "endFrame": 354, "nz": 4, "ext": "jpg", "anno_path": "Tiger1/groundtruth_rect.txt", "initOmit": 5,
+             "object_class": "other"},
+            {"name": "Tiger2", "path": "Tiger2/img", "startFrame": 1, "endFrame": 365, "nz": 4, "ext": "jpg", "anno_path": "Tiger2/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Toy", "path": "Toy/img", "startFrame": 1, "endFrame": 271, "nz": 4, "ext": "jpg", "anno_path": "Toy/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Trans", "path": "Trans/img", "startFrame": 1, "endFrame": 124, "nz": 4, "ext": "jpg", "anno_path": "Trans/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Trellis", "path": "Trellis/img", "startFrame": 1, "endFrame": 569, "nz": 4, "ext": "jpg", "anno_path": "Trellis/groundtruth_rect.txt",
+             "object_class": "face"},
+            {"name": "Twinnings", "path": "Twinnings/img", "startFrame": 1, "endFrame": 472, "nz": 4, "ext": "jpg", "anno_path": "Twinnings/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Vase", "path": "Vase/img", "startFrame": 1, "endFrame": 271, "nz": 4, "ext": "jpg", "anno_path": "Vase/groundtruth_rect.txt",
+             "object_class": "other"},
+            {"name": "Walking", "path": "Walking/img", "startFrame": 1, "endFrame": 412, "nz": 4, "ext": "jpg", "anno_path": "Walking/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Walking2", "path": "Walking2/img", "startFrame": 1, "endFrame": 500, "nz": 4, "ext": "jpg", "anno_path": "Walking2/groundtruth_rect.txt",
+             "object_class": "person"},
+            {"name": "Woman", "path": "Woman/img", "startFrame": 1, "endFrame": 597, "nz": 4, "ext": "jpg", "anno_path": "Woman/groundtruth_rect.txt",
+             "object_class": "person"}
+        ]
+    
+        return sequence_info_list

lib/test/evaluation/running.py

@@ -0,0 +1,183 @@
+import numpy as np
+import multiprocessing
+import os
+import sys
+from itertools import product
+from collections import OrderedDict
+from lib.test.evaluation import Sequence, Tracker
+import torch
+
+
+def _save_tracker_output(seq: Sequence, tracker: Tracker, output: dict):
+    """Saves the output of the tracker."""
+
+    if not os.path.exists(tracker.results_dir):
+        print("create tracking result dir:", tracker.results_dir)
+        os.makedirs(tracker.results_dir)
+    if seq.dataset in ['trackingnet', 'got10k']:
+        if not os.path.exists(os.path.join(tracker.results_dir, seq.dataset)):
+            os.makedirs(os.path.join(tracker.results_dir, seq.dataset))
+    '''2021.1.5 create new folder for these two datasets'''
+    if seq.dataset in ['trackingnet', 'got10k']:
+        base_results_path = os.path.join(tracker.results_dir, seq.dataset, seq.name)
+    else:
+        base_results_path = os.path.join(tracker.results_dir, seq.name)
+
+    def save_bb(file, data):
+        tracked_bb = np.array(data).astype(int)
+        np.savetxt(file, tracked_bb, delimiter='\t', fmt='%d')
+
+    def save_time(file, data):
+        exec_times = np.array(data).astype(float)
+        np.savetxt(file, exec_times, delimiter='\t', fmt='%f')
+
+    def save_score(file, data):
+        scores = np.array(data).astype(float)
+        np.savetxt(file, scores, delimiter='\t', fmt='%.2f')
+
+    def _convert_dict(input_dict):
+        data_dict = {}
+        for elem in input_dict:
+            for k, v in elem.items():
+                if k in data_dict.keys():
+                    data_dict[k].append(v)
+                else:
+                    data_dict[k] = [v, ]
+        return data_dict
+
+    for key, data in output.items():
+        # If data is empty
+        if not data:
+            continue
+
+        if key == 'target_bbox':
+            if isinstance(data[0], (dict, OrderedDict)):
+                data_dict = _convert_dict(data)
+
+                for obj_id, d in data_dict.items():
+                    bbox_file = '{}_{}.txt'.format(base_results_path, obj_id)
+                    save_bb(bbox_file, d)
+            else:
+                # Single-object mode
+                bbox_file = '{}.txt'.format(base_results_path)
+                save_bb(bbox_file, data)
+
+        if key == 'all_boxes':
+            if isinstance(data[0], (dict, OrderedDict)):
+                data_dict = _convert_dict(data)
+
+                for obj_id, d in data_dict.items():
+                    bbox_file = '{}_{}_all_boxes.txt'.format(base_results_path, obj_id)
+                    save_bb(bbox_file, d)
+            else:
+                # Single-object mode
+                bbox_file = '{}_all_boxes.txt'.format(base_results_path)
+                save_bb(bbox_file, data)
+
+        if key == 'all_scores':
+            if isinstance(data[0], (dict, OrderedDict)):
+                data_dict = _convert_dict(data)
+
+                for obj_id, d in data_dict.items():
+                    bbox_file = '{}_{}_all_scores.txt'.format(base_results_path, obj_id)
+                    save_score(bbox_file, d)
+            else:
+                # Single-object mode
+                print("saving scores...")
+                bbox_file = '{}_all_scores.txt'.format(base_results_path)
+                save_score(bbox_file, data)
+
+        elif key == 'time':
+            if isinstance(data[0], dict):
+                data_dict = _convert_dict(data)
+
+                for obj_id, d in data_dict.items():
+                    timings_file = '{}_{}_time.txt'.format(base_results_path, obj_id)
+                    save_time(timings_file, d)
+            else:
+                timings_file = '{}_time.txt'.format(base_results_path)
+                save_time(timings_file, data)
+
+
+def run_sequence(seq: Sequence, tracker: Tracker, debug=False, num_gpu=8):
+    """Runs a tracker on a sequence."""
+    '''2021.1.2 Add multiple gpu support'''
+    try:
+        worker_name = multiprocessing.current_process().name
+        worker_id = int(worker_name[worker_name.find('-') + 1:]) - 1
+        gpu_id = worker_id % num_gpu
+        torch.cuda.set_device(gpu_id)
+    except:
+        pass
+
+    def _results_exist():
+        if seq.object_ids is None:
+            if seq.dataset in ['trackingnet', 'got10k']:
+                base_results_path = os.path.join(tracker.results_dir, seq.dataset, seq.name)
+                bbox_file = '{}.txt'.format(base_results_path)
+            else:
+                bbox_file = '{}/{}.txt'.format(tracker.results_dir, seq.name)
+            return os.path.isfile(bbox_file)
+        else:
+            bbox_files = ['{}/{}_{}.txt'.format(tracker.results_dir, seq.name, obj_id) for obj_id in seq.object_ids]
+            missing = [not os.path.isfile(f) for f in bbox_files]
+            return sum(missing) == 0
+
+    if _results_exist() and not debug:
+        print('FPS: {}'.format(-1))
+        return
+
+    print('Tracker: {} {} {} ,  Sequence: {}'.format(tracker.name, tracker.parameter_name, tracker.run_id, seq.name))
+
+    if debug:
+        output = tracker.run_sequence(seq, debug=debug)
+    else:
+        try:
+            output = tracker.run_sequence(seq, debug=debug)
+        except Exception as e:
+            print(e)
+            return
+
+    sys.stdout.flush()
+
+    if isinstance(output['time'][0], (dict, OrderedDict)):
+        exec_time = sum([sum(times.values()) for times in output['time']])
+        num_frames = len(output['time'])
+    else:
+        exec_time = sum(output['time'])
+        num_frames = len(output['time'])
+
+    print('FPS: {}'.format(num_frames / exec_time))
+
+    if not debug:
+        _save_tracker_output(seq, tracker, output)
+
+
+def run_dataset(dataset, trackers, debug=False, threads=0, num_gpus=8):
+    """Runs a list of trackers on a dataset.
+    args:
+        dataset: List of Sequence instances, forming a dataset.
+        trackers: List of Tracker instances.
+        debug: Debug level.
+        threads: Number of threads to use (default 0).
+    """
+    multiprocessing.set_start_method('spawn', force=True)
+
+    print('Evaluating {:4d} trackers on {:5d} sequences'.format(len(trackers), len(dataset)))
+
+    multiprocessing.set_start_method('spawn', force=True)
+
+    if threads == 0:
+        mode = 'sequential'
+    else:
+        mode = 'parallel'
+
+    if mode == 'sequential':
+        for seq in dataset:
+            for tracker_info in trackers:
+                run_sequence(seq, tracker_info, debug=debug)
+    elif mode == 'parallel':
+        param_list = [(seq, tracker_info, debug, num_gpus) for seq, tracker_info in product(dataset, trackers)]
+        with multiprocessing.Pool(processes=threads) as pool:
+            pool.starmap(run_sequence, param_list)
+    print('Done')

lib/test/evaluation/tc128cedataset.py

@@ -0,0 +1,46 @@
+import numpy as np
+from lib.test.evaluation.data import Sequence, BaseDataset, SequenceList
+import os
+import glob
+import six
+
+
+class TC128CEDataset(BaseDataset):
+    """
+    TC-128 Dataset (78 newly added sequences)
+    modified from the implementation in got10k-toolkit (https://github.com/got-10k/toolkit)
+    """
+    def __init__(self):
+        super().__init__()
+        self.base_path = self.env_settings.tc128_path
+        self.anno_files = sorted(glob.glob(
+            os.path.join(self.base_path, '*/*_gt.txt')))
+        """filter the newly added sequences (_ce)"""
+        self.anno_files = [s for s in self.anno_files if "_ce" in s]
+        self.seq_dirs = [os.path.dirname(f) for f in self.anno_files]
+        self.seq_names = [os.path.basename(d) for d in self.seq_dirs]
+        # valid frame range for each sequence
+        self.range_files = [glob.glob(os.path.join(d, '*_frames.txt'))[0] for d in self.seq_dirs]
+
+    def get_sequence_list(self):
+        return SequenceList([self._construct_sequence(s) for s in self.seq_names])
+
+    def _construct_sequence(self, sequence_name):
+        if isinstance(sequence_name, six.string_types):
+            if not sequence_name in self.seq_names:
+                raise Exception('Sequence {} not found.'.format(sequence_name))
+            index = self.seq_names.index(sequence_name)
+        # load valid frame range
+        frames = np.loadtxt(self.range_files[index], dtype=int, delimiter=',')
+        img_files = [os.path.join(self.seq_dirs[index], 'img/%04d.jpg' % f) for f in range(frames[0], frames[1] + 1)]
+
+        # load annotations
+        anno = np.loadtxt(self.anno_files[index], delimiter=',')
+        assert len(img_files) == len(anno)
+        assert anno.shape[1] == 4
+
+        # return img_files, anno
+        return Sequence(sequence_name, img_files, 'tc128', anno.reshape(-1, 4))
+
+    def __len__(self):
+        return len(self.seq_names)

lib/test/evaluation/tc128dataset.py

@@ -0,0 +1,44 @@
+import numpy as np
+from lib.test.evaluation.data import Sequence, BaseDataset, SequenceList
+import os
+import glob
+import six
+
+
+class TC128Dataset(BaseDataset):
+    """
+    TC-128 Dataset
+    modified from the implementation in got10k-toolkit (https://github.com/got-10k/toolkit)
+    """
+    def __init__(self):
+        super().__init__()
+        self.base_path = self.env_settings.tc128_path
+        self.anno_files = sorted(glob.glob(
+            os.path.join(self.base_path, '*/*_gt.txt')))
+        self.seq_dirs = [os.path.dirname(f) for f in self.anno_files]
+        self.seq_names = [os.path.basename(d) for d in self.seq_dirs]
+        # valid frame range for each sequence
+        self.range_files = [glob.glob(os.path.join(d, '*_frames.txt'))[0] for d in self.seq_dirs]
+
+    def get_sequence_list(self):
+        return SequenceList([self._construct_sequence(s) for s in self.seq_names])
+
+    def _construct_sequence(self, sequence_name):
+        if isinstance(sequence_name, six.string_types):
+            if not sequence_name in self.seq_names:
+                raise Exception('Sequence {} not found.'.format(sequence_name))
+            index = self.seq_names.index(sequence_name)
+        # load valid frame range
+        frames = np.loadtxt(self.range_files[index], dtype=int, delimiter=',')
+        img_files = [os.path.join(self.seq_dirs[index], 'img/%04d.jpg' % f) for f in range(frames[0], frames[1] + 1)]
+
+        # load annotations
+        anno = np.loadtxt(self.anno_files[index], delimiter=',')
+        assert len(img_files) == len(anno)
+        assert anno.shape[1] == 4
+
+        # return img_files, anno
+        return Sequence(sequence_name, img_files, 'tc128', anno.reshape(-1, 4))
+
+    def __len__(self):
+        return len(self.seq_names)

lib/test/evaluation/tnl2kdataset.py

@@ -0,0 +1,50 @@
+import os
+
+import numpy as np
+from lib.test.evaluation.data import Sequence, BaseDataset, SequenceList
+from lib.test.utils.load_text import load_text, load_str
+
+############
+# current 00000492.png of test_015_Sord_video_Q01_done is damaged and replaced by a copy of 00000491.png
+############
+
+
+class TNL2kDataset(BaseDataset):
+    """
+    TNL2k test set
+    """
+    def __init__(self):
+        super().__init__()
+        self.base_path = self.env_settings.tnl2k_path
+        self.sequence_list = self._get_sequence_list()
+
+    def get_sequence_list(self):
+        return SequenceList([self._construct_sequence(s) for s in self.sequence_list])
+
+    def _construct_sequence(self, sequence_name):
+        # class_name = sequence_name.split('-')[0]
+        anno_path = '{}/{}/groundtruth.txt'.format(self.base_path, sequence_name)
+
+        ground_truth_rect = load_text(str(anno_path), delimiter=',', dtype=np.float64)
+
+        text_dsp_path = '{}/{}/language.txt'.format(self.base_path, sequence_name)
+        text_dsp = load_str(text_dsp_path)
+
+        frames_path = '{}/{}/imgs'.format(self.base_path, sequence_name)
+        frames_list = [f for f in os.listdir(frames_path)]
+        frames_list = sorted(frames_list)
+        frames_list = ['{}/{}'.format(frames_path, frame_i) for frame_i in frames_list]
+
+        # target_class = class_name
+        return Sequence(sequence_name, frames_list, 'tnl2k', ground_truth_rect.reshape(-1, 4), text_dsp=text_dsp)
+
+    def __len__(self):
+        return len(self.sequence_list)
+
+    def _get_sequence_list(self):
+        sequence_list = []
+        for seq in os.listdir(self.base_path):
+            if os.path.isdir(os.path.join(self.base_path, seq)):
+                sequence_list.append(seq)
+
+        return sequence_list

lib/test/evaluation/tracker.py

@@ -0,0 +1,291 @@
+import importlib
+import os
+from collections import OrderedDict
+from lib.test.evaluation.environment import env_settings
+import time
+import cv2 as cv
+
+from lib.utils.lmdb_utils import decode_img
+from pathlib import Path
+import numpy as np
+
+
+def trackerlist(name: str, parameter_name: str, dataset_name: str, run_ids = None, display_name: str = None,
+                result_only=False):
+    """Generate list of trackers.
+    args:
+        name: Name of tracking method.
+        parameter_name: Name of parameter file.
+        run_ids: A single or list of run_ids.
+        display_name: Name to be displayed in the result plots.
+    """
+    if run_ids is None or isinstance(run_ids, int):
+        run_ids = [run_ids]
+    return [Tracker(name, parameter_name, dataset_name, run_id, display_name, result_only) for run_id in run_ids]
+
+
+class Tracker:
+    """Wraps the tracker for evaluation and running purposes.
+    args:
+        name: Name of tracking method.
+        parameter_name: Name of parameter file.
+        run_id: The run id.
+        display_name: Name to be displayed in the result plots.
+    """
+
+    def __init__(self, name: str, parameter_name: str, dataset_name: str, run_id: int = None, display_name: str = None,
+                 result_only=False):
+        assert run_id is None or isinstance(run_id, int)
+
+        self.name = name
+        self.parameter_name = parameter_name
+        self.dataset_name = dataset_name
+        self.run_id = run_id
+        self.display_name = display_name
+
+        env = env_settings()
+        if self.run_id is None:
+            self.results_dir = '{}/{}/{}'.format(env.results_path, self.name, self.parameter_name)
+        else:
+            self.results_dir = '{}/{}/{}_{:03d}'.format(env.results_path, self.name, self.parameter_name, self.run_id)
+        if result_only:
+            self.results_dir = '{}/{}'.format(env.results_path, self.name)
+
+        tracker_module_abspath = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                                              '..', 'tracker', '%s.py' % self.name))
+        if os.path.isfile(tracker_module_abspath):
+            tracker_module = importlib.import_module('lib.test.tracker.{}'.format(self.name))
+            self.tracker_class = tracker_module.get_tracker_class()
+        else:
+            self.tracker_class = None
+
+    def create_tracker(self, params):
+        tracker = self.tracker_class(params, self.dataset_name)
+        return tracker
+
+    def run_sequence(self, seq, debug=None):
+        """Run tracker on sequence.
+        args:
+            seq: Sequence to run the tracker on.
+            visualization: Set visualization flag (None means default value specified in the parameters).
+            debug: Set debug level (None means default value specified in the parameters).
+            multiobj_mode: Which mode to use for multiple objects.
+        """
+        params = self.get_parameters()
+
+        debug_ = debug
+        if debug is None:
+            debug_ = getattr(params, 'debug', 0)
+
+        params.debug = debug_
+
+        # Get init information
+        init_info = seq.init_info()
+
+        tracker = self.create_tracker(params)
+
+        output = self._track_sequence(tracker, seq, init_info)
+        return output
+
+    def _track_sequence(self, tracker, seq, init_info):
+        # Define outputs
+        # Each field in output is a list containing tracker prediction for each frame.
+
+        # In case of single object tracking mode:
+        # target_bbox[i] is the predicted bounding box for frame i
+        # time[i] is the processing time for frame i
+
+        # In case of multi object tracking mode:
+        # target_bbox[i] is an OrderedDict, where target_bbox[i][obj_id] is the predicted box for target obj_id in
+        # frame i
+        # time[i] is either the processing time for frame i, or an OrderedDict containing processing times for each
+        # object in frame i
+
+        output = {'target_bbox': [],
+                  'time': []}
+        if tracker.params.save_all_boxes:
+            output['all_boxes'] = []
+            output['all_scores'] = []
+
+        def _store_outputs(tracker_out: dict, defaults=None):
+            defaults = {} if defaults is None else defaults
+            for key in output.keys():
+                val = tracker_out.get(key, defaults.get(key, None))
+                if key in tracker_out or val is not None:
+                    output[key].append(val)
+
+        # Initialize
+        image = self._read_image(seq.frames[0])
+
+        start_time = time.time()
+        out = tracker.initialize(image, init_info)
+        if out is None:
+            out = {}
+
+        prev_output = OrderedDict(out)
+        init_default = {'target_bbox': init_info.get('init_bbox'),
+                        'time': time.time() - start_time}
+        if tracker.params.save_all_boxes:
+            init_default['all_boxes'] = out['all_boxes']
+            init_default['all_scores'] = out['all_scores']
+
+        _store_outputs(out, init_default)
+
+        for frame_num, frame_path in enumerate(seq.frames[1:], start=1):
+            image = self._read_image(frame_path)
+
+            start_time = time.time()
+
+            info = seq.frame_info(frame_num)
+            info['previous_output'] = prev_output
+
+            if len(seq.ground_truth_rect) > 1:
+                info['gt_bbox'] = seq.ground_truth_rect[frame_num]
+            out = tracker.track(image, info)
+            prev_output = OrderedDict(out)
+            _store_outputs(out, {'time': time.time() - start_time})
+
+        for key in ['target_bbox', 'all_boxes', 'all_scores']:
+            if key in output and len(output[key]) <= 1:
+                output.pop(key)
+
+        return output
+
+    def run_video(self, videofilepath, optional_box=None, debug=None, visdom_info=None, save_results=False):
+        """Run the tracker with the vieofile.
+        args:
+            debug: Debug level.
+        """
+
+        params = self.get_parameters()
+
+        debug_ = debug
+        if debug is None:
+            debug_ = getattr(params, 'debug', 0)
+        params.debug = debug_
+
+        params.tracker_name = self.name
+        params.param_name = self.parameter_name
+        # self._init_visdom(visdom_info, debug_)
+
+        multiobj_mode = getattr(params, 'multiobj_mode', getattr(self.tracker_class, 'multiobj_mode', 'default'))
+
+        if multiobj_mode == 'default':
+            tracker = self.create_tracker(params)
+
+        elif multiobj_mode == 'parallel':
+            tracker = MultiObjectWrapper(self.tracker_class, params, self.visdom, fast_load=True)
+        else:
+            raise ValueError('Unknown multi object mode {}'.format(multiobj_mode))
+
+        assert os.path.isfile(videofilepath), "Invalid param {}".format(videofilepath)
+        ", videofilepath must be a valid videofile"
+
+        output_boxes = []
+
+        cap = cv.VideoCapture(videofilepath)
+        display_name = 'Display: ' + tracker.params.tracker_name
+        cv.namedWindow(display_name, cv.WINDOW_NORMAL | cv.WINDOW_KEEPRATIO)
+        cv.resizeWindow(display_name, 960, 720)
+        success, frame = cap.read()
+        cv.imshow(display_name, frame)
+
+        def _build_init_info(box):
+            return {'init_bbox': box}
+
+        if success is not True:
+            print("Read frame from {} failed.".format(videofilepath))
+            exit(-1)
+        if optional_box is not None:
+            assert isinstance(optional_box, (list, tuple))
+            assert len(optional_box) == 4, "valid box's foramt is [x,y,w,h]"
+            tracker.initialize(frame, _build_init_info(optional_box))
+            output_boxes.append(optional_box)
+        else:
+            while True:
+                # cv.waitKey()
+                frame_disp = frame.copy()
+
+                cv.putText(frame_disp, 'Select target ROI and press ENTER', (20, 30), cv.FONT_HERSHEY_COMPLEX_SMALL,
+                           1.5, (0, 0, 0), 1)
+
+                x, y, w, h = cv.selectROI(display_name, frame_disp, fromCenter=False)
+                init_state = [x, y, w, h]
+                tracker.initialize(frame, _build_init_info(init_state))
+                output_boxes.append(init_state)
+                break
+
+        while True:
+            ret, frame = cap.read()
+
+            if frame is None:
+                break
+
+            frame_disp = frame.copy()
+
+            # Draw box
+            out = tracker.track(frame)
+            state = [int(s) for s in out['target_bbox']]
+            output_boxes.append(state)
+
+            cv.rectangle(frame_disp, (state[0], state[1]), (state[2] + state[0], state[3] + state[1]),
+                         (0, 255, 0), 5)
+
+            font_color = (0, 0, 0)
+            cv.putText(frame_disp, 'Tracking!', (20, 30), cv.FONT_HERSHEY_COMPLEX_SMALL, 1,
+                       font_color, 1)
+            cv.putText(frame_disp, 'Press r to reset', (20, 55), cv.FONT_HERSHEY_COMPLEX_SMALL, 1,
+                       font_color, 1)
+            cv.putText(frame_disp, 'Press q to quit', (20, 80), cv.FONT_HERSHEY_COMPLEX_SMALL, 1,
+                       font_color, 1)
+
+            # Display the resulting frame
+            cv.imshow(display_name, frame_disp)
+            key = cv.waitKey(1)
+            if key == ord('q'):
+                break
+            elif key == ord('r'):
+                ret, frame = cap.read()
+                frame_disp = frame.copy()
+
+                cv.putText(frame_disp, 'Select target ROI and press ENTER', (20, 30), cv.FONT_HERSHEY_COMPLEX_SMALL, 1.5,
+                           (0, 0, 0), 1)
+
+                cv.imshow(display_name, frame_disp)
+                x, y, w, h = cv.selectROI(display_name, frame_disp, fromCenter=False)
+                init_state = [x, y, w, h]
+                tracker.initialize(frame, _build_init_info(init_state))
+                output_boxes.append(init_state)
+
+        # When everything done, release the capture
+        cap.release()
+        cv.destroyAllWindows()
+
+        if save_results:
+            if not os.path.exists(self.results_dir):
+                os.makedirs(self.results_dir)
+            video_name = Path(videofilepath).stem
+            base_results_path = os.path.join(self.results_dir, 'video_{}'.format(video_name))
+
+            tracked_bb = np.array(output_boxes).astype(int)
+            bbox_file = '{}.txt'.format(base_results_path)
+            np.savetxt(bbox_file, tracked_bb, delimiter='\t', fmt='%d')
+
+
+    def get_parameters(self):
+        """Get parameters."""
+        param_module = importlib.import_module('lib.test.parameter.{}'.format(self.name))
+        params = param_module.parameters(self.parameter_name)
+        return params
+
+    def _read_image(self, image_file: str):
+        if isinstance(image_file, str):
+            im = cv.imread(image_file)
+            return cv.cvtColor(im, cv.COLOR_BGR2RGB)
+        elif isinstance(image_file, list) and len(image_file) == 2:
+            return decode_img(image_file[0], image_file[1])
+        else:
+            raise ValueError("type of image_file should be str or list")
+
+
+

lib/test/evaluation/trackingnetdataset.py

@@ -0,0 +1,58 @@
+import numpy as np
+from lib.test.evaluation.data import Sequence, BaseDataset, SequenceList
+import os
+from lib.test.utils.load_text import load_text
+
+
+class TrackingNetDataset(BaseDataset):
+    """ TrackingNet test set.
+
+    Publication:
+        TrackingNet: A Large-Scale Dataset and Benchmark for Object Tracking in the Wild.
+        Matthias Mueller,Adel Bibi, Silvio Giancola, Salman Al-Subaihi and Bernard Ghanem
+        ECCV, 2018
+        https://ivul.kaust.edu.sa/Documents/Publications/2018/TrackingNet%20A%20Large%20Scale%20Dataset%20and%20Benchmark%20for%20Object%20Tracking%20in%20the%20Wild.pdf
+
+    Download the dataset using the toolkit https://github.com/SilvioGiancola/TrackingNet-devkit.
+    """
+    def __init__(self):
+        super().__init__()
+        self.base_path = self.env_settings.trackingnet_path
+
+        sets = 'TEST'
+        if not isinstance(sets, (list, tuple)):
+            if sets == 'TEST':
+                sets = ['TEST']
+            elif sets == 'TRAIN':
+                sets = ['TRAIN_{}'.format(i) for i in range(5)]
+
+        self.sequence_list = self._list_sequences(self.base_path, sets)
+
+    def get_sequence_list(self):
+        return SequenceList([self._construct_sequence(set, seq_name) for set, seq_name in self.sequence_list])
+
+    def _construct_sequence(self, set, sequence_name):
+        anno_path = '{}/{}/anno/{}.txt'.format(self.base_path, set, sequence_name)
+
+        ground_truth_rect = load_text(str(anno_path), delimiter=',', dtype=np.float64, backend='numpy')
+
+        frames_path = '{}/{}/frames/{}'.format(self.base_path, set, sequence_name)
+        frame_list = [frame for frame in os.listdir(frames_path) if frame.endswith(".jpg")]
+        frame_list.sort(key=lambda f: int(f[:-4]))
+        frames_list = [os.path.join(frames_path, frame) for frame in frame_list]
+
+        return Sequence(sequence_name, frames_list, 'trackingnet', ground_truth_rect.reshape(-1, 4))
+
+    def __len__(self):
+        return len(self.sequence_list)
+
+    def _list_sequences(self, root, set_ids):
+        sequence_list = []
+
+        for s in set_ids:
+            anno_dir = os.path.join(root, s, "anno")
+            sequences_cur_set = [(s, os.path.splitext(f)[0]) for f in os.listdir(anno_dir) if f.endswith('.txt')]
+
+            sequence_list += sequences_cur_set
+
+        return sequence_list

lib/test/evaluation/uavdataset.py

@@ -0,0 +1,298 @@
+import numpy as np
+from lib.test.evaluation.data import Sequence, BaseDataset, SequenceList
+from lib.test.utils.load_text import load_text
+
+
+class UAVDataset(BaseDataset):
+    """ UAV123 dataset.
+    Publication:
+        A Benchmark and Simulator for UAV Tracking.
+        Matthias Mueller, Neil Smith and Bernard Ghanem
+        ECCV, 2016
+        https://ivul.kaust.edu.sa/Documents/Publications/2016/A%20Benchmark%20and%20Simulator%20for%20UAV%20Tracking.pdf
+    Download the dataset from https://ivul.kaust.edu.sa/Pages/pub-benchmark-simulator-uav.aspx
+    """
+    def __init__(self):
+        super().__init__()
+        self.base_path = self.env_settings.uav_path
+        self.sequence_info_list = self._get_sequence_info_list()
+
+    def get_sequence_list(self):
+        # return SequenceList([self._construct_sequence(s) for s in self.sequence_info_list])
+        return SequenceList([self._construct_sequence(s) for s in self.sequence_info_list])
+
+    def _construct_sequence(self, sequence_info):
+        sequence_path = sequence_info['path']
+        nz = sequence_info['nz']
+        ext = sequence_info['ext']
+        start_frame = sequence_info['startFrame']
+        end_frame = sequence_info['endFrame']
+
+        init_omit = 0
+        if 'initOmit' in sequence_info:
+            init_omit = sequence_info['initOmit']
+
+        frames = ['{base_path}/{sequence_path}/{frame:0{nz}}.{ext}'.format(base_path=self.base_path, 
+        sequence_path=sequence_path, frame=frame_num, nz=nz, ext=ext) for frame_num in range(start_frame+init_omit, end_frame+1)]
+
+        anno_path = '{}/{}'.format(self.base_path, sequence_info['anno_path'])
+
+        ground_truth_rect = load_text(str(anno_path), delimiter=',', dtype=np.float64, backend='numpy')
+
+        return Sequence(sequence_info['name'][4:], frames, 'uav', ground_truth_rect[init_omit:,:],
+                        object_class=sequence_info['object_class'])
+
+    def __len__(self):
+        return len(self.sequence_info_list)
+
+    def _get_sequence_info_list(self):
+        sequence_info_list = [
+            {"name": "uav_bike1", "path": "data_seq/UAV123/bike1", "startFrame": 1, "endFrame": 3085, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/bike1.txt", "object_class": "vehicle"},
+            {"name": "uav_bike2", "path": "data_seq/UAV123/bike2", "startFrame": 1, "endFrame": 553, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/bike2.txt", "object_class": "vehicle"},
+            {"name": "uav_bike3", "path": "data_seq/UAV123/bike3", "startFrame": 1, "endFrame": 433, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/bike3.txt", "object_class": "vehicle"},
+            {"name": "uav_bird1_1", "path": "data_seq/UAV123/bird1", "startFrame": 1, "endFrame": 253, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/bird1_1.txt", "object_class": "bird"},
+            {"name": "uav_bird1_2", "path": "data_seq/UAV123/bird1", "startFrame": 775, "endFrame": 1477, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/bird1_2.txt", "object_class": "bird"},
+            {"name": "uav_bird1_3", "path": "data_seq/UAV123/bird1", "startFrame": 1573, "endFrame": 2437, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/bird1_3.txt", "object_class": "bird"},
+            {"name": "uav_boat1", "path": "data_seq/UAV123/boat1", "startFrame": 1, "endFrame": 901, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/boat1.txt", "object_class": "vessel"},
+            {"name": "uav_boat2", "path": "data_seq/UAV123/boat2", "startFrame": 1, "endFrame": 799, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/boat2.txt", "object_class": "vessel"},
+            {"name": "uav_boat3", "path": "data_seq/UAV123/boat3", "startFrame": 1, "endFrame": 901, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/boat3.txt", "object_class": "vessel"},
+            {"name": "uav_boat4", "path": "data_seq/UAV123/boat4", "startFrame": 1, "endFrame": 553, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/boat4.txt", "object_class": "vessel"},
+            {"name": "uav_boat5", "path": "data_seq/UAV123/boat5", "startFrame": 1, "endFrame": 505, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/boat5.txt", "object_class": "vessel"},
+            {"name": "uav_boat6", "path": "data_seq/UAV123/boat6", "startFrame": 1, "endFrame": 805, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/boat6.txt", "object_class": "vessel"},
+            {"name": "uav_boat7", "path": "data_seq/UAV123/boat7", "startFrame": 1, "endFrame": 535, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/boat7.txt", "object_class": "vessel"},
+            {"name": "uav_boat8", "path": "data_seq/UAV123/boat8", "startFrame": 1, "endFrame": 685, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/boat8.txt", "object_class": "vessel"},
+            {"name": "uav_boat9", "path": "data_seq/UAV123/boat9", "startFrame": 1, "endFrame": 1399, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/boat9.txt", "object_class": "vessel"},
+            {"name": "uav_building1", "path": "data_seq/UAV123/building1", "startFrame": 1, "endFrame": 469, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/building1.txt", "object_class": "other"},
+            {"name": "uav_building2", "path": "data_seq/UAV123/building2", "startFrame": 1, "endFrame": 577, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/building2.txt", "object_class": "other"},
+            {"name": "uav_building3", "path": "data_seq/UAV123/building3", "startFrame": 1, "endFrame": 829, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/building3.txt", "object_class": "other"},
+            {"name": "uav_building4", "path": "data_seq/UAV123/building4", "startFrame": 1, "endFrame": 787, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/building4.txt", "object_class": "other"},
+            {"name": "uav_building5", "path": "data_seq/UAV123/building5", "startFrame": 1, "endFrame": 481, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/building5.txt", "object_class": "other"},
+            {"name": "uav_car1_1", "path": "data_seq/UAV123/car1", "startFrame": 1, "endFrame": 751, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car1_1.txt", "object_class": "car"},
+            {"name": "uav_car1_2", "path": "data_seq/UAV123/car1", "startFrame": 751, "endFrame": 1627, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car1_2.txt", "object_class": "car"},
+            {"name": "uav_car1_3", "path": "data_seq/UAV123/car1", "startFrame": 1627, "endFrame": 2629, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car1_3.txt", "object_class": "car"},
+            {"name": "uav_car10", "path": "data_seq/UAV123/car10", "startFrame": 1, "endFrame": 1405, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car10.txt", "object_class": "car"},
+            {"name": "uav_car11", "path": "data_seq/UAV123/car11", "startFrame": 1, "endFrame": 337, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car11.txt", "object_class": "car"},
+            {"name": "uav_car12", "path": "data_seq/UAV123/car12", "startFrame": 1, "endFrame": 499, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car12.txt", "object_class": "car"},
+            {"name": "uav_car13", "path": "data_seq/UAV123/car13", "startFrame": 1, "endFrame": 415, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car13.txt", "object_class": "car"},
+            {"name": "uav_car14", "path": "data_seq/UAV123/car14", "startFrame": 1, "endFrame": 1327, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car14.txt", "object_class": "car"},
+            {"name": "uav_car15", "path": "data_seq/UAV123/car15", "startFrame": 1, "endFrame": 469, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car15.txt", "object_class": "car"},
+            {"name": "uav_car16_1", "path": "data_seq/UAV123/car16", "startFrame": 1, "endFrame": 415, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car16_1.txt", "object_class": "car"},
+            {"name": "uav_car16_2", "path": "data_seq/UAV123/car16", "startFrame": 415, "endFrame": 1993, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car16_2.txt", "object_class": "car"},
+            {"name": "uav_car17", "path": "data_seq/UAV123/car17", "startFrame": 1, "endFrame": 1057, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car17.txt", "object_class": "car"},
+            {"name": "uav_car18", "path": "data_seq/UAV123/car18", "startFrame": 1, "endFrame": 1207, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car18.txt", "object_class": "car"},
+            {"name": "uav_car1_s", "path": "data_seq/UAV123/car1_s", "startFrame": 1, "endFrame": 1475, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car1_s.txt", "object_class": "car"},
+            {"name": "uav_car2", "path": "data_seq/UAV123/car2", "startFrame": 1, "endFrame": 1321, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car2.txt", "object_class": "car"},
+            {"name": "uav_car2_s", "path": "data_seq/UAV123/car2_s", "startFrame": 1, "endFrame": 320, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car2_s.txt", "object_class": "car"},
+            {"name": "uav_car3", "path": "data_seq/UAV123/car3", "startFrame": 1, "endFrame": 1717, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car3.txt", "object_class": "car"},
+            {"name": "uav_car3_s", "path": "data_seq/UAV123/car3_s", "startFrame": 1, "endFrame": 1300, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car3_s.txt", "object_class": "car"},
+            {"name": "uav_car4", "path": "data_seq/UAV123/car4", "startFrame": 1, "endFrame": 1345, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car4.txt", "object_class": "car"},
+            {"name": "uav_car4_s", "path": "data_seq/UAV123/car4_s", "startFrame": 1, "endFrame": 830, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car4_s.txt", "object_class": "car"},
+            {"name": "uav_car5", "path": "data_seq/UAV123/car5", "startFrame": 1, "endFrame": 745, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car5.txt", "object_class": "car"},
+            {"name": "uav_car6_1", "path": "data_seq/UAV123/car6", "startFrame": 1, "endFrame": 487, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car6_1.txt", "object_class": "car"},
+            {"name": "uav_car6_2", "path": "data_seq/UAV123/car6", "startFrame": 487, "endFrame": 1807, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car6_2.txt", "object_class": "car"},
+            {"name": "uav_car6_3", "path": "data_seq/UAV123/car6", "startFrame": 1807, "endFrame": 2953, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car6_3.txt", "object_class": "car"},
+            {"name": "uav_car6_4", "path": "data_seq/UAV123/car6", "startFrame": 2953, "endFrame": 3925, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car6_4.txt", "object_class": "car"},
+            {"name": "uav_car6_5", "path": "data_seq/UAV123/car6", "startFrame": 3925, "endFrame": 4861, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car6_5.txt", "object_class": "car"},
+            {"name": "uav_car7", "path": "data_seq/UAV123/car7", "startFrame": 1, "endFrame": 1033, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car7.txt", "object_class": "car"},
+            {"name": "uav_car8_1", "path": "data_seq/UAV123/car8", "startFrame": 1, "endFrame": 1357, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car8_1.txt", "object_class": "car"},
+            {"name": "uav_car8_2", "path": "data_seq/UAV123/car8", "startFrame": 1357, "endFrame": 2575, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car8_2.txt", "object_class": "car"},
+            {"name": "uav_car9", "path": "data_seq/UAV123/car9", "startFrame": 1, "endFrame": 1879, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/car9.txt", "object_class": "car"},
+            {"name": "uav_group1_1", "path": "data_seq/UAV123/group1", "startFrame": 1, "endFrame": 1333, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/group1_1.txt", "object_class": "person"},
+            {"name": "uav_group1_2", "path": "data_seq/UAV123/group1", "startFrame": 1333, "endFrame": 2515, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/group1_2.txt", "object_class": "person"},
+            {"name": "uav_group1_3", "path": "data_seq/UAV123/group1", "startFrame": 2515, "endFrame": 3925, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/group1_3.txt", "object_class": "person"},
+            {"name": "uav_group1_4", "path": "data_seq/UAV123/group1", "startFrame": 3925, "endFrame": 4873, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/group1_4.txt", "object_class": "person"},
+            {"name": "uav_group2_1", "path": "data_seq/UAV123/group2", "startFrame": 1, "endFrame": 907, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/group2_1.txt", "object_class": "person"},
+            {"name": "uav_group2_2", "path": "data_seq/UAV123/group2", "startFrame": 907, "endFrame": 1771, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/group2_2.txt", "object_class": "person"},
+            {"name": "uav_group2_3", "path": "data_seq/UAV123/group2", "startFrame": 1771, "endFrame": 2683, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/group2_3.txt", "object_class": "person"},
+            {"name": "uav_group3_1", "path": "data_seq/UAV123/group3", "startFrame": 1, "endFrame": 1567, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/group3_1.txt", "object_class": "person"},
+            {"name": "uav_group3_2", "path": "data_seq/UAV123/group3", "startFrame": 1567, "endFrame": 2827, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/group3_2.txt", "object_class": "person"},
+            {"name": "uav_group3_3", "path": "data_seq/UAV123/group3", "startFrame": 2827, "endFrame": 4369, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/group3_3.txt", "object_class": "person"},
+            {"name": "uav_group3_4", "path": "data_seq/UAV123/group3", "startFrame": 4369, "endFrame": 5527, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/group3_4.txt", "object_class": "person"},
+            {"name": "uav_person1", "path": "data_seq/UAV123/person1", "startFrame": 1, "endFrame": 799, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person1.txt", "object_class": "person"},
+            {"name": "uav_person10", "path": "data_seq/UAV123/person10", "startFrame": 1, "endFrame": 1021, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person10.txt", "object_class": "person"},
+            {"name": "uav_person11", "path": "data_seq/UAV123/person11", "startFrame": 1, "endFrame": 721, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person11.txt", "object_class": "person"},
+            {"name": "uav_person12_1", "path": "data_seq/UAV123/person12", "startFrame": 1, "endFrame": 601, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person12_1.txt", "object_class": "person"},
+            {"name": "uav_person12_2", "path": "data_seq/UAV123/person12", "startFrame": 601, "endFrame": 1621, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person12_2.txt", "object_class": "person"},
+            {"name": "uav_person13", "path": "data_seq/UAV123/person13", "startFrame": 1, "endFrame": 883, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person13.txt", "object_class": "person"},
+            {"name": "uav_person14_1", "path": "data_seq/UAV123/person14", "startFrame": 1, "endFrame": 847, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person14_1.txt", "object_class": "person"},
+            {"name": "uav_person14_2", "path": "data_seq/UAV123/person14", "startFrame": 847, "endFrame": 1813, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person14_2.txt", "object_class": "person"},
+            {"name": "uav_person14_3", "path": "data_seq/UAV123/person14", "startFrame": 1813, "endFrame": 2923,
+             "nz": 6, "ext": "jpg", "anno_path": "anno/UAV123/person14_3.txt", "object_class": "person"},
+            {"name": "uav_person15", "path": "data_seq/UAV123/person15", "startFrame": 1, "endFrame": 1339, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person15.txt", "object_class": "person"},
+            {"name": "uav_person16", "path": "data_seq/UAV123/person16", "startFrame": 1, "endFrame": 1147, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person16.txt", "object_class": "person"},
+            {"name": "uav_person17_1", "path": "data_seq/UAV123/person17", "startFrame": 1, "endFrame": 1501, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person17_1.txt", "object_class": "person"},
+            {"name": "uav_person17_2", "path": "data_seq/UAV123/person17", "startFrame": 1501, "endFrame": 2347,
+             "nz": 6, "ext": "jpg", "anno_path": "anno/UAV123/person17_2.txt", "object_class": "person"},
+            {"name": "uav_person18", "path": "data_seq/UAV123/person18", "startFrame": 1, "endFrame": 1393, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person18.txt", "object_class": "person"},
+            {"name": "uav_person19_1", "path": "data_seq/UAV123/person19", "startFrame": 1, "endFrame": 1243, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person19_1.txt", "object_class": "person"},
+            {"name": "uav_person19_2", "path": "data_seq/UAV123/person19", "startFrame": 1243, "endFrame": 2791,
+             "nz": 6, "ext": "jpg", "anno_path": "anno/UAV123/person19_2.txt", "object_class": "person"},
+            {"name": "uav_person19_3", "path": "data_seq/UAV123/person19", "startFrame": 2791, "endFrame": 4357,
+             "nz": 6, "ext": "jpg", "anno_path": "anno/UAV123/person19_3.txt", "object_class": "person"},
+            {"name": "uav_person1_s", "path": "data_seq/UAV123/person1_s", "startFrame": 1, "endFrame": 1600, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person1_s.txt", "object_class": "person"},
+            {"name": "uav_person2_1", "path": "data_seq/UAV123/person2", "startFrame": 1, "endFrame": 1189, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person2_1.txt", "object_class": "person"},
+            {"name": "uav_person2_2", "path": "data_seq/UAV123/person2", "startFrame": 1189, "endFrame": 2623, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person2_2.txt", "object_class": "person"},
+            {"name": "uav_person20", "path": "data_seq/UAV123/person20", "startFrame": 1, "endFrame": 1783, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person20.txt", "object_class": "person"},
+            {"name": "uav_person21", "path": "data_seq/UAV123/person21", "startFrame": 1, "endFrame": 487, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person21.txt", "object_class": "person"},
+            {"name": "uav_person22", "path": "data_seq/UAV123/person22", "startFrame": 1, "endFrame": 199, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person22.txt", "object_class": "person"},
+            {"name": "uav_person23", "path": "data_seq/UAV123/person23", "startFrame": 1, "endFrame": 397, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person23.txt", "object_class": "person"},
+            {"name": "uav_person2_s", "path": "data_seq/UAV123/person2_s", "startFrame": 1, "endFrame": 250, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person2_s.txt", "object_class": "person"},
+            {"name": "uav_person3", "path": "data_seq/UAV123/person3", "startFrame": 1, "endFrame": 643, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person3.txt", "object_class": "person"},
+            {"name": "uav_person3_s", "path": "data_seq/UAV123/person3_s", "startFrame": 1, "endFrame": 505, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person3_s.txt", "object_class": "person"},
+            {"name": "uav_person4_1", "path": "data_seq/UAV123/person4", "startFrame": 1, "endFrame": 1501, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person4_1.txt", "object_class": "person"},
+            {"name": "uav_person4_2", "path": "data_seq/UAV123/person4", "startFrame": 1501, "endFrame": 2743, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person4_2.txt", "object_class": "person"},
+            {"name": "uav_person5_1", "path": "data_seq/UAV123/person5", "startFrame": 1, "endFrame": 877, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person5_1.txt", "object_class": "person"},
+            {"name": "uav_person5_2", "path": "data_seq/UAV123/person5", "startFrame": 877, "endFrame": 2101, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person5_2.txt", "object_class": "person"},
+            {"name": "uav_person6", "path": "data_seq/UAV123/person6", "startFrame": 1, "endFrame": 901, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person6.txt", "object_class": "person"},
+            {"name": "uav_person7_1", "path": "data_seq/UAV123/person7", "startFrame": 1, "endFrame": 1249, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person7_1.txt", "object_class": "person"},
+            {"name": "uav_person7_2", "path": "data_seq/UAV123/person7", "startFrame": 1249, "endFrame": 2065, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person7_2.txt", "object_class": "person"},
+            {"name": "uav_person8_1", "path": "data_seq/UAV123/person8", "startFrame": 1, "endFrame": 1075, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person8_1.txt", "object_class": "person"},
+            {"name": "uav_person8_2", "path": "data_seq/UAV123/person8", "startFrame": 1075, "endFrame": 1525, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person8_2.txt", "object_class": "person"},
+            {"name": "uav_person9", "path": "data_seq/UAV123/person9", "startFrame": 1, "endFrame": 661, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/person9.txt", "object_class": "person"},
+            {"name": "uav_truck1", "path": "data_seq/UAV123/truck1", "startFrame": 1, "endFrame": 463, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/truck1.txt", "object_class": "truck"},
+            {"name": "uav_truck2", "path": "data_seq/UAV123/truck2", "startFrame": 1, "endFrame": 385, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/truck2.txt", "object_class": "truck"},
+            {"name": "uav_truck3", "path": "data_seq/UAV123/truck3", "startFrame": 1, "endFrame": 535, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/truck3.txt", "object_class": "truck"},
+            {"name": "uav_truck4_1", "path": "data_seq/UAV123/truck4", "startFrame": 1, "endFrame": 577, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/truck4_1.txt", "object_class": "truck"},
+            {"name": "uav_truck4_2", "path": "data_seq/UAV123/truck4", "startFrame": 577, "endFrame": 1261, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/truck4_2.txt", "object_class": "truck"},
+            {"name": "uav_uav1_1", "path": "data_seq/UAV123/uav1", "startFrame": 1, "endFrame": 1555, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/uav1_1.txt", "object_class": "aircraft"},
+            {"name": "uav_uav1_2", "path": "data_seq/UAV123/uav1", "startFrame": 1555, "endFrame": 2377, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/uav1_2.txt", "object_class": "aircraft"},
+            {"name": "uav_uav1_3", "path": "data_seq/UAV123/uav1", "startFrame": 2473, "endFrame": 3469, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/uav1_3.txt", "object_class": "aircraft"},
+            {"name": "uav_uav2", "path": "data_seq/UAV123/uav2", "startFrame": 1, "endFrame": 133, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/uav2.txt", "object_class": "aircraft"},
+            {"name": "uav_uav3", "path": "data_seq/UAV123/uav3", "startFrame": 1, "endFrame": 265, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/uav3.txt", "object_class": "aircraft"},
+            {"name": "uav_uav4", "path": "data_seq/UAV123/uav4", "startFrame": 1, "endFrame": 157, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/uav4.txt", "object_class": "aircraft"},
+            {"name": "uav_uav5", "path": "data_seq/UAV123/uav5", "startFrame": 1, "endFrame": 139, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/uav5.txt", "object_class": "aircraft"},
+            {"name": "uav_uav6", "path": "data_seq/UAV123/uav6", "startFrame": 1, "endFrame": 109, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/uav6.txt", "object_class": "aircraft"},
+            {"name": "uav_uav7", "path": "data_seq/UAV123/uav7", "startFrame": 1, "endFrame": 373, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/uav7.txt", "object_class": "aircraft"},
+            {"name": "uav_uav8", "path": "data_seq/UAV123/uav8", "startFrame": 1, "endFrame": 301, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/uav8.txt", "object_class": "aircraft"},
+            {"name": "uav_wakeboard1", "path": "data_seq/UAV123/wakeboard1", "startFrame": 1, "endFrame": 421, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/wakeboard1.txt", "object_class": "person"},
+            {"name": "uav_wakeboard10", "path": "data_seq/UAV123/wakeboard10", "startFrame": 1, "endFrame": 469,
+             "nz": 6, "ext": "jpg", "anno_path": "anno/UAV123/wakeboard10.txt", "object_class": "person"},
+            {"name": "uav_wakeboard2", "path": "data_seq/UAV123/wakeboard2", "startFrame": 1, "endFrame": 733, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/wakeboard2.txt", "object_class": "person"},
+            {"name": "uav_wakeboard3", "path": "data_seq/UAV123/wakeboard3", "startFrame": 1, "endFrame": 823, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/wakeboard3.txt", "object_class": "person"},
+            {"name": "uav_wakeboard4", "path": "data_seq/UAV123/wakeboard4", "startFrame": 1, "endFrame": 697, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/wakeboard4.txt", "object_class": "person"},
+            {"name": "uav_wakeboard5", "path": "data_seq/UAV123/wakeboard5", "startFrame": 1, "endFrame": 1675, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/wakeboard5.txt", "object_class": "person"},
+            {"name": "uav_wakeboard6", "path": "data_seq/UAV123/wakeboard6", "startFrame": 1, "endFrame": 1165, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/wakeboard6.txt", "object_class": "person"},
+            {"name": "uav_wakeboard7", "path": "data_seq/UAV123/wakeboard7", "startFrame": 1, "endFrame": 199, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/wakeboard7.txt", "object_class": "person"},
+            {"name": "uav_wakeboard8", "path": "data_seq/UAV123/wakeboard8", "startFrame": 1, "endFrame": 1543, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/wakeboard8.txt", "object_class": "person"},
+            {"name": "uav_wakeboard9", "path": "data_seq/UAV123/wakeboard9", "startFrame": 1, "endFrame": 355, "nz": 6,
+             "ext": "jpg", "anno_path": "anno/UAV123/wakeboard9.txt", "object_class": "person"}
+        ]
+
+        return sequence_info_list

lib/test/evaluation/votdataset.py

@@ -0,0 +1,349 @@
+from typing import Union, TextIO
+
+import numpy as np
+from numba import jit
+
+from lib.test.evaluation.data import SequenceList, BaseDataset, Sequence
+
+
+class VOTDataset(BaseDataset):
+    """
+    VOT2018 dataset
+
+    Publication:
+        The sixth Visual Object Tracking VOT2018 challenge results.
+        Matej Kristan, Ales Leonardis, Jiri Matas, Michael Felsberg, Roman Pfugfelder, Luka Cehovin Zajc, Tomas Vojir,
+        Goutam Bhat, Alan Lukezic et al.
+        ECCV, 2018
+        https://prints.vicos.si/publications/365
+
+    Download the dataset from http://www.votchallenge.net/vot2018/dataset.html
+    """
+    def __init__(self, year=18):
+        super().__init__()
+        self.year = year
+        if year == 18:
+            self.base_path = self.env_settings.vot18_path
+        elif year == 20:
+            self.base_path = self.env_settings.vot20_path
+        elif year == 22:
+            self.base_path = self.env_settings.vot22_path
+        self.sequence_list = self._get_sequence_list(year)
+
+    def get_sequence_list(self):
+        return SequenceList([self._construct_sequence(s) for s in self.sequence_list])
+
+    def _construct_sequence(self, sequence_name):
+        sequence_path = sequence_name
+        nz = 8
+        ext = 'jpg'
+        start_frame = 1
+
+        anno_path = '{}/{}/groundtruth.txt'.format(self.base_path, sequence_name)
+
+        if self.year == 18 or self.year == 22:
+            try:
+                ground_truth_rect = np.loadtxt(str(anno_path), dtype=np.float64)
+            except:
+                ground_truth_rect = np.loadtxt(str(anno_path), delimiter=',', dtype=np.float64)
+
+            end_frame = ground_truth_rect.shape[0]
+
+            frames = ['{base_path}/{sequence_path}/color/{frame:0{nz}}.{ext}'.format(base_path=self.base_path,
+                      sequence_path=sequence_path, frame=frame_num, nz=nz, ext=ext)
+                      for frame_num in range(start_frame, end_frame+1)]
+
+            # Convert gt
+            if ground_truth_rect.shape[1] > 4:
+                gt_x_all = ground_truth_rect[:, [0, 2, 4, 6]]
+                gt_y_all = ground_truth_rect[:, [1, 3, 5, 7]]
+
+                x1 = np.amin(gt_x_all, 1).reshape(-1,1)
+                y1 = np.amin(gt_y_all, 1).reshape(-1,1)
+                x2 = np.amax(gt_x_all, 1).reshape(-1,1)
+                y2 = np.amax(gt_y_all, 1).reshape(-1,1)
+
+                ground_truth_rect = np.concatenate((x1, y1, x2-x1, y2-y1), 1)
+
+        elif self.year == 20:
+            ground_truth_rect = read_file(str(anno_path))
+            ground_truth_rect = np.array(ground_truth_rect, dtype=np.float64)
+            end_frame = ground_truth_rect.shape[0]
+
+            frames = ['{base_path}/{sequence_path}/color/{frame:0{nz}}.{ext}'.format(base_path=self.base_path,
+                                                                                     sequence_path=sequence_path,
+                                                                                     frame=frame_num, nz=nz, ext=ext)
+                      for frame_num in range(start_frame, end_frame + 1)]
+
+        else:
+            raise NotImplementedError
+
+        return Sequence(sequence_name, frames, 'vot', ground_truth_rect)
+
+    def __len__(self):
+        return len(self.sequence_list)
+
+    def _get_sequence_list(self, year):
+        if year == 18:
+            sequence_list= ['ants1',
+                            'ants3',
+                            'bag',
+                            'ball1',
+                            'ball2',
+                            'basketball',
+                            'birds1',
+                            'blanket',
+                            'bmx',
+                            'bolt1',
+                            'bolt2',
+                            'book',
+                            'butterfly',
+                            'car1',
+                            'conduction1',
+                            'crabs1',
+                            'crossing',
+                            'dinosaur',
+                            'drone_across',
+                            'drone_flip',
+                            'drone1',
+                            'fernando',
+                            'fish1',
+                            'fish2',
+                            'fish3',
+                            'flamingo1',
+                            'frisbee',
+                            'girl',
+                            'glove',
+                            'godfather',
+                            'graduate',
+                            'gymnastics1',
+                            'gymnastics2',
+                            'gymnastics3',
+                            'hand',
+                            'handball1',
+                            'handball2',
+                            'helicopter',
+                            'iceskater1',
+                            'iceskater2',
+                            'leaves',
+                            'matrix',
+                            'motocross1',
+                            'motocross2',
+                            'nature',
+                            'pedestrian1',
+                            'rabbit',
+                            'racing',
+                            'road',
+                            'shaking',
+                            'sheep',
+                            'singer2',
+                            'singer3',
+                            'soccer1',
+                            'soccer2',
+                            'soldier',
+                            'tiger',
+                            'traffic',
+                            'wiper',
+                            'zebrafish1']
+        elif year == 20:
+
+            sequence_list= ['agility',
+                            'ants1',
+                            'ball2',
+                            'ball3',
+                            'basketball',
+                            'birds1',
+                            'bolt1',
+                            'book',
+                            'butterfly',
+                            'car1',
+                            'conduction1',
+                            'crabs1',
+                            'dinosaur',
+                            'dribble',
+                            'drone1',
+                            'drone_across',
+                            'drone_flip',
+                            'fernando',
+                            'fish1',
+                            'fish2',
+                            'flamingo1',
+                            'frisbee',
+                            'girl',
+                            'glove',
+                            'godfather',
+                            'graduate',
+                            'gymnastics1',
+                            'gymnastics2',
+                            'gymnastics3',
+                            'hand',
+                            'hand02',
+                            'hand2',
+                            'handball1',
+                            'handball2',
+                            'helicopter',
+                            'iceskater1',
+                            'iceskater2',
+                            'lamb',
+                            'leaves',
+                            'marathon',
+                            'matrix',
+                            'monkey',
+                            'motocross1',
+                            'nature',
+                            'polo',
+                            'rabbit',
+                            'rabbit2',
+                            'road',
+                            'rowing',
+                            'shaking',
+                            'singer2',
+                            'singer3',
+                            'soccer1',
+                            'soccer2',
+                            'soldier',
+                            'surfing',
+                            'tiger',
+                            'wheel',
+                            'wiper',
+                            'zebrafish1']
+        elif year == 22:
+            sequence_list= ['agility',
+                            'animal',
+                            'ants1',
+                            'bag',
+                            'ball2',
+                            'ball3',
+                            'basketball',
+                            'birds1',
+                            'birds2',
+                            'bolt1',
+                            'book',
+                            'bubble',
+                            'butterfly',
+                            'car1',
+                            'conduction1',
+                            'crabs1',
+                            'dinosaur',
+                            'diver',
+                            'drone1',
+                            'drone_across',
+                            'fernando',
+                            'fish1',
+                            'fish2',
+                            'flamingo1',
+                            'frisbee',
+                            'girl',
+                            'graduate',
+                            'gymnastics1',
+                            'gymnastics2',
+                            'gymnastics3',
+                            'hand',
+                            'hand2',
+                            'handball1',
+                            'handball2',
+                            'helicopter',
+                            'iceskater1',
+                            'iceskater2',
+                            'kangaroo',
+                            'lamb',
+                            'leaves',
+                            'marathon',
+                            'matrix',
+                            'monkey',
+                            'motocross1',
+                            'nature',
+                            'polo',
+                            'rabbit',
+                            'rabbit2',
+                            'rowing',
+                            'shaking',
+                            'singer2',
+                            'singer3',
+                            'snake',
+                            'soccer1',
+                            'soccer2',
+                            'soldier',
+                            'surfing',
+                            'tennis',
+                            'tiger',
+                            'wheel',
+                            'wiper',
+                            'zebrafish1']
+
+        else:
+            raise NotImplementedError
+
+        return sequence_list
+
+
+def parse(string):
+    """
+    parse string to the appropriate region format and return region object
+    """
+    from vot.region.shapes import Rectangle, Polygon, Mask
+
+
+    if string[0] == 'm':
+        # input is a mask - decode it
+        m_, offset_, region = create_mask_from_string(string[1:].split(','))
+        # return Mask(m_, offset=offset_)
+        return region
+    else:
+        # input is not a mask - check if special, rectangle or polygon
+        raise NotImplementedError
+    print('Unknown region format.')
+    return None
+
+
+def read_file(fp: Union[str, TextIO]):
+    if isinstance(fp, str):
+        with open(fp) as file:
+            lines = file.readlines()
+    else:
+        lines = fp.readlines()
+
+    regions = []
+    # iterate over all lines in the file
+    for i, line in enumerate(lines):
+        regions.append(parse(line.strip()))
+    return regions
+
+
+def create_mask_from_string(mask_encoding):
+    """
+    mask_encoding: a string in the following format: x0, y0, w, h, RLE
+    output: mask, offset
+    mask: 2-D binary mask, size defined in the mask encoding
+    offset: (x, y) offset of the mask in the image coordinates
+    """
+    elements = [int(el) for el in mask_encoding]
+    tl_x, tl_y, region_w, region_h = elements[:4]
+    rle = np.array([el for el in elements[4:]], dtype=np.int32)
+
+    # create mask from RLE within target region
+    mask = rle_to_mask(rle, region_w, region_h)
+    region = [tl_x, tl_y, region_w, region_h]
+
+    return mask, (tl_x, tl_y), region
+
+@jit(nopython=True)
+def rle_to_mask(rle, width, height):
+    """
+    rle: input rle mask encoding
+    each evenly-indexed element represents number of consecutive 0s
+    each oddly indexed element represents number of consecutive 1s
+    width and height are dimensions of the mask
+    output: 2-D binary mask
+    """
+    # allocate list of zeros
+    v = [0] * (width * height)
+
+    # set id of the last different element to the beginning of the vector
+    idx_ = 0
+    for i in range(len(rle)):
+        if i % 2 != 0:
+            # write as many 1s as RLE says (zeros are already in the vector)
+            for j in range(rle[i]):
+                v[idx_+j] = 1
+        idx_ += rle[i]

lib/test/parameter/artrack.py

@@ -0,0 +1,30 @@
+from lib.test.utils import TrackerParams
+import os
+from lib.test.evaluation.environment import env_settings
+from lib.config.artrack.config import cfg, update_config_from_file
+
+
+def parameters(yaml_name: str):
+    params = TrackerParams()
+    prj_dir = env_settings().prj_dir
+    save_dir = env_settings().save_dir
+    # update default config from yaml file
+    yaml_file = os.path.join(prj_dir, 'experiments/artrack/%s.yaml' % yaml_name)
+    update_config_from_file(yaml_file)
+    params.cfg = cfg
+    print("test config: ", cfg)
+
+    # template and search region
+    params.template_factor = cfg.TEST.TEMPLATE_FACTOR
+    params.template_size = cfg.TEST.TEMPLATE_SIZE
+    params.search_factor = cfg.TEST.SEARCH_FACTOR
+    params.search_size = cfg.TEST.SEARCH_SIZE
+
+    # Network checkpoint path
+    params.checkpoint = os.path.join(save_dir, "checkpoints/train/artrack/%s/ARTrack_ep%04d.pth.tar" %
+                                     (yaml_name, cfg.TEST.EPOCH))
+
+    # whether to save boxes from all queries
+    params.save_all_boxes = False
+
+    return params

lib/test/parameter/artrack_seq.py

@@ -0,0 +1,30 @@
+from lib.test.utils import TrackerParams
+import os
+from lib.test.evaluation.environment import env_settings
+from lib.config.artrack_seq.config import cfg, update_config_from_file
+
+
+def parameters(yaml_name: str):
+    params = TrackerParams()
+    prj_dir = env_settings().prj_dir
+    save_dir = env_settings().save_dir
+    # update default config from yaml file
+    yaml_file = os.path.join(prj_dir, 'experiments/artrack_seq/%s.yaml' % yaml_name)
+    update_config_from_file(yaml_file)
+    params.cfg = cfg
+    print("test config: ", cfg)
+
+    # template and search region
+    params.template_factor = cfg.TEST.TEMPLATE_FACTOR
+    params.template_size = cfg.TEST.TEMPLATE_SIZE
+    params.search_factor = cfg.TEST.SEARCH_FACTOR
+    params.search_size = cfg.TEST.SEARCH_SIZE
+
+    # Network checkpoint path
+    params.checkpoint = os.path.join(save_dir, "checkpoints/train/artrack_seq/%s/ARTrackSeq_ep%04d.pth.tar" %
+                                     (yaml_name, cfg.TEST.EPOCH))
+
+    # whether to save boxes from all queries
+    params.save_all_boxes = False
+
+    return params

lib/test/tracker/artrack.py

@@ -0,0 +1,225 @@
+import math
+
+from lib.models.artrack import build_artrack
+from lib.test.tracker.basetracker import BaseTracker
+import torch
+
+from lib.test.tracker.vis_utils import gen_visualization
+from lib.test.utils.hann import hann2d
+from lib.train.data.processing_utils import sample_target
+# for debug
+import cv2
+import os
+
+from lib.test.tracker.data_utils import Preprocessor
+from lib.utils.box_ops import clip_box
+from lib.utils.ce_utils import generate_mask_cond
+import random
+
+class RandomErasing(object):
+    def __init__(self, EPSILON=0.5, sl=0.02, sh=0.33, r1=0.3, mean=[0.4914, 0.4822, 0.4465]):
+        self.EPSILON = EPSILON
+        self.mean = mean
+        self.sl = sl
+        self.sh = sh
+        self.r1 = r1
+
+    def __call__(self, img):
+
+        if random.uniform(0, 1) > self.EPSILON:
+            return img
+
+        for attempt in range(100):
+            print(img.size())
+            area = img.size()[1] * img.size()[2]
+
+            target_area = random.uniform(self.sl, self.sh) * area
+            aspect_ratio = random.uniform(self.r1, 1 / self.r1)
+
+            h = int(round(math.sqrt(target_area * aspect_ratio)))
+            w = int(round(math.sqrt(target_area / aspect_ratio)))
+
+            if w < img.size()[2] and h < img.size()[1]:
+                x1 = random.randint(0, img.size()[1] - h)
+                y1 = random.randint(0, img.size()[2] - w)
+                if img.size()[0] == 3:
+                    # img[0, x1:x1+h, y1:y1+w] = random.uniform(0, 1)
+                    # img[1, x1:x1+h, y1:y1+w] = random.uniform(0, 1)
+                    # img[2, x1:x1+h, y1:y1+w] = random.uniform(0, 1)
+                    img[0, x1:x1 + h, y1:y1 + w] = self.mean[0]
+                    img[1, x1:x1 + h, y1:y1 + w] = self.mean[1]
+                    img[2, x1:x1 + h, y1:y1 + w] = self.mean[2]
+                    # img[:, x1:x1+h, y1:y1+w] = torch.from_numpy(np.random.rand(3, h, w))
+                else:
+                    img[0, x1:x1 + h, y1:y1 + w] = self.mean[1]
+                    # img[0, x1:x1+h, y1:y1+w] = torch.from_numpy(np.random.rand(1, h, w))
+                return img
+
+        return img
+
+
+class ARTrack(BaseTracker):
+    def __init__(self, params, dataset_name):
+        super(ARTrack, self).__init__(params)
+        network = build_artrack(params.cfg, training=False)
+        print(self.params.checkpoint)
+        network.load_state_dict(torch.load(self.params.checkpoint, map_location='cpu')['net'], strict=True)
+        self.cfg = params.cfg
+        self.bins = self.cfg.MODEL.BINS
+        self.network = network.cuda()
+        self.network.eval()
+        self.preprocessor = Preprocessor()
+        self.state = None
+        self.range = self.cfg.MODEL.RANGE
+
+        self.feat_sz = self.cfg.TEST.SEARCH_SIZE // self.cfg.MODEL.BACKBONE.STRIDE
+        # motion constrain
+        self.output_window = hann2d(torch.tensor([self.feat_sz, self.feat_sz]).long(), centered=True).cuda()
+
+        # for debug
+        self.debug = params.debug
+        self.use_visdom = params.debug
+        self.frame_id = 0
+        self.erase = RandomErasing()
+        if self.debug:
+            if not self.use_visdom:
+                self.save_dir = "debug"
+                if not os.path.exists(self.save_dir):
+                    os.makedirs(self.save_dir)
+            else:
+                # self.add_hook()
+                self._init_visdom(None, 1)
+        # for save boxes from all queries
+        self.save_all_boxes = params.save_all_boxes
+        self.z_dict1 = {}
+
+    def initialize(self, image, info: dict):
+        # forward the template once
+
+        z_patch_arr, resize_factor, z_amask_arr = sample_target(image, info['init_bbox'], self.params.template_factor,
+                                                    output_sz=self.params.template_size)#output_sz=self.params.template_size
+        self.z_patch_arr = z_patch_arr
+        template = self.preprocessor.process(z_patch_arr, z_amask_arr)
+        with torch.no_grad():
+            self.z_dict1 = template
+
+        self.box_mask_z = None
+        #if self.cfg.MODEL.BACKBONE.CE_LOC:
+        #    template_bbox = self.transform_bbox_to_crop(info['init_bbox'], resize_factor,
+        #                                                template.tensors.device).squeeze(1)
+        #    self.box_mask_z = generate_mask_cond(self.cfg, 1, template.tensors.device, template_bbox)
+
+        # save states
+        self.state = info['init_bbox']
+        self.frame_id = 0
+        if self.save_all_boxes:
+            '''save all predicted boxes'''
+            all_boxes_save = info['init_bbox'] * self.cfg.MODEL.NUM_OBJECT_QUERIES
+            return {"all_boxes": all_boxes_save}
+
+    def track(self, image, info: dict = None):
+        magic_num = (self.range - 1) * 0.5
+        H, W, _ = image.shape
+        self.frame_id += 1
+        x_patch_arr, resize_factor, x_amask_arr = sample_target(image, self.state, self.params.search_factor,
+                                                                output_sz=self.params.search_size)  # (x1, y1, w, h)
+        search = self.preprocessor.process(x_patch_arr, x_amask_arr)
+
+
+        with torch.no_grad():
+            x_dict = search
+            # merge the template and the search
+            # run the transformer
+            out_dict = self.network.forward(
+                template=self.z_dict1.tensors, search=x_dict.tensors)
+
+        # add hann windows
+        # pred_score_map = out_dict['score_map']
+        # response = self.output_window * pred_score_map
+        # pred_boxes = self.network.box_head.cal_bbox(response, out_dict['size_map'], out_dict['offset_map'])
+        # pred_boxes = pred_boxes.view(-1, 4)
+
+        pred_boxes = out_dict['seqs'][:, 0:4] / (self.bins - 1) - magic_num
+        pred_boxes = pred_boxes.view(-1, 4).mean(dim=0)
+        pred_new = pred_boxes
+        pred_new[2] = pred_boxes[2] - pred_boxes[0]
+        pred_new[3] = pred_boxes[3] - pred_boxes[1]
+        pred_new[0] = pred_boxes[0] + pred_boxes[2]/2
+        pred_new[1] = pred_boxes[1] + pred_boxes[3]/2
+
+        pred_boxes = (pred_new * self.params.search_size / resize_factor).tolist()
+
+        # Baseline: Take the mean of all pred boxes as the final result
+        #pred_box = (pred_boxes.mean(
+        #    dim=0) * self.params.search_size / resize_factor).tolist()  # (cx, cy, w, h) [0,1]
+        # get the final box result
+        self.state = clip_box(self.map_box_back(pred_boxes, resize_factor), H, W, margin=10)
+
+        # for debug
+        if self.debug:
+            if not self.use_visdom:
+                x1, y1, w, h = self.state
+                image_BGR = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+                cv2.rectangle(image_BGR, (int(x1),int(y1)), (int(x1+w),int(y1+h)), color=(0,0,255), thickness=2)
+                save_path = os.path.join(self.save_dir, "%04d.jpg" % self.frame_id)
+                cv2.imwrite(save_path, image_BGR)
+            else:
+                self.visdom.register((image, info['gt_bbox'].tolist(), self.state), 'Tracking', 1, 'Tracking')
+
+                self.visdom.register(torch.from_numpy(x_patch_arr).permute(2, 0, 1), 'image', 1, 'search_region')
+                self.visdom.register(torch.from_numpy(self.z_patch_arr).permute(2, 0, 1), 'image', 1, 'template')
+                self.visdom.register(pred_score_map.view(self.feat_sz, self.feat_sz), 'heatmap', 1, 'score_map')
+                self.visdom.register((pred_score_map * self.output_window).view(self.feat_sz, self.feat_sz), 'heatmap', 1, 'score_map_hann')
+
+                if 'removed_indexes_s' in out_dict and out_dict['removed_indexes_s']:
+                    removed_indexes_s = out_dict['removed_indexes_s']
+                    removed_indexes_s = [removed_indexes_s_i.cpu().numpy() for removed_indexes_s_i in removed_indexes_s]
+                    masked_search = gen_visualization(x_patch_arr, removed_indexes_s)
+                    self.visdom.register(torch.from_numpy(masked_search).permute(2, 0, 1), 'image', 1, 'masked_search')
+
+                while self.pause_mode:
+                    if self.step:
+                        self.step = False
+                        break
+
+        if self.save_all_boxes:
+            '''save all predictions'''
+            all_boxes = self.map_box_back_batch(pred_boxes * self.params.search_size / resize_factor, resize_factor)
+            all_boxes_save = all_boxes.view(-1).tolist()  # (4N, )
+            return {"target_bbox": self.state,
+                    "all_boxes": all_boxes_save}
+        else:
+            return {"target_bbox": self.state}
+
+    def map_box_back(self, pred_box: list, resize_factor: float):
+        cx_prev, cy_prev = self.state[0] + 0.5 * self.state[2], self.state[1] + 0.5 * self.state[3]
+        cx, cy, w, h = pred_box
+        half_side = 0.5 * self.params.search_size / resize_factor
+        cx_real = cx + (cx_prev - half_side)
+        cy_real = cy + (cy_prev - half_side)
+        #cx_real = cx + cx_prev
+        #cy_real = cy + cy_prev
+        return [cx_real - 0.5 * w, cy_real - 0.5 * h, w, h]
+
+    def map_box_back_batch(self, pred_box: torch.Tensor, resize_factor: float):
+        cx_prev, cy_prev = self.state[0] + 0.5 * self.state[2], self.state[1] + 0.5 * self.state[3]
+        cx, cy, w, h = pred_box.unbind(-1) # (N,4) --> (N,)
+        half_side = 0.5 * self.params.search_size / resize_factor
+        cx_real = cx + (cx_prev - half_side)
+        cy_real = cy + (cy_prev - half_side)
+        return torch.stack([cx_real - 0.5 * w, cy_real - 0.5 * h, w, h], dim=-1)
+
+    def add_hook(self):
+        conv_features, enc_attn_weights, dec_attn_weights = [], [], []
+
+        for i in range(12):
+            self.network.backbone.blocks[i].attn.register_forward_hook(
+                # lambda self, input, output: enc_attn_weights.append(output[1])
+                lambda self, input, output: enc_attn_weights.append(output[1])
+            )
+
+        self.enc_attn_weights = enc_attn_weights
+
+
+def get_tracker_class():
+    return ARTrack

lib/test/tracker/artrack_seq.py

@@ -0,0 +1,209 @@
+import math
+
+from lib.models.artrack_seq import build_artrack_seq
+from lib.test.tracker.basetracker import BaseTracker
+import torch
+
+from lib.test.tracker.vis_utils import gen_visualization
+from lib.test.utils.hann import hann2d
+from lib.train.data.processing_utils import sample_target, transform_image_to_crop
+# for debug
+import cv2
+import os
+
+from lib.test.tracker.data_utils import Preprocessor
+from lib.utils.box_ops import clip_box
+from lib.utils.ce_utils import generate_mask_cond
+
+
+class ARTrackSeq(BaseTracker):
+    def __init__(self, params, dataset_name):
+        super(ARTrackSeq, self).__init__(params)
+        network = build_artrack_seq(params.cfg, training=False)
+        print(self.params.checkpoint)
+        network.load_state_dict(torch.load(self.params.checkpoint, map_location='cpu')['net'], strict=True)
+        self.cfg = params.cfg
+        self.bins = self.cfg.MODEL.BINS
+        self.network = network.cuda()
+        self.network.eval()
+        self.preprocessor = Preprocessor()
+        self.state = None
+
+        self.feat_sz = self.cfg.TEST.SEARCH_SIZE // self.cfg.MODEL.BACKBONE.STRIDE
+        # motion constrain
+        self.output_window = hann2d(torch.tensor([self.feat_sz, self.feat_sz]).long(), centered=True).cuda()
+
+        # for debug
+        self.debug = params.debug
+        self.use_visdom = params.debug
+        self.frame_id = 0
+        if self.debug:
+            if not self.use_visdom:
+                self.save_dir = "debug"
+                if not os.path.exists(self.save_dir):
+                    os.makedirs(self.save_dir)
+            else:
+                # self.add_hook()
+                self._init_visdom(None, 1)
+        # for save boxes from all queries
+        self.save_all_boxes = params.save_all_boxes
+        self.z_dict1 = {}
+        self.store_result = None
+        self.save_all = 7
+        self.x_feat = None
+        self.update = None
+        self.update_threshold = 5.0
+        self.update_intervals = 1
+
+    def initialize(self, image, info: dict):
+        # forward the template once
+        self.x_feat = None
+
+        z_patch_arr, resize_factor, z_amask_arr = sample_target(image, info['init_bbox'], self.params.template_factor,
+                                                                output_sz=self.params.template_size)  # output_sz=self.params.template_size
+        self.z_patch_arr = z_patch_arr
+        template = self.preprocessor.process(z_patch_arr, z_amask_arr)
+        with torch.no_grad():
+            self.z_dict1 = template
+
+        self.box_mask_z = None
+        # if self.cfg.MODEL.BACKBONE.CE_LOC:
+        #    template_bbox = self.transform_bbox_to_crop(info['init_bbox'], resize_factor,
+        #                                                template.tensors.device).squeeze(1)
+        #    self.box_mask_z = generate_mask_cond(self.cfg, 1, template.tensors.device, template_bbox)
+
+        # save states
+        self.state = info['init_bbox']
+        self.store_result = [info['init_bbox'].copy()]
+        for i in range(self.save_all - 1):
+            self.store_result.append(info['init_bbox'].copy())
+        self.frame_id = 0
+        self.update = None
+        if self.save_all_boxes:
+            '''save all predicted boxes'''
+            all_boxes_save = info['init_bbox'] * self.cfg.MODEL.NUM_OBJECT_QUERIES
+            return {"all_boxes": all_boxes_save}
+
+    def track(self, image, info: dict = None):
+        H, W, _ = image.shape
+        self.frame_id += 1
+        x_patch_arr, resize_factor, x_amask_arr = sample_target(image, self.state, self.params.search_factor,
+                                                                output_sz=self.params.search_size)  # (x1, y1, w, h)
+        for i in range(len(self.store_result)):
+            box_temp = self.store_result[i].copy()
+            box_out_i = transform_image_to_crop(torch.Tensor(self.store_result[i]), torch.Tensor(self.state),
+                                                resize_factor,
+                                                torch.Tensor([self.cfg.TEST.SEARCH_SIZE, self.cfg.TEST.SEARCH_SIZE]),
+                                                normalize=True)
+            box_out_i[2] = box_out_i[2] + box_out_i[0]
+            box_out_i[3] = box_out_i[3] + box_out_i[1]
+            box_out_i = box_out_i.clamp(min=-0.5, max=1.5)
+            box_out_i = (box_out_i + 0.5) * (self.bins - 1)
+            if i == 0:
+                seqs_out = box_out_i
+            else:
+                seqs_out = torch.cat((seqs_out, box_out_i), dim=-1)
+        seqs_out = seqs_out.unsqueeze(0)
+        search = self.preprocessor.process(x_patch_arr, x_amask_arr)
+        with torch.no_grad():
+            x_dict = search
+            # merge the template and the search
+            # run the transformer
+            out_dict = self.network.forward(
+                template=self.z_dict1.tensors, search=x_dict.tensors,
+                seq_input=seqs_out, stage="sequence", search_feature=self.x_feat, update=None)
+
+        self.x_feat = out_dict['x_feat']
+
+        pred_boxes = out_dict['seqs'][:, 0:4] / (self.bins - 1) - 0.5
+        pred_boxes = pred_boxes.view(-1, 4).mean(dim=0)
+        pred_new = pred_boxes
+        pred_new[2] = pred_boxes[2] - pred_boxes[0]
+        pred_new[3] = pred_boxes[3] - pred_boxes[1]
+        pred_new[0] = pred_boxes[0] + pred_new[2] / 2
+        pred_new[1] = pred_boxes[1] + pred_new[3] / 2
+        pred_boxes = (pred_new * self.params.search_size / resize_factor).tolist()
+
+        # Baseline: Take the mean of all pred boxes as the final result
+        # pred_box = (pred_boxes.mean(
+        #    dim=0) * self.params.search_size / resize_factor).tolist()  # (cx, cy, w, h) [0,1]
+        # get the final box result
+        self.state = clip_box(self.map_box_back(pred_boxes, resize_factor), H, W, margin=10)
+        if len(self.store_result) < self.save_all:
+            self.store_result.append(self.state.copy())
+        else:
+            for i in range(self.save_all):
+                if i != self.save_all - 1:
+                    self.store_result[i] = self.store_result[i + 1]
+                else:
+                    self.store_result[i] = self.state.copy()
+
+        # for debug
+        if self.debug:
+            if not self.use_visdom:
+                x1, y1, w, h = self.state
+                image_BGR = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+                cv2.rectangle(image_BGR, (int(x1), int(y1)), (int(x1 + w), int(y1 + h)), color=(0, 0, 255), thickness=2)
+                save_path = os.path.join(self.save_dir, "%04d.jpg" % self.frame_id)
+                cv2.imwrite(save_path, image_BGR)
+            else:
+                self.visdom.register((image, info['gt_bbox'].tolist(), self.state), 'Tracking', 1, 'Tracking')
+
+                self.visdom.register(torch.from_numpy(x_patch_arr).permute(2, 0, 1), 'image', 1, 'search_region')
+                self.visdom.register(torch.from_numpy(self.z_patch_arr).permute(2, 0, 1), 'image', 1, 'template')
+                self.visdom.register(pred_score_map.view(self.feat_sz, self.feat_sz), 'heatmap', 1, 'score_map')
+                self.visdom.register((pred_score_map * self.output_window).view(self.feat_sz, self.feat_sz), 'heatmap',
+                                     1, 'score_map_hann')
+
+                if 'removed_indexes_s' in out_dict and out_dict['removed_indexes_s']:
+                    removed_indexes_s = out_dict['removed_indexes_s']
+                    removed_indexes_s = [removed_indexes_s_i.cpu().numpy() for removed_indexes_s_i in removed_indexes_s]
+                    masked_search = gen_visualization(x_patch_arr, removed_indexes_s)
+                    self.visdom.register(torch.from_numpy(masked_search).permute(2, 0, 1), 'image', 1, 'masked_search')
+
+                while self.pause_mode:
+                    if self.step:
+                        self.step = False
+                        break
+
+        if self.save_all_boxes:
+            '''save all predictions'''
+            all_boxes = self.map_box_back_batch(pred_boxes * self.params.search_size / resize_factor, resize_factor)
+            all_boxes_save = all_boxes.view(-1).tolist()  # (4N, )
+            return {"target_bbox": self.state,
+                    "all_boxes": all_boxes_save}
+        else:
+            return {"target_bbox": self.state}
+
+    def map_box_back(self, pred_box: list, resize_factor: float):
+        cx_prev, cy_prev = self.state[0] + 0.5 * self.state[2], self.state[1] + 0.5 * self.state[3]
+        cx, cy, w, h = pred_box
+        half_side = 0.5 * self.params.search_size / resize_factor
+        cx_real = cx + (cx_prev - half_side)
+        cy_real = cy + (cy_prev - half_side)
+        # cx_real = cx + cx_prev
+        # cy_real = cy + cy_prev
+        return [cx_real - 0.5 * w, cy_real - 0.5 * h, w, h]
+
+    def map_box_back_batch(self, pred_box: torch.Tensor, resize_factor: float):
+        cx_prev, cy_prev = self.state[0] + 0.5 * self.state[2], self.state[1] + 0.5 * self.state[3]
+        cx, cy, w, h = pred_box.unbind(-1)  # (N,4) --> (N,)
+        half_side = 0.5 * self.params.search_size / resize_factor
+        cx_real = cx + (cx_prev - half_side)
+        cy_real = cy + (cy_prev - half_side)
+        return torch.stack([cx_real - 0.5 * w, cy_real - 0.5 * h, w, h], dim=-1)
+
+    def add_hook(self):
+        conv_features, enc_attn_weights, dec_attn_weights = [], [], []
+
+        for i in range(12):
+            self.network.backbone.blocks[i].attn.register_forward_hook(
+                # lambda self, input, output: enc_attn_weights.append(output[1])
+                lambda self, input, output: enc_attn_weights.append(output[1])
+            )
+
+        self.enc_attn_weights = enc_attn_weights
+
+
+def get_tracker_class():
+    return ARTrackSeq

lib/test/tracker/basetracker.py

@@ -0,0 +1,89 @@
+import time
+
+import torch
+from _collections import OrderedDict
+
+from lib.train.data.processing_utils import transform_image_to_crop
+from lib.vis.visdom_cus import Visdom
+
+
+class BaseTracker:
+    """Base class for all trackers."""
+
+    def __init__(self, params):
+        self.params = params
+        self.visdom = None
+
+    def predicts_segmentation_mask(self):
+        return False
+
+    def initialize(self, image, info: dict) -> dict:
+        """Overload this function in your tracker. This should initialize the model."""
+        raise NotImplementedError
+
+    def track(self, image, info: dict = None) -> dict:
+        """Overload this function in your tracker. This should track in the frame and update the model."""
+        raise NotImplementedError
+
+    def visdom_draw_tracking(self, image, box, segmentation=None):
+        if isinstance(box, OrderedDict):
+            box = [v for k, v in box.items()]
+        else:
+            box = (box,)
+        if segmentation is None:
+            self.visdom.register((image, *box), 'Tracking', 1, 'Tracking')
+        else:
+            self.visdom.register((image, *box, segmentation), 'Tracking', 1, 'Tracking')
+
+    def transform_bbox_to_crop(self, box_in, resize_factor, device, box_extract=None, crop_type='template'):
+        # box_in: list [x1, y1, w, h], not normalized
+        # box_extract: same as box_in
+        # out bbox: Torch.tensor [1, 1, 4], x1y1wh, normalized
+        if crop_type == 'template':
+            crop_sz = torch.Tensor([self.params.template_size, self.params.template_size])
+        elif crop_type == 'search':
+            crop_sz = torch.Tensor([self.params.search_size, self.params.search_size])
+        else:
+            raise NotImplementedError
+
+        box_in = torch.tensor(box_in)
+        if box_extract is None:
+            box_extract = box_in
+        else:
+            box_extract = torch.tensor(box_extract)
+        template_bbox = transform_image_to_crop(box_in, box_extract, resize_factor, crop_sz, normalize=True)
+        template_bbox = template_bbox.view(1, 1, 4).to(device)
+
+        return template_bbox
+
+    def _init_visdom(self, visdom_info, debug):
+        visdom_info = {} if visdom_info is None else visdom_info
+        self.pause_mode = False
+        self.step = False
+        self.next_seq = False
+        if debug > 0 and visdom_info.get('use_visdom', True):
+            try:
+                self.visdom = Visdom(debug, {'handler': self._visdom_ui_handler, 'win_id': 'Tracking'},
+                                     visdom_info=visdom_info)
+
+                # # Show help
+                # help_text = 'You can pause/unpause the tracker by pressing ''space'' with the ''Tracking'' window ' \
+                #             'selected. During paused mode, you can track for one frame by pressing the right arrow key.' \
+                #             'To enable/disable plotting of a data block, tick/untick the corresponding entry in ' \
+                #             'block list.'
+                # self.visdom.register(help_text, 'text', 1, 'Help')
+            except:
+                time.sleep(0.5)
+                print('!!! WARNING: Visdom could not start, so using matplotlib visualization instead !!!\n'
+                      '!!! Start Visdom in a separate terminal window by typing \'visdom\' !!!')
+
+    def _visdom_ui_handler(self, data):
+        if data['event_type'] == 'KeyPress':
+            if data['key'] == ' ':
+                self.pause_mode = not self.pause_mode
+
+            elif data['key'] == 'ArrowRight' and self.pause_mode:
+                self.step = True
+
+            elif data['key'] == 'n':
+                self.next_seq = True

lib/test/tracker/data_utils.py

@@ -0,0 +1,46 @@
+import torch
+import numpy as np
+from lib.utils.misc import NestedTensor
+
+
+class Preprocessor(object):
+    def __init__(self):
+        self.mean = torch.tensor([0.485, 0.456, 0.406]).view((1, 3, 1, 1)).cuda()
+        self.std = torch.tensor([0.229, 0.224, 0.225]).view((1, 3, 1, 1)).cuda()
+
+    def process(self, img_arr: np.ndarray, amask_arr: np.ndarray):
+        # Deal with the image patch
+        img_tensor = torch.tensor(img_arr).cuda().float().permute((2,0,1)).unsqueeze(dim=0)
+        img_tensor_norm = ((img_tensor / 255.0) - self.mean) / self.std  # (1,3,H,W)
+        # Deal with the attention mask
+        amask_tensor = torch.from_numpy(amask_arr).to(torch.bool).cuda().unsqueeze(dim=0)  # (1,H,W)
+        return NestedTensor(img_tensor_norm, amask_tensor)
+
+
+class PreprocessorX(object):
+    def __init__(self):
+        self.mean = torch.tensor([0.485, 0.456, 0.406]).view((1, 3, 1, 1)).cuda()
+        self.std = torch.tensor([0.229, 0.224, 0.225]).view((1, 3, 1, 1)).cuda()
+
+    def process(self, img_arr: np.ndarray, amask_arr: np.ndarray):
+        # Deal with the image patch
+        img_tensor = torch.tensor(img_arr).cuda().float().permute((2,0,1)).unsqueeze(dim=0)
+        img_tensor_norm = ((img_tensor / 255.0) - self.mean) / self.std  # (1,3,H,W)
+        # Deal with the attention mask
+        amask_tensor = torch.from_numpy(amask_arr).to(torch.bool).cuda().unsqueeze(dim=0)  # (1,H,W)
+        return img_tensor_norm, amask_tensor
+
+
+class PreprocessorX_onnx(object):
+    def __init__(self):
+        self.mean = np.array([0.485, 0.456, 0.406]).reshape((1, 3, 1, 1))
+        self.std = np.array([0.229, 0.224, 0.225]).reshape((1, 3, 1, 1))
+
+    def process(self, img_arr: np.ndarray, amask_arr: np.ndarray):
+        """img_arr: (H,W,3), amask_arr: (H,W)"""
+        # Deal with the image patch
+        img_arr_4d = img_arr[np.newaxis, :, :, :].transpose(0, 3, 1, 2)
+        img_arr_4d = (img_arr_4d / 255.0 - self.mean) / self.std  # (1, 3, H, W)
+        # Deal with the attention mask
+        amask_arr_3d = amask_arr[np.newaxis, :, :]  # (1,H,W)
+        return img_arr_4d.astype(np.float32), amask_arr_3d.astype(np.bool)

lib/test/tracker/vis_utils.py

@@ -0,0 +1,59 @@
+import numpy as np
+
+
+############## used for visulize eliminated tokens #################
+def get_keep_indices(decisions):
+    keep_indices = []
+    for i in range(3):
+        if i == 0:
+            keep_indices.append(decisions[i])
+        else:
+            keep_indices.append(keep_indices[-1][decisions[i]])
+    return keep_indices
+
+
+def gen_masked_tokens(tokens, indices, alpha=0.2):
+    # indices = [i for i in range(196) if i not in indices]
+    indices = indices[0].astype(int)
+    tokens = tokens.copy()
+    tokens[indices] = alpha * tokens[indices] + (1 - alpha) * 255
+    return tokens
+
+
+def recover_image(tokens, H, W, Hp, Wp, patch_size):
+    # image: (C, 196, 16, 16)
+    image = tokens.reshape(Hp, Wp, patch_size, patch_size, 3).swapaxes(1, 2).reshape(H, W, 3)
+    return image
+
+
+def pad_img(img):
+    height, width, channels = img.shape
+    im_bg = np.ones((height, width + 8, channels)) * 255
+    im_bg[0:height, 0:width, :] = img
+    return im_bg
+
+
+def gen_visualization(image, mask_indices, patch_size=16):
+    # image [224, 224, 3]
+    # mask_indices, list of masked token indices
+
+    # mask mask_indices need to cat
+    # mask_indices = mask_indices[::-1]
+    num_stages = len(mask_indices)
+    for i in range(1, num_stages):
+        mask_indices[i] = np.concatenate([mask_indices[i-1], mask_indices[i]], axis=1)
+
+    # keep_indices = get_keep_indices(decisions)
+    image = np.asarray(image)
+    H, W, C = image.shape
+    Hp, Wp = H // patch_size, W // patch_size
+    image_tokens = image.reshape(Hp, patch_size, Wp, patch_size, 3).swapaxes(1, 2).reshape(Hp * Wp, patch_size, patch_size, 3)
+
+    stages = [
+        recover_image(gen_masked_tokens(image_tokens, mask_indices[i]), H, W, Hp, Wp, patch_size)
+        for i in range(num_stages)
+    ]
+    imgs = [image] + stages
+    imgs = [pad_img(img) for img in imgs]
+    viz = np.concatenate(imgs, axis=1)
+    return viz

lib/test/utils/__init__.py

@@ -0,0 +1 @@
+from .params import TrackerParams, FeatureParams, Choice

lib/test/utils/_init_paths.py

@@ -0,0 +1,17 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os.path as osp
+import sys
+
+
+def add_path(path):
+    if path not in sys.path:
+        sys.path.insert(0, path)
+
+
+this_dir = osp.dirname(__file__)
+
+prj_path = osp.join(this_dir, '..', '..', '..')
+add_path(prj_path)

lib/test/utils/hann.py

@@ -0,0 +1,93 @@
+import torch
+import math
+import torch.nn.functional as F
+
+
+def hann1d(sz: int, centered = True) -> torch.Tensor:
+    """1D cosine window."""
+    if centered:
+        return 0.5 * (1 - torch.cos((2 * math.pi / (sz + 1)) * torch.arange(1, sz + 1).float()))
+    w = 0.5 * (1 + torch.cos((2 * math.pi / (sz + 2)) * torch.arange(0, sz//2 + 1).float()))
+    return torch.cat([w, w[1:sz-sz//2].flip((0,))])
+
+
+def hann2d(sz: torch.Tensor, centered = True) -> torch.Tensor:
+    """2D cosine window."""
+    return hann1d(sz[0].item(), centered).reshape(1, 1, -1, 1) * hann1d(sz[1].item(), centered).reshape(1, 1, 1, -1)
+
+
+def hann2d_bias(sz: torch.Tensor, ctr_point: torch.Tensor, centered = True) -> torch.Tensor:
+    """2D cosine window."""
+    distance = torch.stack([ctr_point, sz-ctr_point], dim=0)
+    max_distance, _ = distance.max(dim=0)
+
+    hann1d_x = hann1d(max_distance[0].item() * 2, centered)
+    hann1d_x = hann1d_x[max_distance[0] - distance[0, 0]: max_distance[0] + distance[1, 0]]
+    hann1d_y = hann1d(max_distance[1].item() * 2, centered)
+    hann1d_y = hann1d_y[max_distance[1] - distance[0, 1]: max_distance[1] + distance[1, 1]]
+
+    return hann1d_y.reshape(1, 1, -1, 1) * hann1d_x.reshape(1, 1, 1, -1)
+
+
+
+def hann2d_clipped(sz: torch.Tensor, effective_sz: torch.Tensor, centered = True) -> torch.Tensor:
+    """1D clipped cosine window."""
+
+    # Ensure that the difference is even
+    effective_sz += (effective_sz - sz) % 2
+    effective_window = hann1d(effective_sz[0].item(), True).reshape(1, 1, -1, 1) * hann1d(effective_sz[1].item(), True).reshape(1, 1, 1, -1)
+
+    pad = (sz - effective_sz) // 2
+
+    window = F.pad(effective_window, (pad[1].item(), pad[1].item(), pad[0].item(), pad[0].item()), 'replicate')
+
+    if centered:
+        return window
+    else:
+        mid = (sz / 2).int()
+        window_shift_lr = torch.cat((window[:, :, :, mid[1]:], window[:, :, :, :mid[1]]), 3)
+        return torch.cat((window_shift_lr[:, :, mid[0]:, :], window_shift_lr[:, :, :mid[0], :]), 2)
+
+
+def gauss_fourier(sz: int, sigma: float, half: bool = False) -> torch.Tensor:
+    if half:
+        k = torch.arange(0, int(sz/2+1))
+    else:
+        k = torch.arange(-int((sz-1)/2), int(sz/2+1))
+    return (math.sqrt(2*math.pi) * sigma / sz) * torch.exp(-2 * (math.pi * sigma * k.float() / sz)**2)
+
+
+def gauss_spatial(sz, sigma, center=0, end_pad=0):
+    k = torch.arange(-(sz-1)/2, (sz+1)/2+end_pad)
+    return torch.exp(-1.0/(2*sigma**2) * (k - center)**2)
+
+
+def label_function(sz: torch.Tensor, sigma: torch.Tensor):
+    return gauss_fourier(sz[0].item(), sigma[0].item()).reshape(1, 1, -1, 1) * gauss_fourier(sz[1].item(), sigma[1].item(), True).reshape(1, 1, 1, -1)
+
+def label_function_spatial(sz: torch.Tensor, sigma: torch.Tensor, center: torch.Tensor = torch.zeros(2), end_pad: torch.Tensor = torch.zeros(2)):
+    """The origin is in the middle of the image."""
+    return gauss_spatial(sz[0].item(), sigma[0].item(), center[0], end_pad[0].item()).reshape(1, 1, -1, 1) * \
+           gauss_spatial(sz[1].item(), sigma[1].item(), center[1], end_pad[1].item()).reshape(1, 1, 1, -1)
+
+
+def cubic_spline_fourier(f, a):
+    """The continuous Fourier transform of a cubic spline kernel."""
+
+    bf = (6*(1 - torch.cos(2 * math.pi * f)) + 3*a*(1 - torch.cos(4 * math.pi * f))
+           - (6 + 8*a)*math.pi*f*torch.sin(2 * math.pi * f) - 2*a*math.pi*f*torch.sin(4 * math.pi * f)) \
+         / (4 * math.pi**4 * f**4)
+
+    bf[f == 0] = 1
+
+    return bf
+
+def max2d(a: torch.Tensor) -> (torch.Tensor, torch.Tensor):
+    """Computes maximum and argmax in the last two dimensions."""
+
+    max_val_row, argmax_row = torch.max(a, dim=-2)
+    max_val, argmax_col = torch.max(max_val_row, dim=-1)
+    argmax_row = argmax_row.view(argmax_col.numel(),-1)[torch.arange(argmax_col.numel()), argmax_col.view(-1)]
+    argmax_row = argmax_row.reshape(argmax_col.shape)
+    argmax = torch.cat((argmax_row.unsqueeze(-1), argmax_col.unsqueeze(-1)), -1)
+    return max_val, argmax

lib/test/utils/load_text.py

@@ -0,0 +1,47 @@
+import numpy as np
+import pandas as pd
+
+
+def load_text_numpy(path, delimiter, dtype):
+    if isinstance(delimiter, (tuple, list)):
+        for d in delimiter:
+            try:
+                ground_truth_rect = np.loadtxt(path, delimiter=d, dtype=dtype)
+                return ground_truth_rect
+            except:
+                pass
+
+        raise Exception('Could not read file {}'.format(path))
+    else:
+        ground_truth_rect = np.loadtxt(path, delimiter=delimiter, dtype=dtype)
+        return ground_truth_rect
+
+
+def load_text_pandas(path, delimiter, dtype):
+    if isinstance(delimiter, (tuple, list)):
+        for d in delimiter:
+            try:
+                ground_truth_rect = pd.read_csv(path, delimiter=d, header=None, dtype=dtype, na_filter=False,
+                                                low_memory=False).values
+                return ground_truth_rect
+            except Exception as e:
+                pass
+
+        raise Exception('Could not read file {}'.format(path))
+    else:
+        ground_truth_rect = pd.read_csv(path, delimiter=delimiter, header=None, dtype=dtype, na_filter=False,
+                                        low_memory=False).values
+        return ground_truth_rect
+
+
+def load_text(path, delimiter=' ', dtype=np.float32, backend='numpy'):
+    if backend == 'numpy':
+        return load_text_numpy(path, delimiter, dtype)
+    elif backend == 'pandas':
+        return load_text_pandas(path, delimiter, dtype)
+
+
+def load_str(path):
+    with open(path, "r") as f:
+        text_str = f.readline().strip().lower()
+    return text_str

lib/test/utils/params.py

@@ -0,0 +1,43 @@
+from lib.utils import TensorList
+import random
+
+
+class TrackerParams:
+    """Class for tracker parameters."""
+    def set_default_values(self, default_vals: dict):
+        for name, val in default_vals.items():
+            if not hasattr(self, name):
+                setattr(self, name, val)
+
+    def get(self, name: str, *default):
+        """Get a parameter value with the given name. If it does not exists, it return the default value given as a
+        second argument or returns an error if no default value is given."""
+        if len(default) > 1:
+            raise ValueError('Can only give one default value.')
+
+        if not default:
+            return getattr(self, name)
+
+        return getattr(self, name, default[0])
+
+    def has(self, name: str):
+        """Check if there exist a parameter with the given name."""
+        return hasattr(self, name)
+
+
+class FeatureParams:
+    """Class for feature specific parameters"""
+    def __init__(self, *args, **kwargs):
+        if len(args) > 0:
+            raise ValueError
+
+        for name, val in kwargs.items():
+            if isinstance(val, list):
+                setattr(self, name, TensorList(val))
+            else:
+                setattr(self, name, val)
+
+
+def Choice(*args):
+    """Can be used to sample random parameter values."""
+    return random.choice(args)

lib/test/utils/transform_got10k.py

@@ -0,0 +1,52 @@
+import numpy as np
+import os
+import shutil
+import argparse
+import _init_paths
+from lib.test.evaluation.environment import env_settings
+
+
+def transform_got10k(tracker_name, cfg_name):
+    env = env_settings()
+    result_dir = env.results_path
+    src_dir = os.path.join(result_dir, "%s/%s/got10k/" % (tracker_name, cfg_name))
+    dest_dir = os.path.join(result_dir, "%s/%s/got10k_submit/" % (tracker_name, cfg_name))
+    if not os.path.exists(dest_dir):
+        os.makedirs(dest_dir)
+    items = os.listdir(src_dir)
+    for item in items:
+        if "all" in item:
+            continue
+        src_path = os.path.join(src_dir, item)
+        if "time" not in item:
+            seq_name = item.replace(".txt", '')
+            seq_dir = os.path.join(dest_dir, seq_name)
+            if not os.path.exists(seq_dir):
+                os.makedirs(seq_dir)
+            new_item = item.replace(".txt", '_001.txt')
+            dest_path = os.path.join(seq_dir, new_item)
+            bbox_arr = np.loadtxt(src_path, dtype=np.int, delimiter='\t')
+            np.savetxt(dest_path, bbox_arr, fmt='%d', delimiter=',')
+        else:
+            seq_name = item.replace("_time.txt", '')
+            seq_dir = os.path.join(dest_dir, seq_name)
+            if not os.path.exists(seq_dir):
+                os.makedirs(seq_dir)
+            dest_path = os.path.join(seq_dir, item)
+            os.system("cp %s %s" % (src_path, dest_path))
+    # make zip archive
+    shutil.make_archive(src_dir, "zip", src_dir)
+    shutil.make_archive(dest_dir, "zip", dest_dir)
+    # Remove the original files
+    shutil.rmtree(src_dir)
+    shutil.rmtree(dest_dir)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='transform got10k results.')
+    parser.add_argument('--tracker_name', type=str, help='Name of tracking method.')
+    parser.add_argument('--cfg_name', type=str, help='Name of config file.')
+
+    args = parser.parse_args()
+    transform_got10k(args.tracker_name, args.cfg_name)
+

lib/test/utils/transform_trackingnet.py

@@ -0,0 +1,39 @@
+import numpy as np
+import os
+import shutil
+import argparse
+import _init_paths
+from lib.test.evaluation.environment import env_settings
+
+
+def transform_trackingnet(tracker_name, cfg_name):
+    env = env_settings()
+    result_dir = env.results_path
+    src_dir = os.path.join(result_dir, "%s/%s/trackingnet/" % (tracker_name, cfg_name))
+    dest_dir = os.path.join(result_dir, "%s/%s/trackingnet_submit/" % (tracker_name, cfg_name))
+    if not os.path.exists(dest_dir):
+        os.makedirs(dest_dir)
+    items = os.listdir(src_dir)
+    for item in items:
+        if "all" in item:
+            continue
+        if "time" not in item:
+            src_path = os.path.join(src_dir, item)
+            dest_path = os.path.join(dest_dir, item)
+            bbox_arr = np.loadtxt(src_path, dtype=np.int, delimiter='\t')
+            np.savetxt(dest_path, bbox_arr, fmt='%d', delimiter=',')
+    # make zip archive
+    shutil.make_archive(src_dir, "zip", src_dir)
+    shutil.make_archive(dest_dir, "zip", dest_dir)
+    # Remove the original files
+    shutil.rmtree(src_dir)
+    shutil.rmtree(dest_dir)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='transform trackingnet results.')
+    parser.add_argument('--tracker_name', type=str, help='Name of tracking method.')
+    parser.add_argument('--cfg_name', type=str, help='Name of config file.')
+
+    args = parser.parse_args()
+    transform_trackingnet(args.tracker_name, args.cfg_name)