From ccdb20acf06ffc58f3ab3bdebbb907fe3885f73b Mon Sep 17 00:00:00 2001 From: huanglianghua Date: Fri, 30 Nov 2018 19:42:35 +0800 Subject: [PATCH] implementation without docs and tutorials --- .gitignore | 10 + LICENSE | 21 ++ README.md | 0 examples/quick_examples.ipynb | 0 examples/quick_examples.py | 0 got10k/__init__.py | 0 got10k/datasets/__init__.py | 6 + got10k/datasets/got10k.py | 108 ++++++ got10k/datasets/otb.py | 197 ++++++++++ got10k/datasets/vid.py | 153 ++++++++ got10k/datasets/vot.py | 242 ++++++++++++ got10k/experiments/__init__.py | 5 + got10k/experiments/got10k.py | 286 ++++++++++++++ got10k/experiments/otb.py | 241 ++++++++++++ got10k/experiments/vot.py | 558 ++++++++++++++++++++++++++++ got10k/trackers/__init__.py | 44 +++ got10k/trackers/identity_tracker.py | 17 + got10k/utils/__init__.py | 0 got10k/utils/ioutils.py | 50 +++ got10k/utils/metrics.py | 137 +++++++ got10k/utils/viz.py | 73 ++++ requirements.txt | 9 + setup.cfg | 2 + setup.py | 17 + tests/test_datasets.py | 56 +++ tests/test_experiments.py | 46 +++ tests/test_trackers.py | 33 ++ tests/test_utils.py | 39 ++ 28 files changed, 2350 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 examples/quick_examples.ipynb create mode 100644 examples/quick_examples.py create mode 100644 got10k/__init__.py create mode 100644 got10k/datasets/__init__.py create mode 100644 got10k/datasets/got10k.py create mode 100644 got10k/datasets/otb.py create mode 100644 got10k/datasets/vid.py create mode 100644 got10k/datasets/vot.py create mode 100644 got10k/experiments/__init__.py create mode 100644 got10k/experiments/got10k.py create mode 100644 got10k/experiments/otb.py create mode 100644 got10k/experiments/vot.py create mode 100644 got10k/trackers/__init__.py create mode 100644 got10k/trackers/identity_tracker.py create mode 100644 got10k/utils/__init__.py create mode 100644 got10k/utils/ioutils.py create mode 100644 got10k/utils/metrics.py create mode 100644 got10k/utils/viz.py create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 tests/test_datasets.py create mode 100644 tests/test_experiments.py create mode 100644 tests/test_trackers.py create mode 100644 tests/test_utils.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..71acd1e --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.* +*.pyc +__pycache__/ +data +data/ +cache/ +results/ +reports/ +venv/ +!.gitignore diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ab60297 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/examples/quick_examples.ipynb b/examples/quick_examples.ipynb new file mode 100644 index 0000000..e69de29 diff --git a/examples/quick_examples.py b/examples/quick_examples.py new file mode 100644 index 0000000..e69de29 diff --git a/got10k/__init__.py b/got10k/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/got10k/datasets/__init__.py b/got10k/datasets/__init__.py new file mode 100644 index 0000000..50ed3b8 --- /dev/null +++ b/got10k/datasets/__init__.py @@ -0,0 +1,6 @@ +from __future__ import absolute_import + +from .got10k import GOT10k +from .otb import OTB +from .vot import VOT +from .vid import ImageNetVID diff --git a/got10k/datasets/got10k.py b/got10k/datasets/got10k.py new file mode 100644 index 0000000..f0aed33 --- /dev/null +++ b/got10k/datasets/got10k.py @@ -0,0 +1,108 @@ +from __future__ import absolute_import, print_function + +import os +import glob +import numpy as np +import six + + +class GOT10k(object): + r"""`GOT-10K `_ Dataset. + + Publication: + ``GOT-10k: A Large High-Diversity Benchmark for Generic Object + Tracking in the Wild``, L. Huang, X. Zhao and K. Huang, ArXiv 2018. + + Args: + root_dir (string): Root directory of dataset where ``train``, + ``val`` and ``test`` folders exist. + subset (string, optional): Specify ``train``, ``val`` or ``test`` + subset of GOT-10k. + return_meta (string, optional): If True, returns ``meta`` + of each sequence in ``__getitem__`` function, otherwise + only returns ``img_files`` and ``anno``. + """ + + def __init__(self, root_dir, subset='test', return_meta=False): + super(GOT10k, self).__init__() + assert subset in ['train', 'val', 'test'], 'Unknown subset.' + + self.root_dir = root_dir + self.subset = subset + self.return_meta = False if subset == 'test' else return_meta + self._check_integrity(root_dir, subset) + + list_file = os.path.join(root_dir, subset, 'list.txt') + with open(list_file, 'r') as f: + self.seq_names = f.read().strip().split('\n') + self.seq_dirs = [os.path.join(root_dir, subset, s) + for s in self.seq_names] + self.anno_files = [os.path.join(d, 'groundtruth.txt') + for d in self.seq_dirs] + + def __getitem__(self, index): + r""" + Args: + index (integer or string): Index or name of a sequence. + + Returns: + tuple: (img_files, anno) if ``return_meta`` is False, otherwise + (img_files, anno, meta), where ``img_files`` is a list of + file names, ``anno`` is a N x 4 (rectangles) numpy array, while + ``meta`` is a dict contains meta information about the sequence. + """ + if isinstance(index, six.string_types): + if not index in self.seq_names: + raise Exception('Sequence {} not found.'.format(index)) + index = self.seq_names.index(index) + + img_files = sorted(glob.glob(os.path.join( + self.seq_dirs[index], '*.jpg'))) + anno = np.loadtxt(self.anno_files[index], delimiter=',') + + if self.subset == 'test': + assert anno.ndim == 1 + anno = anno[np.newaxis, :] + else: + assert len(img_files) == len(anno) + + if self.return_meta: + meta = self._fetch_meta(self.seq_dirs[index]) + return img_files, anno, meta + else: + return img_files, anno + + def __len__(self): + return len(self.seq_names) + + def _check_integrity(self, root_dir, subset): + assert subset in ['train', 'val', 'test'] + list_file = os.path.join(root_dir, subset, 'list.txt') + + if os.path.isfile(list_file): + with open(list_file, 'r') as f: + seq_names = f.read().strip().split('\n') + + # check each sequence folder + for seq_name in seq_names: + seq_dir = os.path.join(root_dir, subset, seq_name) + if not os.path.isdir(seq_dir): + print('Warning: sequence %s not exist.' % seq_name) + else: + # dataset not exist + raise Exception('Dataset not found or corrupted.') + + def _fetch_meta(self, seq_dir): + # meta information + meta_file = os.path.join(seq_dir, 'meta_info.ini') + with open(meta_file) as f: + meta = f.read().strip().split('\n')[1:] + meta = [line.split(': ') for line in meta] + meta = {line[0]: line[1] for line in meta} + + # attributes + attributes = ['cover', 'absence', 'cut_by_image'] + for att in attributes: + meta[att] = np.loadtxt(os.path.join(seq_dir, att + '.label')) + + return meta diff --git a/got10k/datasets/otb.py b/got10k/datasets/otb.py new file mode 100644 index 0000000..ef02d7e --- /dev/null +++ b/got10k/datasets/otb.py @@ -0,0 +1,197 @@ +from __future__ import absolute_import, print_function + +import os +import glob +import numpy as np +import io +import six +from itertools import chain + +from ..utils.ioutils import download, extract + + +class OTB(object): + r"""`OTB `_ Datasets. + + Publication: + ``Object Tracking Benchmark``, Y. Wu, J. Lim and M.-H. Yang, IEEE TPAMI 2015. + + Args: + root_dir (string): Root directory of dataset where sequence + folders exist. + version (integer or string): Specify the benchmark version, specify as one of + ``2013``, ``2015``, ``tb50`` and ``tb100``. + download (boolean, optional): If True, downloads the dataset from the internet + and puts it in root directory. If dataset is downloaded, it is not + downloaded again. + """ + + __otb13_seqs = ['Basketball', 'Bolt', 'Boy', 'Car4', 'CarDark', + 'CarScale', 'Coke', 'Couple', 'Crossing', 'David', + 'David2', 'David3', 'Deer', 'Dog1', 'Doll', 'Dudek', + 'FaceOcc1', 'FaceOcc2', 'Fish', 'FleetFace', + 'Football', 'Football1', 'Freeman1', 'Freeman3', + 'Freeman4', 'Girl', 'Ironman', 'Jogging', 'Jumping', + 'Lemming', 'Liquor', 'Matrix', 'Mhyang', 'MotorRolling', + 'MountainBike', 'Shaking', 'Singer1', 'Singer2', + 'Skating1', 'Skiing', 'Soccer', 'Subway', 'Suv', + 'Sylvester', 'Tiger1', 'Tiger2', 'Trellis', 'Walking', + 'Walking2', 'Woman'] + + __tb50_seqs = ['Basketball', 'Biker', 'Bird1', 'BlurBody', 'BlurCar2', + 'BlurFace', 'BlurOwl', 'Bolt', 'Box', 'Car1', 'Car4', + 'CarDark', 'CarScale', 'ClifBar', 'Couple', 'Crowds', + 'David', 'Deer', 'Diving', 'DragonBaby', 'Dudek', + 'Football', 'Freeman4', 'Girl', 'Human3', 'Human4', + 'Human6', 'Human9', 'Ironman', 'Jump', 'Jumping', + 'Liquor', 'Matrix', 'MotorRolling', 'Panda', 'RedTeam', + 'Shaking', 'Singer2', 'Skating1', 'Skating2', 'Skiing', + 'Soccer', 'Surfer', 'Sylvester', 'Tiger2', 'Trellis', + 'Walking', 'Walking2', 'Woman'] + + __tb100_seqs = ['Bird2', 'BlurCar1', 'BlurCar3', 'BlurCar4', 'Board', + 'Bolt2', 'Boy', 'Car2', 'Car24', 'Coke', 'Coupon', + 'Crossing', 'Dancer', 'Dancer2', 'David2', 'David3', + 'Dog', 'Dog1', 'Doll', 'FaceOcc1', 'FaceOcc2', 'Fish', + 'FleetFace', 'Football1', 'Freeman1', 'Freeman3', + 'Girl2', 'Gym', 'Human2', 'Human5', 'Human7', 'Human8', + 'Jogging', 'KiteSurf', 'Lemming', 'Man', 'Mhyang', + 'MountainBike', 'Rubik', 'Singer1', 'Skater', + 'Skater2', 'Subway', 'Suv', 'Tiger1', 'Toy', 'Trans', + 'Twinnings', 'Vase'] + __tb50_seqs + + __otb15_seqs = __tb100_seqs + + __version_dict = { + 2013: __otb13_seqs, + 2015: __otb15_seqs, + 'otb2013': __otb13_seqs, + 'otb2015': __otb15_seqs, + 'tb50': __tb50_seqs, + 'tb100': __tb100_seqs} + + def __init__(self, root_dir, version=2015, download=True): + super(OTB, self).__init__() + assert version in self.__version_dict + + self.root_dir = root_dir + self.version = version + if download: + self._download(root_dir, version) + self._check_integrity(root_dir, version) + + valid_seqs = self.__version_dict[version] + self.anno_files = sorted(list(chain.from_iterable(glob.glob( + os.path.join(root_dir, s, 'groundtruth*.txt')) for s in valid_seqs))) + # remove empty annotation files + # (e.g., groundtruth_rect.1.txt of Human4) + self.anno_files = self._filter_files(self.anno_files) + self.seq_dirs = [os.path.dirname(f) for f in self.anno_files] + self.seq_names = [os.path.basename(d) for d in self.seq_dirs] + # rename repeated sequence names + # (e.g., Jogging and Skating2) + self.seq_names = self._rename_seqs(self.seq_names) + + def __getitem__(self, index): + r""" + Args: + index (integer or string): Index or name of a sequence. + + Returns: + tuple: (img_files, anno), where ``img_files`` is a list of + file names and ``anno`` is a N x 4 (rectangles) numpy array. + """ + if isinstance(index, six.string_types): + if not index in self.seq_names: + raise Exception('Sequence {} not found.'.format(index)) + index = self.seq_names.index(index) + + img_files = sorted(glob.glob( + os.path.join(self.seq_dirs[index], 'img/*.jpg'))) + + # special sequences + # (visit http://cvlab.hanyang.ac.kr/tracker_benchmark/index.html for detail) + seq_name = self.seq_names[index] + if seq_name.lower() == 'david': + img_files = img_files[300-1:770] + elif seq_name.lower() == 'football1': + img_files = img_files[:74] + elif seq_name.lower() == 'freeman3': + img_files = img_files[:460] + elif seq_name.lower() == 'freeman4': + img_files = img_files[:283] + elif seq_name.lower() == 'diving': + img_files = img_files[:215] + + # to deal with different delimeters + with open(self.anno_files[index], 'r') as f: + anno = np.loadtxt(io.StringIO(f.read().replace(',', ' '))) + assert len(img_files) == len(anno) + assert anno.shape[1] == 4 + + return img_files, anno + + def __len__(self): + return len(self.seq_names) + + def _filter_files(self, filenames): + filtered_files = [] + for filename in filenames: + with open(filename, 'r') as f: + if f.read().strip() == '': + print('Warning: %s is empty.' % filename) + else: + filtered_files.append(filename) + + return filtered_files + + def _rename_seqs(self, seq_names): + # in case some sequences may have multiple targets + renamed_seqs = [] + for i, seq_name in enumerate(seq_names): + if seq_names.count(seq_name) == 1: + renamed_seqs.append(seq_name) + else: + ind = seq_names[:i + 1].count(seq_name) + renamed_seqs.append('%s.%d' % (seq_name, ind)) + + return renamed_seqs + + def _download(self, root_dir, version): + assert version in self.__version_dict + seq_names = self.__version_dict[version] + + if not os.path.isdir(root_dir): + os.makedirs(root_dir) + elif all([os.path.isdir(os.path.join(root_dir, s)) for s in seq_names]): + print('Files already downloaded.') + return + + url_fmt = 'http://cvlab.hanyang.ac.kr/tracker_benchmark/seq/%s.zip' + for seq_name in seq_names: + seq_dir = os.path.join(root_dir, seq_name) + if os.path.isdir(seq_dir): + continue + url = url_fmt % seq_name + zip_file = os.path.join(root_dir, seq_name + '.zip') + print('Downloading to %s...' % zip_file) + download(url, zip_file) + print('\nExtracting to %s...' % root_dir) + extract(zip_file, root_dir) + + return root_dir + + def _check_integrity(self, root_dir, version): + assert version in self.__version_dict + seq_names = self.__version_dict[version] + + if os.path.isdir(root_dir) and len(os.listdir(root_dir)) > 0: + # check each sequence folder + for seq_name in seq_names: + seq_dir = os.path.join(root_dir, seq_name) + if not os.path.isdir(seq_dir): + print('Warning: sequence %s not exist.' % seq_name) + else: + # dataset not exist + raise Exception('Dataset not found or corrupted. ' + + 'You can use download=True to download it.') diff --git a/got10k/datasets/vid.py b/got10k/datasets/vid.py new file mode 100644 index 0000000..ba104a7 --- /dev/null +++ b/got10k/datasets/vid.py @@ -0,0 +1,153 @@ +from __future__ import absolute_import, print_function + +import os +import glob +import six +import numpy as np +import xml.etree.ElementTree as ET +import json +from collections import OrderedDict + + +class ImageNetVID(object): + r"""`ImageNet Video Image Detection (VID) `_ Dataset. + + Publication: + ``ImageNet Large Scale Visual Recognition Challenge``, O. Russakovsky, + J. deng, H. Su, etc. IJCV, 2015. + + Args: + root_dir (string): Root directory of dataset where ``Data``, and + ``Annotation`` folders exist. + subset (string, optional): Specify ``train``, ``val`` or (``train``, ``val``) + subset(s) of ImageNet-VID. Default is a tuple (``train``, ``val``). + cache_dir (string, optional): Directory for caching the paths and annotations + for speeding up loading. Default is ``cache/imagenet_vid``. + """ + + def __init__(self, root_dir, subset=('train', 'val'), + cache_dir='cache/imagenet_vid'): + self.root_dir = root_dir + self.cache_dir = cache_dir + if isinstance(subset, str): + assert subset in ['train', 'val'] + self.subset = [subset] + elif isinstance(subset, (list, tuple)): + assert all([s in ['train', 'val'] for s in subset]) + self.subset = subset + else: + raise Exception('Unknown subset') + + # cache filenames and annotations to speed up training + self.seq_dict = self._cache_meta() + self.seq_names = [n for n in self.seq_dict] + + def __getitem__(self, index): + r""" + Args: + index (integer or string): Index or name of a sequence. + + Returns: + tuple: (img_files, anno), where ``img_files`` is a list of + file names and ``anno`` is a N x 4 (rectangles) numpy array. + """ + if isinstance(index, six.string_types): + seq_name = index + else: + seq_name = self.seq_names[index] + + seq_dir, frames, anno_file = self.seq_dict[seq_name] + img_files = [os.path.join( + seq_dir, '%06d.JPEG' % f) for f in frames] + anno = np.loadtxt(anno_file, delimiter=',') + + return img_files, anno + + def __len__(self): + return len(self.seq_dict) + + def _cache_meta(self): + cache_file = os.path.join(self.cache_dir, 'seq_dict.json') + if os.path.isfile(cache_file): + print('Dataset already cached.') + with open(cache_file) as f: + seq_dict = json.load(f) + return seq_dict + + # image and annotation paths + print('Gather sequence paths...') + seq_dirs = [] + anno_dirs = [] + if 'train' in self.subset: + seq_dirs_ = sorted(glob.glob(os.path.join( + self.root_dir, 'Data/VID/train/ILSVRC*/ILSVRC*'))) + anno_dirs_ = [os.path.join( + self.root_dir, 'Annotations/VID/train', + *s.split('/')[-2:]) for s in seq_dirs_] + seq_dirs += seq_dirs_ + anno_dirs += anno_dirs_ + if 'val' in self.subset: + seq_dirs_ = sorted(glob.glob(os.path.join( + self.root_dir, 'Data/VID/val/ILSVRC2015_val_*'))) + anno_dirs_ = [os.path.join( + self.root_dir, 'Annotations/VID/val', + s.split('/')[-1]) for s in seq_dirs_] + seq_dirs += seq_dirs_ + anno_dirs += anno_dirs_ + seq_names = [os.path.basename(s) for s in seq_dirs] + + # cache paths and annotations + print('Caching annotations to %s, ' % self.cache_dir + \ + 'it may take a few minutes...') + seq_dict = OrderedDict() + cache_anno_dir = os.path.join(self.cache_dir, 'anno') + if not os.path.isdir(cache_anno_dir): + os.makedirs(cache_anno_dir) + + for s, seq_name in enumerate(seq_names): + if s % 100 == 0 or s == len(seq_names) - 1: + print('--Caching sequence %d/%d: %s' % \ + (s + 1, len(seq_names), seq_name)) + anno_files = sorted(glob.glob(os.path.join( + anno_dirs[s], '*.xml'))) + objects = [ET.ElementTree(file=f).findall('object') + for f in anno_files] + + # find all track ids + track_ids, counts = np.unique([ + obj.find('trackid').text for group in objects + for obj in group], return_counts=True) + + # fetch paths and annotations for each track id + for t, track_id in enumerate(track_ids): + if counts[t] < 2: + continue + frames = [] + anno = [] + for f, group in enumerate(objects): + for obj in group: + if not obj.find('trackid').text == track_id: + continue + frames.append(f) + anno.append([ + int(obj.find('bndbox/xmin').text), + int(obj.find('bndbox/ymin').text), + int(obj.find('bndbox/xmax').text), + int(obj.find('bndbox/ymax').text)]) + anno = np.array(anno, dtype=int) + anno[:, 2:] -= anno[:, :2] - 1 + + # store annotations + key = '%s.%d' % (seq_name, int(track_id)) + cache_anno_file = os.path.join(cache_anno_dir, key + '.txt') + np.savetxt(cache_anno_file, anno, fmt='%d', delimiter=',') + + # store paths + seq_dict.update([(key, [ + seq_dirs[s], frames, cache_anno_file])]) + + # store seq_dict + with open(cache_file, 'w') as f: + json.dump(seq_dict, f) + + return seq_dict diff --git a/got10k/datasets/vot.py b/got10k/datasets/vot.py new file mode 100644 index 0000000..4f2d190 --- /dev/null +++ b/got10k/datasets/vot.py @@ -0,0 +1,242 @@ +from __future__ import absolute_import, print_function, division + +import os +import glob +import numpy as np +import six +import json +import hashlib + +from ..utils.ioutils import download, extract + + +class VOT(object): + r"""`VOT `_ Datasets. + + Publication: + ``The Visual Object Tracking VOT2017 challenge results``, M. Kristan, A. Leonardis + and J. Matas, etc. 2017. + + Args: + root_dir (string): Root directory of dataset where sequence + folders exist. + version (integer, optional): Specify the benchmark version. Specify as + one of 2013~2018. Default is 2017. + anno_type (string, optional): Returned annotation types, chosen as one of + ``rect`` and ``corner``. Default is ``rect``. + download (boolean, optional): If True, downloads the dataset from the internet + and puts it in root directory. If dataset is downloaded, it is not + downloaded again. + return_meta (string, optional): If True, returns ``meta`` + of each sequence in ``__getitem__`` function, otherwise + only returns ``img_files`` and ``anno``. + """ + + __valid_versions = [2013, 2014, 2015, 2016, 2017, 2018, 'LT2018'] + + def __init__(self, root_dir, version=2017, + anno_type='rect', download=True, return_meta=False): + super(VOT, self).__init__() + assert version in self.__valid_versions, 'Unsupport VOT version.' + assert anno_type in ['default', 'rect'], 'Unknown annotation type.' + + self.root_dir = root_dir + self.version = version + self.anno_type = anno_type + if download: + self._download(root_dir, version) + self.return_meta = return_meta + self._check_integrity(root_dir, version) + + list_file = os.path.join(root_dir, 'list.txt') + with open(list_file, 'r') as f: + self.seq_names = f.read().strip().split('\n') + self.seq_dirs = [os.path.join(root_dir, s) for s in self.seq_names] + self.anno_files = [os.path.join(s, 'groundtruth.txt') + for s in self.seq_dirs] + + def __getitem__(self, index): + r""" + Args: + index (integer or string): Index or name of a sequence. + + Returns: + tuple: (img_files, anno) if ``return_meta`` is False, otherwise + (img_files, anno, meta), where ``img_files`` is a list of + file names, ``anno`` is a N x 4 (rectangles) or N x 8 (corners) numpy array, + while ``meta`` is a dict contains meta information about the sequence. + """ + if isinstance(index, six.string_types): + if not index in self.seq_names: + raise Exception('Sequence {} not found.'.format(index)) + index = self.seq_names.index(index) + + img_files = sorted(glob.glob( + os.path.join(self.seq_dirs[index], '*.jpg'))) + anno = np.loadtxt(self.anno_files[index], delimiter=',') + assert len(img_files) == len(anno) + assert anno.shape[1] in [4, 8] + if self.anno_type == 'rect' and anno.shape[1] == 8: + anno = self._corner2rect(anno) + + if self.return_meta: + meta = self._fetch_meta( + self.seq_dirs[index], len(img_files)) + return img_files, anno, meta + else: + return img_files, anno + + def __len__(self): + return len(self.seq_names) + + def _download(self, root_dir, version): + assert version in self.__valid_versions + + if not os.path.isdir(root_dir): + os.makedirs(root_dir) + elif os.path.isfile(os.path.join(root_dir, 'list.txt')): + with open(os.path.join(root_dir, 'list.txt')) as f: + seq_names = f.read().strip().split('\n') + if all([os.path.isdir(os.path.join(root_dir, s)) for s in seq_names]): + print('Files already downloaded.') + return + + if version in range(2013, 2017 + 1): + version_str = 'vot%d' % version + url = 'http://data.votchallenge.net/%s/%s.zip' % ( + version_str, version_str) + zip_file = os.path.join(root_dir, version_str + '.zip') + + print('Downloading to %s...' % zip_file) + download(url, zip_file) + print('\nExtracting to %s...' % root_dir) + extract(zip_file, root_dir) + else: + url = 'http://data.votchallenge.net/' + if version == 2018: + # main challenge + homepage = url + 'vot2018/main/' + elif version == 'LT2018': + # long-term tracking challenge + homepage = url + 'vot2018/longterm/' + + # download description file + bundle_url = homepage + 'description.json' + bundle_file = os.path.join(root_dir, 'description.json') + if not os.path.isfile(bundle_file): + print('Downloading description file...') + download(bundle_url, bundle_file) + + # read description file + print('\nParsing description file...') + with open(bundle_file) as f: + bundle = json.load(f) + + # md5 generator + def md5(filename): + hash_md5 = hashlib.md5() + with open(filename, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + # download all sequences + seq_names = [] + for seq in bundle['sequences']: + seq_name = seq['name'] + seq_names.append(seq_name) + + # download image files + seq_url = url + seq['channels']['color']['url'][6:] + seq_file = os.path.join(root_dir, seq_name + '.zip') + if not os.path.isfile(seq_file) or \ + md5(seq_file) != seq['channels']['color']['checksum']: + print('\nDownloading %s...' % seq_name) + download(seq_url, seq_file) + + # download annotations + anno_url = homepage + '%s.zip' % seq_name + anno_file = os.path.join(root_dir, seq_name + '_anno.zip') + if not os.path.isfile(anno_file) or \ + md5(anno_file) != seq['annotations']['checksum']: + download(anno_url, anno_file) + + # unzip compressed files + seq_dir = os.path.join(root_dir, seq_name) + if not os.path.isfile(seq_dir) or len(os.listdir(seq_dir)) < 10: + print('Extracting %s...' % seq_name) + os.makedirs(seq_dir) + extract(seq_file, seq_dir) + extract(anno_file, seq_dir) + + # save list.txt + list_file = os.path.join(root_dir, 'list.txt') + with open(list_file, 'w') as f: + f.write(str.join('\n', seq_names)) + + return root_dir + + def _check_integrity(self, root_dir, version): + assert version in self.__valid_versions + list_file = os.path.join(root_dir, 'list.txt') + + if os.path.isfile(list_file): + with open(list_file, 'r') as f: + seq_names = f.read().strip().split('\n') + + # check each sequence folder + for seq_name in seq_names: + seq_dir = os.path.join(root_dir, seq_name) + if not os.path.isdir(seq_dir): + print('Warning: sequence %s not exist.' % seq_name) + else: + # dataset not exist + raise Exception('Dataset not found or corrupted. ' + + 'You can use download=True to download it.') + + def _corner2rect(self, corners, center=False): + cx = np.mean(corners[:, 0::2], axis=1) + cy = np.mean(corners[:, 1::2], axis=1) + + x1 = np.min(corners[:, 0::2], axis=1) + x2 = np.max(corners[:, 0::2], axis=1) + y1 = np.min(corners[:, 1::2], axis=1) + y2 = np.max(corners[:, 1::2], axis=1) + + area1 = np.linalg.norm(corners[:, 0:2] - corners[:, 2:4], axis=1) * \ + np.linalg.norm(corners[:, 2:4] - corners[:, 4:6], axis=1) + area2 = (x2 - x1) * (y2 - y1) + scale = np.sqrt(area1 / area2) + w = scale * (x2 - x1) + 1 + h = scale * (y2 - y1) + 1 + + if center: + return np.array([cx, cy, w, h]).T + else: + return np.array([cx - w / 2, cy - h / 2, w, h]).T + + def _fetch_meta(self, seq_dir, frame_num): + meta = {} + + # attributes + tag_files = glob.glob(os.path.join(seq_dir, '*.label')) + \ + glob.glob(os.path.join(seq_dir, '*.tag')) + for f in tag_files: + tag = os.path.basename(f) + tag = tag[:tag.rfind('.')] + meta[tag] = np.loadtxt(f) + + # practical + practical_file = os.path.join(seq_dir, 'practical') + if os.path.isfile(practical_file + '.value'): + meta['practical'] = np.loadtxt(practical_file + '.value') + if os.path.isfile(practical_file + '.txt'): + meta['practical_txt'] = np.loadtxt(practical_file + '.txt') + + # pad zeros if necessary + for tag, val in meta.items(): + if len(val) < frame_num: + meta[tag] = np.pad( + val, (0, frame_num - len(val)), 'constant') + + return meta diff --git a/got10k/experiments/__init__.py b/got10k/experiments/__init__.py new file mode 100644 index 0000000..4208e61 --- /dev/null +++ b/got10k/experiments/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import + +from .got10k import ExperimentGOT10k +from .otb import ExperimentOTB +from .vot import ExperimentVOT diff --git a/got10k/experiments/got10k.py b/got10k/experiments/got10k.py new file mode 100644 index 0000000..211f96f --- /dev/null +++ b/got10k/experiments/got10k.py @@ -0,0 +1,286 @@ +from __future__ import absolute_import, division + +import os +import numpy as np +import glob +import ast +import json +import matplotlib.pyplot as plt +import matplotlib +from PIL import Image + +from ..datasets import GOT10k +from ..utils.metrics import rect_iou +from ..utils.viz import show_frame + + +class ExperimentGOT10k(object): + r"""Experiment pipeline and evaluation toolkit for GOT-10k dataset. + + Args: + root_dir (string): Root directory of GOT-10k dataset where + ``train``, ``val`` and ``test`` folders exist. + result_dir (string, optional): Directory for storing tracking + results. Default is ``./results``. + report_dir (string, optional): Directory for storing performance + evaluation results. Default is ``./reports``. + """ + + def __init__(self, root_dir, + result_dir='results', report_dir='reports'): + super(ExperimentGOT10k, self).__init__() + self.dataset = GOT10k(root_dir, subset='test', return_meta=True) + self.result_dir = os.path.join(result_dir, 'GOT-10k') + self.report_dir = os.path.join(report_dir, 'GOT-10k') + self.nbins_iou = 101 + self.repetitions = 3 + + def run(self, tracker, visualize=False): + print('Running tracker %s on GOT-10k...' % tracker.name) + + # loop over the complete dataset + for s, (img_files, anno) in enumerate(self.dataset): + seq_name = self.dataset.seq_names[s] + print('--Sequence %d/%d: %s' % ( + s + 1, len(self.dataset), seq_name)) + + # run multiple repetitions for each sequence + for r in range(self.repetitions): + # check if the tracker is deterministic + if r > 0 and tracker.is_deterministic: + break + elif r == 3 and self._check_deterministic( + tracker.name, seq_name): + print(' Detected a deterministic tracker, ' + + 'skipping remaining trials.') + break + print(' Repetition: %d' % (r + 1)) + + # skip if results exist + record_file = os.path.join( + self.result_dir, tracker.name, seq_name, + '%s_%03d.txt' % (seq_name, r + 1)) + if os.path.exists(record_file): + print(' Found results, skipping', seq_name) + continue + + # tracking loop + boxes, times = tracker.track( + img_files, anno[0, :], visualize=visualize) + + # record results + self._record(record_file, boxes, times) + + def report(self, tracker_names): + assert isinstance(tracker_names, (list, tuple)) + + # assume tracker_names[0] is your tracker + report_dir = os.path.join(self.report_dir, tracker_names[0]) + if not os.path.exists(report_dir): + os.makedirs(report_dir) + report_file = os.path.join(report_dir, 'performance.json') + + # visible ratios of all sequences + seq_names = self.dataset.seq_names + covers = {s: self.dataset[s][2]['cover'][1:] for s in seq_names} + + performance = {} + for name in tracker_names: + print('Evaluating', name) + ious = {} + times = {} + performance.update({name: { + 'overall': {}, + 'seq_wise': {}}}) + + for s, (_, anno, meta) in enumerate(self.dataset): + seq_name = self.dataset.seq_names[s] + record_files = glob.glob(os.path.join( + self.result_dir, name, seq_name, + '%s_[0-9]*.txt' % seq_name)) + if len(record_files) == 0: + raise Exception('Results for sequence %s not found.' % seq_name) + + # read results of all repetitions + boxes = [np.loadtxt(f, delimiter=',') for f in record_files] + assert all([b.shape == anno.shape for b in boxes]) + + # calculate and stack all ious + bound = ast.literal_eval(meta['resolution']) + seq_ious = [rect_iou(b[1:], anno[1:], bound=bound) for b in boxes] + # only consider valid frames where targets are visible + seq_ious = [t[covers[seq_name] > 0] for t in seq_ious] + seq_ious = np.concatenate(seq_ious) + ious[seq_name] = seq_ious + + # stack all tracking times + times[seq_name] = [] + time_file = os.path.join( + self.result_dir, name, seq_name, + '%s_time.txt' % seq_name) + if os.path.exists(time_file): + seq_times = np.loadtxt(time_file, delimiter=',') + seq_times = seq_times[~np.isnan(seq_times)] + seq_times = seq_times[seq_times > 0] + if len(seq_times) > 0: + times[seq_name] = seq_times + + # store sequence-wise performance + ao, sr, speed, _ = self._evaluate(seq_ious, seq_times) + performance[name]['seq_wise'].update({seq_name: { + 'ao': ao, + 'sr': sr, + 'speed_fps': speed, + 'length': len(anno) - 1}}) + + ious = np.concatenate(list(ious.values())) + times = np.concatenate(list(times.values())) + + # store overall performance + ao, sr, speed, succ_curve = self._evaluate(ious, times) + performance[name].update({'overall': { + 'ao': ao, + 'sr': sr, + 'speed_fps': speed, + 'succ_curve': succ_curve.tolist()}}) + + # save performance + with open(report_file, 'w') as f: + json.dump(performance, f, indent=4) + # plot success curves + self._plot_curves(performance, report_dir) + + return performance + + def show(self, tracker_names, seq_names=None, play_speed=1): + if seq_names is None: + seq_names = self.dataset.seq_names + elif isinstance(seq_names, str): + seq_names = [seq_names] + assert isinstance(tracker_names, (list, tuple)) + assert isinstance(seq_names, (list, tuple)) + + play_speed = int(round(play_speed)) + assert play_speed > 0 + + for s, seq_name in enumerate(seq_names): + print('[%d/%d] Showing results on %s...' % ( + s + 1, len(seq_names), seq_name)) + + # load all tracking results + records = {} + for name in tracker_names: + record_file = os.path.join( + self.result_dir, name, seq_name, + '%s_001.txt' % seq_name) + records[name] = np.loadtxt(record_file, delimiter=',') + + # loop over the sequence and display results + img_files, anno, _ = self.dataset[seq_name] + for f, img_file in enumerate(img_files): + if not f % play_speed == 0: + continue + image = Image.open(img_file) + boxes = [anno[f]] + [ + records[name][f] for name in tracker_names] + show_frame(image, boxes, + legends=['GroundTruth'] + tracker_names, + colors=['w', 'r', 'g', 'b', 'c', 'm', 'y', + 'orange', 'purple', 'brown', 'pink']) + + def _record(self, record_file, boxes, times): + # record bounding boxes + record_dir = os.path.dirname(record_file) + if not os.path.isdir(record_dir): + os.makedirs(record_dir) + np.savetxt(record_file, boxes, fmt='%.3f', delimiter=',') + print(' Results recorded at', record_file) + + # record running times + time_file = record_file[:record_file.rfind('_')] + '_time.txt' + times = times[:, np.newaxis] + if os.path.exists(time_file): + exist_times = np.loadtxt(time_file, delimiter=',') + if exist_times.ndim == 1: + exist_times = exist_times[:, np.newaxis] + times = np.concatenate((exist_times, times), axis=1) + np.savetxt(time_file, times, fmt='%.8f', delimiter=',') + + def _check_deterministic(self, tracker_name, seq_name): + record_dir = os.path.join( + self.result_dir, tracker_name, seq_name) + record_files = sorted(glob.glob(os.path.join( + record_dir, '%s_[0-9]*.txt' % seq_name))) + + if len(record_files) < 3: + return False + + records = [] + for record_file in record_files: + with open(record_file, 'r') as f: + records.append(f.read()) + + return len(set(records)) == 1 + + def _evaluate(self, ious, times): + # AO, SR and tracking speed + ao = np.mean(ious) + sr = np.mean(ious > 0.5) + if len(times) > 0: + # times has to be an array of positive values + speed_fps = np.mean(1. / times) + else: + speed_fps = -1 + + # success curve + thr_iou = np.linspace(0, 1, 101) + bin_iou = np.greater(ious[:, None], thr_iou[None, :]) + succ_curve = np.mean(bin_iou, axis=0) + + return ao, sr, speed_fps, succ_curve + + def _plot_curves(self, performance, report_dir): + if not os.path.exists(report_dir): + os.makedirs(report_dir) + succ_file = os.path.join(report_dir, 'success_plot.png') + key = 'overall' + + # sort trackers by AO + tracker_names = list(performance.keys()) + aos = [t[key]['ao'] for t in performance.values()] + inds = np.argsort(aos)[::-1] + tracker_names = [tracker_names[i] for i in inds] + + # markers + markers = ['-', '--', '-.'] + markers = [c + m for m in markers for c in [''] * 10] + + # plot success curves + thr_iou = np.linspace(0, 1, self.nbins_iou) + fig, ax = plt.subplots() + lines = [] + legends = [] + for i, name in enumerate(tracker_names): + line, = ax.plot(thr_iou, + performance[name][key]['succ_curve'], + markers[i % len(markers)]) + lines.append(line) + legends.append('%s: [%.3f]' % ( + name, performance[name][key]['ao'])) + matplotlib.rcParams.update({'font.size': 7.4}) + legend = ax.legend(lines, legends, loc='center left', + bbox_to_anchor=(1, 0.5)) + + matplotlib.rcParams.update({'font.size': 9}) + ax.set(xlabel='Overlap threshold', + ylabel='Success rate', + xlim=(0, 1), ylim=(0, 1), + title='Success plots on GOT-10k') + ax.grid(True) + fig.tight_layout() + + print('Saving success plots to', succ_file) + fig.savefig(succ_file, + bbox_extra_artists=(legend,), + bbox_inches='tight', + dpi=300) diff --git a/got10k/experiments/otb.py b/got10k/experiments/otb.py new file mode 100644 index 0000000..f45ab0e --- /dev/null +++ b/got10k/experiments/otb.py @@ -0,0 +1,241 @@ +from __future__ import absolute_import, division, print_function + +import os +import numpy as np +import matplotlib.pyplot as plt +import json +from PIL import Image + +from ..datasets import OTB +from ..utils.metrics import rect_iou, center_error +from ..utils.viz import show_frame + + +class ExperimentOTB(object): + r"""Experiment pipeline and evaluation toolkit for OTB dataset. + + Args: + root_dir (string): Root directory of OTB dataset. + version (integer or string): Specify the benchmark version, specify as one of + ``2013``, ``2015``, ``tb50`` and ``tb100``. Default is ``2015``. + result_dir (string, optional): Directory for storing tracking + results. Default is ``./results``. + report_dir (string, optional): Directory for storing performance + evaluation results. Default is ``./reports``. + """ + + def __init__(self, root_dir, version=2015, + result_dir='results', report_dir='reports'): + super(ExperimentOTB, self).__init__() + self.dataset = OTB(root_dir, version, download=True) + self.result_dir = os.path.join(result_dir, 'OTB' + str(version)) + self.report_dir = os.path.join(report_dir, 'OTB' + str(version)) + # as nbins_iou increases, the success score + # converges to the average overlap (AO) + self.nbins_iou = 21 + self.nbins_ce = 51 + + def run(self, tracker, visualize=False): + print('Running tracker %s on OTB...' % tracker.name) + + # loop over the complete dataset + for s, (img_files, anno) in enumerate(self.dataset): + seq_name = self.dataset.seq_names[s] + print('--Sequence %d/%d: %s' % (s + 1, len(self.dataset), seq_name)) + + # skip if results exist + record_file = os.path.join( + self.result_dir, tracker.name, '%s.txt' % seq_name) + if os.path.exists(record_file): + print(' Found results, skipping', seq_name) + continue + + # tracking loop + boxes, times = tracker.track( + img_files, anno[0, :], visualize=visualize) + assert len(boxes) == len(anno) + + # record results + self._record(record_file, boxes, times) + + def report(self, tracker_names): + assert isinstance(tracker_names, (list, tuple)) + + # assume tracker_names[0] is your tracker + report_dir = os.path.join(self.report_dir, tracker_names[0]) + if not os.path.isdir(report_dir): + os.makedirs(report_dir) + report_file = os.path.join(report_dir, 'performance.json') + + performance = {} + for name in tracker_names: + seq_num = len(self.dataset) + succ_curve = np.zeros((seq_num, self.nbins_iou)) + prec_curve = np.zeros((seq_num, self.nbins_ce)) + speeds = np.zeros(seq_num) + + performance.update({name: { + 'overall': {}, + 'seq_wise': {}}}) + + for s, (_, anno) in enumerate(self.dataset): + seq_name = self.dataset.seq_names[s] + record_file = os.path.join( + self.result_dir, name, '%s.txt' % seq_name) + boxes = np.loadtxt(record_file, delimiter=',') + boxes[0] = anno[0] + assert len(boxes) == len(anno) + + ious = rect_iou(boxes, anno) + center_errors = center_error(boxes, anno) + succ_curve[s], prec_curve[s] = self._calc_curves(ious, center_errors) + + # calculate average tracking speed + time_file = os.path.join( + self.result_dir, name, 'times/%s_time.txt' % seq_name) + if os.path.isfile(time_file): + times = np.loadtxt(time_file) + times = times[times > 0] + if len(times) > 0: + speeds[s] = np.mean(1. / times) + + # store sequence-wise performance + performance[name]['seq_wise'].update({seq_name: { + 'success_curve': succ_curve[s].tolist(), + 'precision_curve': prec_curve[s].tolist(), + 'success_score': np.mean(succ_curve[s]), + 'precision_score': prec_curve[s][20], + 'success_rate': succ_curve[s][self.nbins_iou // 2], + 'speed_fps': speeds[s] if speeds[s] > 0 else -1}}) + + succ_curve = np.mean(succ_curve, axis=0) + prec_curve = np.mean(prec_curve, axis=0) + succ_score = np.mean(succ_curve) + prec_score = prec_curve[20] + succ_rate = succ_curve[self.nbins_iou // 2] + if np.count_nonzero(speeds) > 0: + avg_speed = np.sum(speeds) / np.count_nonzero(speeds) + else: + avg_speed = -1 + + # store overall performance + performance[name]['overall'].update({ + 'success_curve': succ_curve.tolist(), + 'precision_curve': prec_curve.tolist(), + 'success_score': succ_score, + 'precision_score': prec_score, + 'success_rate': succ_rate, + 'speed_fps': avg_speed}) + + # report the performance + with open(report_file, 'w') as f: + json.dump(performance, f, indent=4) + # plot precision and success curves + self._plot_curves(performance, report_dir) + + return performance + + def show(self, tracker_names, seq_names=None, play_speed=1): + if seq_names is None: + seq_names = self.dataset.seq_names + elif isinstance(seq_names, str): + seq_names = [seq_names] + assert isinstance(tracker_names, (list, tuple)) + assert isinstance(seq_names, (list, tuple)) + + play_speed = int(round(play_speed)) + assert play_speed > 0 + + for s, seq_name in enumerate(seq_names): + print('[%d/%d] Showing results on %s...' % ( + s + 1, len(seq_names), seq_name)) + + # load all tracking results + records = {} + for name in tracker_names: + record_file = os.path.join( + self.result_dir, name, '%s.txt' % seq_name) + records[name] = np.loadtxt(record_file, delimiter=',') + + # loop over the sequence and display results + img_files, anno = self.dataset[seq_name] + for f, img_file in enumerate(img_files): + if not f % play_speed == 0: + continue + image = Image.open(img_file) + boxes = [anno[f]] + [ + records[name][f] for name in tracker_names] + show_frame(image, boxes, + legends=['GroundTruth'] + tracker_names, + colors=['w', 'r', 'g', 'b', 'c', 'm', 'y', + 'orange', 'purple', 'brown', 'pink']) + + def _record(self, record_file, boxes, times): + # record bounding boxes + record_dir = os.path.dirname(record_file) + if not os.path.isdir(record_dir): + os.makedirs(record_dir) + np.savetxt(record_file, boxes, fmt='%.3f', delimiter=',') + print(' Results recorded at', record_file) + + # record running times + time_dir = os.path.join(record_dir, 'times') + if not os.path.isdir(time_dir): + os.makedirs(time_dir) + time_file = os.path.join(time_dir, os.path.basename( + record_file).replace('.txt', '_time.txt')) + np.savetxt(time_file, times, fmt='%.8f') + + def _calc_curves(self, ious, center_errors): + ious = np.asarray(ious, float)[:, np.newaxis] + center_errors = np.asarray(center_errors, float)[:, np.newaxis] + + thr_iou = np.linspace(0, 1, self.nbins_iou)[np.newaxis, :] + thr_ce = np.arange(0, self.nbins_ce)[np.newaxis, :] + + bin_iou = np.greater(ious, thr_iou) + bin_ce = np.less_equal(center_errors, thr_ce) + + succ_curve = np.mean(bin_iou, axis=0) + prec_curve = np.mean(bin_ce, axis=0) + + return succ_curve, prec_curve + + def _plot_curves(self, performance, report_dir): + if not os.path.isdir(report_dir): + os.makedirs(report_dir) + succ_file = os.path.join(report_dir, 'success_plots.png') + prec_file = os.path.join(report_dir, 'precision_plots.png') + key = 'overall' + + # plot success curves + thr_iou = np.linspace(0, 1, self.nbins_iou) + fig, ax = plt.subplots() + lines = [] + legends = [] + for name, perf in performance.items(): + line, = ax.plot(thr_iou, perf[key]['success_curve']) + lines.append(line) + legends.append('%s: [%.3f]' % (name, perf[key]['success_score'])) + ax.legend(lines, legends, loc=1) + ax.set(xlabel='Overlap threshold', ylabel='Success rate', + xlim=(0, 1), ylim=(0, None), title='Success plots of OPE') + + print('Saving success plots to', succ_file) + fig.savefig(succ_file, dpi=300) + + # plot precision curves + thr_ce = np.arange(0, self.nbins_ce) + fig, ax = plt.subplots() + lines = [] + legends = [] + for name, perf in performance.items(): + line, = ax.plot(thr_ce, perf[key]['precision_curve']) + lines.append(line) + legends.append('%s: [%.3f]' % (name, perf[key]['precision_score'])) + ax.legend(lines, legends, loc=2) + ax.set(xlabel='Location error threshold', ylabel='Precision', + xlim=(0, 50), ylim=(0, None), title='Precision plots of OPE') + + print('Saving precision plots to', prec_file) + fig.savefig(prec_file, dpi=300) diff --git a/got10k/experiments/vot.py b/got10k/experiments/vot.py new file mode 100644 index 0000000..edb21de --- /dev/null +++ b/got10k/experiments/vot.py @@ -0,0 +1,558 @@ +from __future__ import absolute_import, division, print_function + +import time +import numpy as np +import os +import glob +import warnings +import json +from PIL import Image + +from ..datasets import VOT +from ..utils.metrics import poly_iou +from ..utils.viz import show_frame + + +class ExperimentVOT(object): + r"""Experiment pipeline and evaluation toolkit for VOT dataset. + + Notes: + - The tracking results of three types of experiments ``supervised`` + ``unsupervised`` and ``realtime`` are compatible with the official + VOT toolkit `. + - TODO: The evaluation function for VOT tracking results is still + under development. + + Args: + root_dir (string): Root directory of VOT dataset where sequence + folders exist. + version (integer, optional): Specify the VOT dataset version. Specify as + one of 2013~2018. Default is 2017. + experiments (string or tuple): Specify the type(s) of experiments to run. + Default is a tuple (``supervised``, ``unsupervised``, ``realtime``). + result_dir (string, optional): Directory for storing tracking + results. Default is ``./results``. + report_dir (string, optional): Directory for storing performance + evaluation results. Default is ``./reports``. + """ + + def __init__(self, root_dir, version=2017, + experiments=('supervised', 'unsupervised', 'realtime'), + result_dir='results', report_dir='reports'): + super(ExperimentVOT, self).__init__() + if isinstance(experiments, str): + experiments = (experiments,) + assert all([e in ['supervised', 'unsupervised', 'realtime'] + for e in experiments]) + self.dataset = VOT( + root_dir, version, anno_type='default', + download=True, return_meta=True) + self.experiments = experiments + if version == 'LT2018': + version = '-' + version + self.result_dir = os.path.join(result_dir, 'VOT' + str(version)) + self.report_dir = os.path.join(report_dir, 'VOT' + str(version)) + self.skip_initialize = 5 + self.burnin = 10 + self.repetitions = 15 + self.sensitive = 100 + self.nbins_eao = 1500 + self.tags = ['camera_motion', 'illum_change', 'occlusion', + 'size_change', 'motion_change', 'empty'] + + def run(self, tracker, visualize=False): + print('Running tracker %s on VOT...' % tracker.name) + + # run all specified experiments + if 'supervised' in self.experiments: + self.run_supervised(tracker, visualize) + if 'unsupervised' in self.experiments: + self.run_unsupervised(tracker, visualize) + if 'realtime' in self.experiments: + self.run_realtime(tracker, visualize) + + def run_supervised(self, tracker, visualize=False): + print('Running supervised experiment...') + + # loop over the complete dataset + for s, (img_files, anno, _) in enumerate(self.dataset): + seq_name = self.dataset.seq_names[s] + print('--Sequence %d/%d: %s' % (s + 1, len(self.dataset), seq_name)) + + # rectangular bounding boxes + anno_rects = anno.copy() + if anno_rects.shape[1] == 8: + anno_rects = self.dataset._corner2rect(anno_rects) + + # run multiple repetitions for each sequence + for r in range(self.repetitions): + # check if the tracker is deterministic + if r > 0 and tracker.is_deterministic: + break + elif r == 3 and self._check_deterministic('baseline', tracker.name, seq_name): + print(' Detected a deterministic tracker, ' + + 'skipping remaining trials.') + break + print(' Repetition: %d' % (r + 1)) + + # skip if results exist + record_file = os.path.join( + self.result_dir, tracker.name, 'baseline', seq_name, + '%s_%03d.txt' % (seq_name, r + 1)) + if os.path.exists(record_file): + print(' Found results, skipping', seq_name) + continue + + # state variables + boxes = [] + times = [] + failure = False + next_start = -1 + + # tracking loop + for f, img_file in enumerate(img_files): + image = Image.open(img_file) + + start_time = time.time() + if f == 0: + # initial frame + tracker.init(image, anno_rects[0]) + boxes.append([1]) + elif failure: + # during failure frames + if f == next_start: + failure = False + tracker.init(image, anno_rects[f]) + boxes.append([1]) + else: + start_time = np.NaN + boxes.append([0]) + else: + # during success frames + box = tracker.update(image) + iou = poly_iou(anno[f], box, bound=image.size) + if iou <= 0.0: + # tracking failure + failure = True + next_start = f + self.skip_initialize + boxes.append([2]) + else: + # tracking succeed + boxes.append(box) + + # store elapsed time + times.append(time.time() - start_time) + + # visualize if required + if visualize: + if len(boxes[-1]) == 4: + show_frame(image, boxes[-1]) + else: + show_frame(image) + + # record results + self._record(record_file, boxes, times) + + def run_unsupervised(self, tracker, visualize=False): + print('Running unsupervised experiment...') + + # loop over the complete dataset + for s, (img_files, anno, _) in enumerate(self.dataset): + seq_name = self.dataset.seq_names[s] + print('--Sequence %d/%d: %s' % (s + 1, len(self.dataset), seq_name)) + + # skip if results exist + record_file = os.path.join( + self.result_dir, tracker.name, 'unsupervised', seq_name, + '%s_001.txt' % seq_name) + if os.path.exists(record_file): + print(' Found results, skipping', seq_name) + continue + + # rectangular bounding boxes + anno_rects = anno.copy() + if anno_rects.shape[1] == 8: + anno_rects = self.dataset._corner2rect(anno_rects) + + # tracking loop + boxes, times = tracker.track( + img_files, anno_rects[0], visualize=visualize) + assert len(boxes) == len(anno) + + # re-formatting + boxes = list(boxes) + boxes[0] = [1] + + # record results + self._record(record_file, boxes, times) + + def run_realtime(self, tracker, visualize=False): + print('Running real-time experiment...') + + # loop over the complete dataset + for s, (img_files, anno, _) in enumerate(self.dataset): + seq_name = self.dataset.seq_names[s] + print('--Sequence %d/%d: %s' % (s + 1, len(self.dataset), seq_name)) + + # skip if results exist + record_file = os.path.join( + self.result_dir, tracker.name, 'realtime', seq_name, + '%s_001.txt' % seq_name) + if os.path.exists(record_file): + print(' Found results, skipping', seq_name) + continue + + # rectangular bounding boxes + anno_rects = anno.copy() + if anno_rects.shape[1] == 8: + anno_rects = self.dataset._corner2rect(anno_rects) + + # state variables + boxes = [] + times = [] + next_start = 0 + failure = False + failed_frame = -1 + total_time = 0.0 + grace = 3 - 1 + offset = 0 + + # tracking loop + for f, img_file in enumerate(img_files): + image = Image.open(img_file) + + start_time = time.time() + if f == next_start: + # during initial frames + tracker.init(image, anno_rects[f]) + boxes.append([1]) + + # reset state variables + failure = False + failed_frame = -1 + total_time = 0.0 + grace = 3 - 1 + offset = f + elif not failure: + # during success frames + # calculate current frame + if grace > 0: + total_time += 1000.0 / 25 + grace -= 1 + else: + total_time += max(1000.0 / 25, last_time * 1000.0) + current = offset + int(np.round(np.floor(total_time * 25) / 1000.0)) + + # delayed/tracked bounding box + if f < current: + box = boxes[-1] + elif f == current: + box = tracker.update(image) + + iou = poly_iou(anno[f], box, bound=image.size) + if iou <= 0.0: + # tracking failure + failure = True + failed_frame = f + next_start = current + self.skip_initialize + boxes.append([2]) + else: + # tracking succeed + boxes.append(box) + else: + # during failure frames + if f < current: + # skipping frame due to slow speed + boxes.append([0]) + start_time = np.NaN + elif f == current: + # current frame + box = tracker.update(image) + iou = poly_iou(anno[f], box, bound=image.size) + if iou <= 0.0: + # tracking failure + boxes.append([2]) + boxes[failed_frame] = [0] + times[failed_frame] = np.NaN + else: + # tracking succeed + boxes.append(box) + elif f < next_start: + # skipping frame due to failure + boxes.append([0]) + start_time = np.NaN + + # store elapsed time + last_time = time.time() - start_time + times.append(last_time) + + # visualize if required + if visualize: + if len(boxes[-1]) == 4: + show_frame(image, boxes[-1]) + else: + show_frame(image) + + # record results + self._record(record_file, boxes, times) + + def report(self, tracker_names): + assert isinstance(tracker_names, (list, tuple)) + + # function for loading results + def read_record(filename): + with open(filename) as f: + record = f.read().strip().split('\n') + record = [[float(t) for t in line.split(',')] + for line in record] + return record + + # assume tracker_names[0] is your tracker + report_dir = os.path.join(self.report_dir, tracker_names[0]) + if not os.path.exists(report_dir): + os.makedirs(report_dir) + report_file = os.path.join(report_dir, 'performance.json') + + performance = {} + for name in tracker_names: + print('Evaluating', name) + ious = {} + ious_full = {} + failures = {} + times = {} + masks = {} # frame masks for attribute tags + + for s, (img_files, anno, meta) in enumerate(self.dataset): + seq_name = self.dataset.seq_names[s] + + # initialize frames scores + frame_num = len(img_files) + ious[seq_name] = np.full( + (self.repetitions, frame_num), np.nan, dtype=float) + ious_full[seq_name] = np.full( + (self.repetitions, frame_num), np.nan, dtype=float) + failures[seq_name] = np.full( + (self.repetitions, frame_num), np.nan, dtype=float) + times[seq_name] = np.full( + (self.repetitions, frame_num), np.nan, dtype=float) + + # read results of all repetitions + record_files = sorted(glob.glob(os.path.join( + self.result_dir, name, 'baseline', seq_name, + '%s_[0-9]*.txt' % seq_name))) + boxes = [read_record(f) for f in record_files] + assert all([len(b) == len(anno) for b in boxes]) + + # calculate frame ious with burnin + bound = Image.open(img_files[0]).size + seq_ious = [self._calc_iou(b, anno, bound, burnin=True) + for b in boxes] + ious[seq_name][:len(seq_ious), :] = seq_ious + + # calculate frame ious without burnin + seq_ious_full = [self._calc_iou(b, anno, bound) + for b in boxes] + ious_full[seq_name][:len(seq_ious_full), :] = seq_ious_full + + # calculate frame failures + seq_failures = [ + [len(b) == 1 and b[0] == 2 for b in boxes_per_rep] + for boxes_per_rep in boxes] + failures[seq_name][:len(seq_failures), :] = seq_failures + + # collect frame runtimes + time_file = os.path.join( + self.result_dir, name, 'baseline', seq_name, + '%s_time.txt' % seq_name) + if os.path.exists(time_file): + seq_times = np.loadtxt(time_file, delimiter=',').T + times[seq_name][:len(seq_times), :] = seq_times + + # collect attribute masks + tag_num = len(self.tags) + masks[seq_name] = np.zeros((tag_num, frame_num), bool) + for i, tag in enumerate(self.tags): + if tag in meta: + masks[seq_name][i, :] = meta[tag] + # frames with no tags + if 'empty' in self.tags: + tag_frames = np.array([ + v for k, v in meta.items() + if not 'practical' in k], dtype=bool) + ind = self.tags.index('empty') + masks[seq_name][ind, :] = \ + ~np.logical_or.reduce(tag_frames, axis=0) + + # concatenate frames + seq_names = self.dataset.seq_names + masks = np.concatenate( + [masks[s] for s in seq_names], axis=1) + ious = np.concatenate( + [ious[s] for s in seq_names], axis=1) + failures = np.concatenate( + [failures[s] for s in seq_names], axis=1) + + with warnings.catch_warnings(): + # average over repetitions + warnings.simplefilter('ignore', category=RuntimeWarning) + ious = np.nanmean(ious, axis=0) + failures = np.nanmean(failures, axis=0) + + # calculate average overlaps and failures for each tag + tag_ious = np.array( + [np.nanmean(ious[m]) for m in masks]) + tag_failures = np.array( + [np.nansum(failures[m]) for m in masks]) + tag_frames = masks.sum(axis=1) + + # remove nan values + tag_ious[np.isnan(tag_ious)] = 0.0 + tag_weights = tag_frames / tag_frames.sum() + + # calculate weighted accuracy and robustness + accuracy = np.sum(tag_ious * tag_weights) + robustness = np.sum(tag_failures * tag_weights) + + # calculate tracking speed + times = np.concatenate([ + t.reshape(-1) for t in times.values()]) + # remove invalid values + times = times[~np.isnan(times)] + times = times[times > 0] + if len(times) > 0: + speed = np.mean(1. / times) + else: + speed = -1 + + performance.update({name: { + 'accuracy': accuracy, + 'robustness': robustness, + 'speed_fps': speed}}) + + # save performance + with open(report_file, 'w') as f: + json.dump(performance, f, indent=4) + print('Performance saved at', report_file) + + return performance + + def show(self, tracker_names, seq_names=None, play_speed=1, + experiment='supervised'): + if seq_names is None: + seq_names = self.dataset.seq_names + elif isinstance(seq_names, str): + seq_names = [seq_names] + assert isinstance(tracker_names, (list, tuple)) + assert isinstance(seq_names, (list, tuple)) + assert experiment in ['supervised', 'unsupervised', 'realtime'] + + play_speed = int(round(play_speed)) + assert play_speed > 0 + + # "supervised" experiment results are stored in "baseline" folder + if experiment == 'supervised': + experiment = 'baseline' + + # function for loading results + def read_record(filename): + with open(filename) as f: + record = f.read().strip().split('\n') + record = [[float(t) for t in line.split(',')] + for line in record] + for i, r in enumerate(record): + if len(r) == 4: + record[i] = np.array(r) + elif len(r) == 8: + r = np.array(r)[np.newaxis, :] + r = self.dataset._corner2rect(r) + record[i] = r[0] + else: + record[i] = np.zeros(4) + return record + + for s, seq_name in enumerate(seq_names): + print('[%d/%d] Showing results on %s...' % ( + s + 1, len(seq_names), seq_name)) + + # load all tracking results + records = {} + for name in tracker_names: + record_file = os.path.join( + self.result_dir, name, experiment, seq_name, + '%s_001.txt' % seq_name) + records[name] = read_record(record_file) + + # loop over the sequence and display results + img_files, anno, _ = self.dataset[seq_name] + if anno.shape[1] == 8: + anno = self.dataset._corner2rect(anno) + for f, img_file in enumerate(img_files): + if not f % play_speed == 0: + continue + image = Image.open(img_file) + boxes = [anno[f]] + [ + records[name][f] for name in tracker_names] + show_frame(image, boxes, + legends=['GroundTruth'] + tracker_names, + colors=['w', 'r', 'g', 'b', 'c', 'm', 'y', + 'orange', 'purple', 'brown', 'pink']) + + def _record(self, record_file, boxes, times): + # convert boxes to string + lines = [] + for box in boxes: + if len(box) == 1: + lines.append('%d' % box[0]) + else: + lines.append(str.join(',', ['%.4f' % t for t in box])) + + # record bounding boxes + record_dir = os.path.dirname(record_file) + if not os.path.isdir(record_dir): + os.makedirs(record_dir) + with open(record_file, 'w') as f: + f.write(str.join('\n', lines)) + print(' Results recorded at', record_file) + + # convert times to string + lines = ['%.4f' % t for t in times] + lines = [t.replace('nan', 'NaN') for t in lines] + + # record running times + time_file = record_file[:record_file.rfind('_')] + '_time.txt' + if os.path.exists(time_file): + with open(time_file) as f: + exist_lines = f.read().strip().split('\n') + lines = [t + ',' + s for t, s in zip(exist_lines, lines)] + with open(time_file, 'w') as f: + f.write(str.join('\n', lines)) + + def _check_deterministic(self, exp, tracker_name, seq_name): + record_dir = os.path.join( + self.result_dir, tracker_name, exp, seq_name) + record_files = sorted(glob.glob(os.path.join( + record_dir, '%s_[0-9]*.txt' % seq_name))) + + if len(record_files) < 3: + return False + + records = [] + for record_file in record_files: + with open(record_file, 'r') as f: + records.append(f.read()) + + return len(set(records)) == 1 + + def _calc_iou(self, boxes, anno, bound, burnin=False): + # skip initialization frames + if burnin: + boxes = boxes.copy() + init_inds = [i for i, box in enumerate(boxes) + if box == [1.0]] + for ind in init_inds: + boxes[ind:ind + self.burnin] = [[0]] * self.burnin + # calculate polygon ious + ious = np.array([poly_iou(np.array(a), b, bound) + if len(a) > 1 else np.NaN + for a, b in zip(boxes, anno)]) + return ious diff --git a/got10k/trackers/__init__.py b/got10k/trackers/__init__.py new file mode 100644 index 0000000..7c54039 --- /dev/null +++ b/got10k/trackers/__init__.py @@ -0,0 +1,44 @@ +from __future__ import absolute_import + +import numpy as np +import time +from PIL import Image + +from ..utils.viz import show_frame + + +class BaseTracker(object): + + def __init__(self, name, is_deterministic=False): + self.name = name + self.is_deterministic = is_deterministic + + def init(self, image, box): + raise NotImplementedError() + + def update(self, image): + raise NotImplementedError() + + def track(self, img_files, box, visualize=False): + frame_num = len(img_files) + boxes = np.zeros((frame_num, 4)) + boxes[0] = box + times = np.zeros(frame_num) + + for f, img_file in enumerate(img_files): + image = Image.open(img_file) + + start_time = time.time() + if f == 0: + self.init(image, box) + else: + boxes[f, :] = self.update(image) + times[f] = time.time() - start_time + + if visualize: + show_frame(image, boxes[f, :]) + + return boxes, times + + +from .identity_tracker import IdentityTracker diff --git a/got10k/trackers/identity_tracker.py b/got10k/trackers/identity_tracker.py new file mode 100644 index 0000000..7fe6bcd --- /dev/null +++ b/got10k/trackers/identity_tracker.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import + +from . import BaseTracker + + +class IdentityTracker(BaseTracker): + + def __init__(self): + super(IdentityTracker, self).__init__( + name='IdentityTracker', + is_deterministic=True) + + def init(self, image, box): + self.box = box + + def update(self, image): + return self.box diff --git a/got10k/utils/__init__.py b/got10k/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/got10k/utils/ioutils.py b/got10k/utils/ioutils.py new file mode 100644 index 0000000..25903d0 --- /dev/null +++ b/got10k/utils/ioutils.py @@ -0,0 +1,50 @@ +from __future__ import absolute_import, division + +import time +import sys +import os +import zipfile +from urllib.request import urlretrieve + + +def download(url, filename): + r"""Download file from the internet. + + Args: + url (string): URL of the internet file. + filename (string): Path to store the downloaded file. + """ + + return urlretrieve(url, filename, _reporthook) + + +def _reporthook(count, block_size, total_size): + global start_time + if count == 0: + start_time = time.time() + return + duration = time.time() - start_time + progress_size = int(count * block_size) + speed = int(progress_size / (1024 * duration)) + percent = int(count * block_size * 100 / total_size) + sys.stdout.write("\r...%d%%, %d MB, %d KB/s, %d seconds passed" % + (percent, progress_size / (1024 * 1024), speed, duration)) + sys.stdout.flush() + + +def extract(filename, extract_dir): + r"""Extract zip file. + + Args: + filename (string): Path of the zip file. + extract_dir (string): Directory to store the extracted results. + """ + + if os.path.splitext(filename)[1] == '.zip': + if not os.path.isdir(extract_dir): + os.makedirs(extract_dir) + with zipfile.ZipFile(filename) as z: + z.extractall(extract_dir) + else: + raise Exception('Unsupport extension {} of the compressed file {}.'.format( + os.path.splitext(filename)[1]), filename) diff --git a/got10k/utils/metrics.py b/got10k/utils/metrics.py new file mode 100644 index 0000000..6a8d594 --- /dev/null +++ b/got10k/utils/metrics.py @@ -0,0 +1,137 @@ +from __future__ import absolute_import, division + +import numpy as np +from shapely.geometry import box, Polygon + + +def center_error(rects1, rects2): + r"""Center error. + + Args: + rects1 (numpy.ndarray): An N x 4 numpy array, each line represent a rectangle + (left, top, width, height). + rects2 (numpy.ndarray): An N x 4 numpy array, each line represent a rectangle + (left, top, width, height). + """ + centers1 = rects1[..., :2] + (rects1[..., 2:] - 1) / 2 + centers2 = rects2[..., :2] + (rects2[..., 2:] - 1) / 2 + errors = np.sqrt(np.sum(np.power(centers1 - centers2, 2), axis=-1)) + + return errors + + +def rect_iou(rects1, rects2, bound=None): + r"""Intersection over union. + + Args: + rects1 (numpy.ndarray): An N x 4 numpy array, each line represent a rectangle + (left, top, width, height). + rects2 (numpy.ndarray): An N x 4 numpy array, each line represent a rectangle + (left, top, width, height). + bound (numpy.ndarray): A 4 dimensional array, denotes the bound + (min_left, min_top, max_width, max_height) for ``rects1`` and ``rects2``. + """ + assert rects1.shape == rects2.shape + if bound is not None: + # bounded rects1 + rects1[:, 0] = np.clip(rects1[:, 0], 0, bound[0]) + rects1[:, 1] = np.clip(rects1[:, 1], 0, bound[1]) + rects1[:, 2] = np.clip(rects1[:, 2], 0, bound[0] - rects1[:, 0]) + rects1[:, 3] = np.clip(rects1[:, 3], 0, bound[1] - rects1[:, 1]) + # bounded rects2 + rects2[:, 0] = np.clip(rects2[:, 0], 0, bound[0]) + rects2[:, 1] = np.clip(rects2[:, 1], 0, bound[1]) + rects2[:, 2] = np.clip(rects2[:, 2], 0, bound[0] - rects2[:, 0]) + rects2[:, 3] = np.clip(rects2[:, 3], 0, bound[1] - rects2[:, 1]) + + rects_inter = _intersection(rects1, rects2) + areas_inter = np.prod(rects_inter[..., 2:], axis=-1) + + areas1 = np.prod(rects1[..., 2:], axis=-1) + areas2 = np.prod(rects2[..., 2:], axis=-1) + areas_union = areas1 + areas2 - areas_inter + + eps = np.finfo(float).eps + ious = areas_inter / (areas_union + eps) + ious = np.clip(ious, 0.0, 1.0) + + return ious + + +def _intersection(rects1, rects2): + r"""Rectangle intersection. + + Args: + rects1 (numpy.ndarray): An N x 4 numpy array, each line represent a rectangle + (left, top, width, height). + rects2 (numpy.ndarray): An N x 4 numpy array, each line represent a rectangle + (left, top, width, height). + """ + assert rects1.shape == rects2.shape + x1 = np.maximum(rects1[..., 0], rects2[..., 0]) + y1 = np.maximum(rects1[..., 1], rects2[..., 1]) + x2 = np.minimum(rects1[..., 0] + rects1[..., 2], + rects2[..., 0] + rects2[..., 2]) + y2 = np.minimum(rects1[..., 1] + rects1[..., 3], + rects2[..., 1] + rects2[..., 3]) + + w = np.maximum(x2 - x1, 0) + h = np.maximum(y2 - y1, 0) + + return np.stack([x1, y1, w, h]).T + + +def poly_iou(polys1, polys2, bound=None): + r"""Intersection over union of polygons. + + Args: + polys1 (numpy.ndarray): An N x 4 numpy array, each line represent a rectangle + (left, top, width, height); or an N x 8 numpy array, each line represent + the coordinates (x1, y1, x2, y2, x3, y3, x4, y4) of 4 corners. + polys2 (numpy.ndarray): An N x 4 numpy array, each line represent a rectangle + (left, top, width, height); or an N x 8 numpy array, each line represent + the coordinates (x1, y1, x2, y2, x3, y3, x4, y4) of 4 corners. + """ + assert polys1.ndim in [1, 2] + if polys1.ndim == 1: + polys1 = np.array([polys1]) + polys2 = np.array([polys2]) + assert len(polys1) == len(polys2) + + polys1 = _to_polygon(polys1) + polys2 = _to_polygon(polys2) + if bound is not None: + bound = box(0, 0, bound[0], bound[1]) + polys1 = [p.intersection(bound) for p in polys1] + polys2 = [p.intersection(bound) for p in polys2] + + eps = np.finfo(float).eps + ious = [] + for poly1, poly2 in zip(polys1, polys2): + area_inter = poly1.intersection(poly2).area + area_union = poly1.union(poly2).area + ious.append(area_inter / (area_union + eps)) + ious = np.clip(ious, 0.0, 1.0) + + return ious + + +def _to_polygon(polys): + r"""Convert 4 or 8 dimensional array to Polygons + + Args: + polys (numpy.ndarray): An N x 4 numpy array, each line represent a rectangle + (left, top, width, height); or an N x 8 numpy array, each line represent + the coordinates (x1, y1, x2, y2, x3, y3, x4, y4) of 4 corners. + """ + def to_polygon(x): + assert len(x) in [4, 8] + if len(x) == 4: + return box(x[0], x[1], x[0] + x[2], x[1] + x[3]) + elif len(x) == 8: + return Polygon([(x[2 * i], x[2 * i + 1]) for i in range(4)]) + + if polys.ndim == 1: + return to_polygon(polys) + else: + return [to_polygon(t) for t in polys] diff --git a/got10k/utils/viz.py b/got10k/utils/viz.py new file mode 100644 index 0000000..402bf55 --- /dev/null +++ b/got10k/utils/viz.py @@ -0,0 +1,73 @@ +from __future__ import absolute_import + +import numpy as np +import matplotlib +import matplotlib.pyplot as plt +import matplotlib.patches as patches +import matplotlib.colors as mcolors +from PIL import Image + + +fig_dict = {} +patch_dict = {} + + +def show_frame(image, boxes=None, fig_n=1, pause=0.001, + linewidth=3, cmap=None, colors=None, legends=None): + r"""Visualize an image w/o drawing rectangle(s). + + Args: + image (numpy.ndarray or PIL.Image): Image to show. + boxes (numpy.array or a list of numpy.ndarray, optional): A 4 dimensional array + specifying rectangle [left, top, width, height] to draw, or a list of arrays + representing multiple rectangles. Default is ``None``. + fig_n (integer, optional): Figure ID. Default is 1. + pause (float, optional): Time delay for the plot. Default is 0.001 second. + linewidth (int, optional): Thickness for drawing the rectangle. Default is 3 pixels. + cmap (string): Color map. Default is None. + color (tuple): Color of drawed rectanlge. Default is None. + """ + if isinstance(image, np.ndarray): + image = Image.fromarray(image[..., ::-1]) + + if not fig_n in fig_dict or \ + fig_dict[fig_n].get_size() != image.size[::-1]: + fig = plt.figure(fig_n) + plt.axis('off') + fig.tight_layout() + fig_dict[fig_n] = plt.imshow(image, cmap=cmap) + else: + fig_dict[fig_n].set_data(image) + + if boxes is not None: + if not isinstance(boxes, (list, tuple)): + boxes = [boxes] + + if colors is None: + colors = ['r', 'g', 'b', 'c', 'm', 'y'] + \ + list(mcolors.CSS4_COLORS.keys()) + elif isinstance(colors, str): + colors = [colors] + + if not fig_n in patch_dict: + patch_dict[fig_n] = [] + for i, box in enumerate(boxes): + patch_dict[fig_n].append(patches.Rectangle( + (box[0], box[1]), box[2], box[3], linewidth=linewidth, + edgecolor=colors[i % len(colors)], facecolor='none', + alpha=0.7 if len(boxes) > 1 else 1.0)) + for patch in patch_dict[fig_n]: + fig_dict[fig_n].axes.add_patch(patch) + else: + for patch, box in zip(patch_dict[fig_n], boxes): + patch.set_xy((box[0], box[1])) + patch.set_width(box[2]) + patch.set_height(box[3]) + + if legends is not None: + fig_dict[fig_n].axes.legend( + patch_dict[fig_n], legends, loc=1, + prop={'size': 8}, fancybox=True, framealpha=0.5) + + plt.pause(pause) + plt.draw() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..65c0171 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +cycler==0.10.0 +kiwisolver==1.0.1 +matplotlib==3.0.2 +numpy==1.15.4 +Pillow==5.3.0 +pyparsing==2.3.0 +python-dateutil==2.7.5 +Shapely==1.6.4.post2 +six==1.11.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b88034e --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..b18810a --- /dev/null +++ b/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup, find_packages + + +setup(name='got10k', + version='0.1.0', + description='GOT-10k benchmark official API', + author='Lianghua Huang', + author_email='lianghua.huang.cs@gmail.com', + url='https://github.com/got-10k/toolkit', + license='MIT', + install_requires=[ + 'numpy', 'matplotlib', 'Pillow', 'Shapely'], + packages=find_packages(), + keywords=[ + 'GOT-10k', + 'Generic Object Tracking', + 'Benchmark',]) diff --git a/tests/test_datasets.py b/tests/test_datasets.py new file mode 100644 index 0000000..e7c45aa --- /dev/null +++ b/tests/test_datasets.py @@ -0,0 +1,56 @@ +from __future__ import absolute_import + +import unittest +import os +import random + +from got10k.datasets import GOT10k, OTB, VOT, ImageNetVID + + +class TestDatasets(unittest.TestCase): + + def setUp(self): + self.data_dir = 'data' + + def tearDown(self): + pass + + def test_got10k(self): + root_dir = os.path.join(self.data_dir, 'GOT-10k') + # without meta + dataset = GOT10k(root_dir, subset='train') + self._check_dataset(dataset) + # with meta + dataset = GOT10k(root_dir, subset='val', return_meta=True) + self._check_dataset(dataset) + + def test_otb(self): + root_dir = os.path.join(self.data_dir, 'OTB') + dataset = OTB(root_dir) + self._check_dataset(dataset) + + def test_vot(self): + root_dir = os.path.join(self.data_dir, 'vot2017') + # without meta + dataset = VOT(root_dir, anno_type='rect') + self._check_dataset(dataset) + # with meta + dataset = VOT(root_dir, anno_type='rect', return_meta=True) + self._check_dataset(dataset) + + def test_vid(self): + root_dir = os.path.join(self.data_dir, 'ILSVRC') + dataset = ImageNetVID(root_dir, subset=('train', 'val')) + self._check_dataset(dataset) + + def _check_dataset(self, dataset): + n = len(dataset) + self.assertGreater(n, 0) + inds = random.sample(range(n), min(n, 100)) + for i in inds: + img_files, anno = dataset[i][:2] + self.assertEqual(len(img_files), len(anno)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_experiments.py b/tests/test_experiments.py new file mode 100644 index 0000000..26e6d3e --- /dev/null +++ b/tests/test_experiments.py @@ -0,0 +1,46 @@ +from __future__ import absolute_import + +import unittest +import os + +from got10k.trackers import IdentityTracker +from got10k.experiments import ExperimentGOT10k, ExperimentOTB, \ + ExperimentVOT + + +class TestExperiments(unittest.TestCase): + + def setUp(self): + self.data_dir = 'data' + self.tracker = IdentityTracker() + + def tearDown(self): + pass + + def test_got10k(self): + root_dir = os.path.join(self.data_dir, 'GOT-10k') + # run experiment + experiment = ExperimentGOT10k(root_dir) + experiment.run(self.tracker, visualize=False) + # report performance + experiment.report([self.tracker.name]) + + def test_otb(self): + root_dir = os.path.join(self.data_dir, 'OTB') + # run experiment + experiment = ExperimentOTB(root_dir) + experiment.run(self.tracker, visualize=False) + # report performance + experiment.report([self.tracker.name]) + + def test_vot(self): + root_dir = os.path.join(self.data_dir, 'vot2018') + # run experiment + experiment = ExperimentVOT(root_dir) + experiment.run(self.tracker, visualize=False) + # report performance + experiment.report([self.tracker.name]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_trackers.py b/tests/test_trackers.py new file mode 100644 index 0000000..fd1f594 --- /dev/null +++ b/tests/test_trackers.py @@ -0,0 +1,33 @@ +from __future__ import absolute_import + +import unittest +import os +import random + +from got10k.trackers import IdentityTracker +from got10k.datasets import GOT10k + + +class TestTrackers(unittest.TestCase): + + def setUp(self): + self.data_dir = 'data' + self.tracker = IdentityTracker() + + def tearDown(self): + pass + + def test_identity_tracker(self): + # setup dataset + root_dir = os.path.join(self.data_dir, 'GOT-10k') + dataset = GOT10k(root_dir, subset='val') + # run experiment + img_files, anno = random.choice(dataset) + boxes, times = self.tracker.track( + img_files, anno[0], visualize=True) + self.assertEqual(boxes.shape, anno.shape) + self.assertEqual(len(times), len(anno)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..717eb49 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,39 @@ +from __future__ import absolute_import + +import unittest +import numpy as np + +from got10k.utils.metrics import rect_iou, poly_iou + + +class TestUtils(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_iou(self): + rects1 = np.random.rand(1000, 4) * 100 + rects2 = np.random.rand(1000, 4) * 100 + bound = (50, 100) + ious1 = rect_iou(rects1, rects2, bound=bound) + ious2 = poly_iou(rects1, rects2, bound=bound) + self.assertTrue((ious1 - ious2).max() < 1e-14) + + polys1 = self._rect2corner(rects1) + polys2 = self._rect2corner(rects2) + ious3 = poly_iou(polys1, polys2, bound=bound) + self.assertTrue((ious1 - ious3).max() < 1e-14) + + def _rect2corner(self, rects): + x1, y1, w, h = rects.T + x2, y2 = x1 + w, y1 + h + corners = np.array([x1, y1, x1, y2, x2, y2, x2, y1]).T + + return corners + + +if __name__ == '__main__': + unittest.main()