[AVA] Correct handling of distractor frames. (#21)

* Support an "excluded timestamps" file that means that detections/ground
truth annotations on these timestamps are ignored.

* Ignore *.pyc and *~ tempfiles.

* Remove unnecessary print.

* label map reader updated to handle files with id: and label_id:

.gitignore

@@ -1 +1,3 @@
 *.pyc
+*.pyo
+*~

Evaluation/get_ava_performance.py

@@ -1,9 +1,13 @@
 r"""Compute action detection performance for the AVA dataset.
 
+Please send any questions about this code to the Google Group ava-dataset-users:
+https://groups.google.com/forum/#!forum/ava-dataset-users
+
 Example usage:
 python -O get_ava_performance.py \
   -l ava/ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt \
   -g ava_val_v2.1.csv \
+  -e ava_val_excluded_timestamps_v2.1.csv \
   -d your_results.csv
 """
 
@@ -28,6 +32,11 @@ def print_time(message, start):
   logging.info("==> %g seconds to %s", time.time() - start, message)
 
 
+def make_image_key(video_id, timestamp):
+  """Returns a unique identifier for a video id & timestamp."""
+  return "%s,%04d" % (video_id, int(timestamp))
+
+
 def read_csv(csv_file, class_whitelist=None):
   """Loads boxes and class labels from a CSV file in the AVA format.
 
@@ -54,23 +63,40 @@ def read_csv(csv_file, class_whitelist=None):
   reader = csv.reader(csv_file)
   for row in reader:
     assert len(row) in [7, 8], "Wrong number of columns: " + row
-    video_id = row[0]
-    timestamp = int(row[1])
+    image_key = make_image_key(row[0], row[1])
     x1, y1, x2, y2 = [float(n) for n in row[2:6]]
     action_id = int(row[6])
     if class_whitelist and action_id not in class_whitelist:
       continue
     score = 1.0
     if len(row) == 8:
       score = float(row[7])
-    image_key = "%s,%04d" % (video_id, timestamp)
     boxes[image_key].append([y1, x1, y2, x2])
     labels[image_key].append(action_id)
     scores[image_key].append(score)
   print_time("read file " + csv_file.name, start)
   return boxes, labels, scores
 
 
+def read_exclusions(exclusions_file):
+  """Reads a CSV file of excluded timestamps.
+
+  Args:
+    exclusions_file: A file object containing a csv of video-id,timestamp.
+
+  Returns:
+    A set of strings containing excluded image keys, e.g. "aaaaaaaaaaa,0904",
+    or an empty set if exclusions file is None.
+  """
+  excluded = set()
+  if exclusions_file:
+    reader = csv.reader(exclusions_file)
+    for row in reader:
+      assert len(row) == 2, "Expected only 2 columns, got: " + row
+      excluded.add(make_image_key(row[0], row[1]))
+  return excluded
+
+
 def read_labelmap(labelmap_file):
   """Reads a labelmap without the dependency on protocol buffers.
 
@@ -89,24 +115,26 @@ def read_labelmap(labelmap_file):
   for line in labelmap_file:
     if line.startswith("  name:"):
       name = line.split('"')[1]
-    elif line.startswith("  id:"):
+    elif line.startswith("  id:") or line.startswith("  label_id:"):
       class_id = int(line.strip().split(" ")[-1])
       labelmap.append({"id": class_id, "name": name})
       class_ids.add(class_id)
   return labelmap, class_ids
 
 
-def run_evaluation(labelmap, groundtruth, detections):
+def run_evaluation(labelmap, groundtruth, detections, exclusions):
   """Runs evaluations given input files.
 
   Args:
     labelmap: file object containing map of labels to consider, in pbtxt format
     groundtruth: file object
     detections: file object
+    exclusions: file object or None.
   """
   categories, class_whitelist = read_labelmap(labelmap)
   logging.info("CATEGORIES (%d):\n%s", len(categories),
                pprint.pformat(categories, indent=2))
+  excluded_keys = read_exclusions(exclusions)
 
   pascal_evaluator = object_detection_evaluation.PascalDetectionEvaluator(
       categories)
@@ -115,6 +143,10 @@ def run_evaluation(labelmap, groundtruth, detections):
   boxes, labels, _ = read_csv(groundtruth, class_whitelist)
   start = time.time()
   for image_key in boxes:
+    if image_key in excluded_keys:
+      logging.info(("Found excluded timestamp in ground truth: %s. "
+                    "It will be ignored."), image_key)
+      continue
     pascal_evaluator.add_single_ground_truth_image_info(
         image_key, {
             standard_fields.InputDataFields.groundtruth_boxes:
@@ -130,6 +162,10 @@ def run_evaluation(labelmap, groundtruth, detections):
   boxes, labels, scores = read_csv(detections, class_whitelist)
   start = time.time()
   for image_key in boxes:
+    if image_key in excluded_keys:
+      logging.info(("Found excluded timestamp in detections: %s. "
+                    "It will be ignored."), image_key)
+      continue
     pascal_evaluator.add_single_detected_image_info(
         image_key, {
             standard_fields.DetectionResultFields.detection_boxes:
@@ -173,6 +209,13 @@ def parse_arguments():
       help="CSV file containing inferred action detections.",
       type=argparse.FileType("r"),
       required=True)
+  parser.add_argument(
+      "-e",
+      "--exclusions",
+      help=("Optional CSV file containing videoid,timestamp pairs to exclude "
+            "from evaluation."),
+      type=argparse.FileType("r"),
+      required=False)
   return parser.parse_args()