Fix evaluation tooling issues with the MessageLite proto change. Also rectify documentation accordingly.

PiperOrigin-RevId: 307060347
Change-Id: I08cf9887d9960965aee7b60af5287d077a4cafc5

Author: Sachin Joglekar

tensorflow/lite/tools/evaluation/stages/BUILD

@@ -232,6 +232,5 @@ cc_library(
         "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
         "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
         "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_protobuf//:protobuf_headers",
     ],
 )

tensorflow/lite/tools/evaluation/stages/object_detection_stage.cc

@@ -16,7 +16,6 @@ limitations under the License.
 
 #include <fstream>
 
-#include "google/protobuf/text_format.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/lite/c/common.h"
 #include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
@@ -171,7 +170,7 @@ TfLiteStatus PopulateGroundTruth(
   std::string proto_str((std::istreambuf_iterator<char>(t)),
                         std::istreambuf_iterator<char>());
   ObjectDetectionGroundTruth ground_truth_proto;
-  google::protobuf::TextFormat::ParseFromString(proto_str, &ground_truth_proto);
+  ground_truth_proto.ParseFromString(proto_str);
 
   for (const auto& image_ground_truth :
        ground_truth_proto.detection_results()) {

tensorflow/lite/tools/evaluation/tasks/coco_object_detection/BUILD

@@ -29,9 +29,9 @@ cc_binary(
         "//conditions:default": [],
     }),
     deps = [
-        "//tensorflow/core:tflite_portable_logging",
         "//tensorflow/lite/c:common",
         "//tensorflow/lite/tools:command_line_flags",
+        "//tensorflow/lite/tools:logging",
         "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
         "//tensorflow/lite/tools/evaluation:evaluation_stage",
         "//tensorflow/lite/tools/evaluation:utils",

tensorflow/lite/tools/evaluation/tasks/coco_object_detection/README.md

@@ -11,75 +11,23 @@ This binary evaluates the following parameters of TFLite models trained for the
 
 The binary takes the path to validation images and a ground truth proto file as
 inputs, along with the model and inference-specific parameters such as delegate
-and number of threads. It outputs the metrics as a text proto to a file, similar
-to the following:
+and number of threads. It outputs the metrics to std-out as follows:
 
 ```
-num_runs: 8059
-process_metrics {
-  object_detection_metrics {
-    pre_processing_latency {
-      last_us: 27197
-      max_us: 61372
-      min_us: 6166
-      sum_us: 189403170
-      avg_us: 23502.068494850479
-    }
-    inference_latency {
-      last_us: 386378
-      max_us: 412804
-      min_us: 378841
-      sum_us: 3122849071
-      avg_us: 387498.33366422635 # Average Inference Latency.
-    }
-    inference_metrics {
-      num_inferences: 8059 # Number of images evaluated.
-    }
-    average_precision_metrics {
-      individual_average_precisions {
-        iou_threshold: 0.5
-        average_precision: 0.26113987
-      }
-      individual_average_precisions {
-        iou_threshold: 0.55
-        average_precision: 0.2456704
-      }
-      individual_average_precisions {
-        iou_threshold: 0.6
-        average_precision: 0.22885525
-      }
-      individual_average_precisions {
-        iou_threshold: 0.65
-        average_precision: 0.20678344
-      }
-      individual_average_precisions {
-        iou_threshold: 0.7
-        average_precision: 0.18185228
-      }
-      individual_average_precisions {
-        iou_threshold: 0.75
-        average_precision: 0.14681709 # AP at IoU threshold of 0.75.
-      }
-      individual_average_precisions {
-        iou_threshold: 0.8
-        average_precision: 0.107850626
-      }
-      individual_average_precisions {
-        iou_threshold: 0.85
-        average_precision: 0.061735578
-      }
-      individual_average_precisions {
-        iou_threshold: 0.9
-        average_precision: 0.017980274
-      }
-      individual_average_precisions {
-        iou_threshold: 0.95
-        average_precision: 0.0010084915
-      }
-      overall_mean_average_precision: 0.14596924 # Overall mAP average.
-    }
-  }
-}
+Num evaluation runs: 8059
+Preprocessing latency: avg=16589.9(us), std_dev=0(us)
+Inference latency: avg=85169.7(us), std_dev=505(us)
+Average Precision [IOU Threshold=0.5]: 0.349581
+Average Precision [IOU Threshold=0.55]: 0.330213
+Average Precision [IOU Threshold=0.6]: 0.307694
+Average Precision [IOU Threshold=0.65]: 0.281025
+Average Precision [IOU Threshold=0.7]: 0.248507
+Average Precision [IOU Threshold=0.75]: 0.210295
+Average Precision [IOU Threshold=0.8]: 0.165011
+Average Precision [IOU Threshold=0.85]: 0.116215
+Average Precision [IOU Threshold=0.9]: 0.0507883
+Average Precision [IOU Threshold=0.95]: 0.0064338
+Overall mAP: 0.206576
 ```
 
 To run the binary, please follow the
@@ -135,15 +83,47 @@ The following optional parameters can be used to modify the inference runtime:
     assumes that `libhexagon_interface.so` and Qualcomm libraries lie in
     `/data/local/tmp`.
 
+This script also supports all applicable runtime/delegate arguments supported on
+the `benchmark_model` tool. If there is any conflict (for example, `num_threads`
+in `benchmark_model` vs `num_interpreter_threads` here), the parameters of this
+script are given precedence.
+
 ### Debug Mode
 
 The script also supports a debug mode with the following parameter:
 
 *   `debug_mode`: `boolean` \
-    Whether to enable debug mode. Per-image predictions are written to the
-    output file along with metrics. NOTE: Its not possible to parse the output
-    file as a proto in this mode, since it contains demarcations between
-    per-file outputs for readability.
+    Whether to enable debug mode. Per-image predictions are written to std-out
+    along with metrics.
+
+Image-wise predictions are output as follows:
+
+```
+======================================================
+
+Image: image_1.jpg
+
+Object [0]
+  Score: 0.585938
+  Class-ID: 5
+  Bounding Box:
+    Normalized Top: 0.23103
+    Normalized Bottom: 0.388524
+    Normalized Left: 0.559144
+    Normalized Right: 0.763928
+Object [1]
+  Score: 0.574219
+  Class-ID: 5
+  Bounding Box:
+    Normalized Top: 0.269571
+    Normalized Bottom: 0.373971
+    Normalized Left: 0.613175
+    Normalized Right: 0.760507
+======================================================
+
+Image: image_2.jpg
+...
+```
 
 This mode lets you debug the output of an object detection model that isn't
 necessarily trained on the COCO dataset (by leaving `ground_truth_proto` empty).
@@ -187,7 +167,7 @@ The script generates the following within the output folder:
 
 *   `images/`: the resulting subset of the 2014 COCO Validation images.
 
-*   `ground_truth.pbtxt`: a `.pbtxt` (text proto) file holding
+*   `ground_truth.pb`: a `.pb` (binary-format proto) file holding
     `tflite::evaluation::ObjectDetectionGroundTruth` corresponding to image
     subset.
 
@@ -248,7 +228,7 @@ adb push /path/to/output/folder /data/local/tmp/coco_validation
 adb shell /data/local/tmp/run_eval \
   --model_file=/data/local/tmp/ssd_mobilenet_v1_float.tflite \
   --ground_truth_images_path=/data/local/tmp/coco_validation/images \
-  --ground_truth_proto=/data/local/tmp/coco_validation/ground_truth.pbtxt \
+  --ground_truth_proto=/data/local/tmp/coco_validation/ground_truth.pb \
   --model_output_labels=/data/local/tmp/labelmap.txt \
   --output_file_path=/data/local/tmp/coco_output.txt
 ```
@@ -266,7 +246,7 @@ bazel run -c opt \
   //tensorflow/lite/tools/evaluation/tasks/coco_object_detection:run_eval \
   --model_file=/path/to/ssd_mobilenet_v1_float.tflite \
   --ground_truth_images_path=/path/to/images \
-  --ground_truth_proto=/path/to/ground_truth.pbtxt \
+  --ground_truth_proto=/path/to/ground_truth.pb \
   --model_output_labels=/path/to/labelmap.txt \
   --output_file_path=/path/to/coco_output.txt
 ```

tensorflow/lite/tools/evaluation/tasks/coco_object_detection/preprocess_coco_minival.py

@@ -134,7 +134,7 @@ def _dump_data(ground_truth_detections, images_folder_path, output_folder_path):
 
   The following are created in output_folder_path:
     images/: sub-folder for whitelisted validation images.
-    ground_truth.pbtxt: A text proto file containing all ground-truth
+    ground_truth.pb: A binary proto file containing all ground-truth
     object-sets.
 
   Args:
@@ -149,7 +149,7 @@ def _dump_data(ground_truth_detections, images_folder_path, output_folder_path):
   output_images_folder = os.path.join(output_folder_path, 'images')
   if not os.path.exists(output_images_folder):
     os.makedirs(output_images_folder)
-  output_proto_file = os.path.join(output_folder_path, 'ground_truth.pbtxt')
+  output_proto_file = os.path.join(output_folder_path, 'ground_truth.pb')
 
   ground_truth_data = evaluation_stages_pb2.ObjectDetectionGroundTruth()
   for image_dict in ground_truth_detections.values():
@@ -170,8 +170,8 @@ def _dump_data(ground_truth_detections, images_folder_path, output_folder_path):
         output_images_folder)
 
   # Dump proto.
-  with open(output_proto_file, 'w') as proto_file:
-    proto_file.write(str(ground_truth_data))
+  with open(output_proto_file, 'wb') as proto_file:
+    proto_file.write(ground_truth_data.SerializeToString())
 
 
 def _parse_args():

tensorflow/lite/tools/evaluation/tasks/coco_object_detection/run_eval.cc

@@ -18,14 +18,14 @@ limitations under the License.
 #include <vector>
 
 #include "absl/container/flat_hash_map.h"
-#include "tensorflow/core/platform/logging.h"
 #include "tensorflow/lite/c/common.h"
 #include "tensorflow/lite/tools/command_line_flags.h"
 #include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
 #include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
 #include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
 #include "tensorflow/lite/tools/evaluation/stages/object_detection_stage.h"
 #include "tensorflow/lite/tools/evaluation/utils.h"
+#include "tensorflow/lite/tools/logging.h"
 
 namespace tflite {
 namespace evaluation {
@@ -62,7 +62,7 @@ bool EvaluateModel(const std::string& model_file_path,
   inference_params->set_delegate(ParseStringToDelegateType(delegate));
   if (!delegate.empty() &&
       inference_params->delegate() == TfliteInferenceParams::NONE) {
-    LOG(WARNING) << "Unsupported TFLite delegate: " << delegate;
+    TFLITE_LOG(WARN) << "Unsupported TFLite delegate: " << delegate;
     return false;
   }
 
@@ -77,14 +77,10 @@ bool EvaluateModel(const std::string& model_file_path,
   eval.SetAllLabels(model_labels);
   if (eval.Init(&delegate_providers) != kTfLiteOk) return false;
 
-  // Open output file for writing.
-  std::ofstream ofile;
-  ofile.open(output_file_path, std::ios::out);
-
   const int step = image_paths.size() / 100;
   for (int i = 0; i < image_paths.size(); ++i) {
     if (step > 1 && i % step == 0) {
-      LOG(INFO) << "Finished: " << i / step << "%";
+      TFLITE_LOG(INFO) << "Finished: " << i / step << "%";
     }
 
     const std::string image_name = GetNameFromPath(image_paths[i]);
@@ -93,22 +89,65 @@ bool EvaluateModel(const std::string& model_file_path,
 
     if (debug_mode) {
       ObjectDetectionResult prediction = *eval.GetLatestPrediction();
-      prediction.set_image_name(image_name);
-      ofile << prediction.DebugString();
-      ofile << "======================================================\n";
+      TFLITE_LOG(INFO) << "Image: " << image_name << "\n";
+      for (int i = 0; i < prediction.objects_size(); ++i) {
+        const auto& object = prediction.objects(i);
+        TFLITE_LOG(INFO) << "Object [" << i << "]";
+        TFLITE_LOG(INFO) << "  Score: " << object.score();
+        TFLITE_LOG(INFO) << "  Class-ID: " << object.class_id();
+        TFLITE_LOG(INFO) << "  Bounding Box:";
+        const auto& bounding_box = object.bounding_box();
+        TFLITE_LOG(INFO) << "    Normalized Top: "
+                         << bounding_box.normalized_top();
+        TFLITE_LOG(INFO) << "    Normalized Bottom: "
+                         << bounding_box.normalized_bottom();
+        TFLITE_LOG(INFO) << "    Normalized Left: "
+                         << bounding_box.normalized_left();
+        TFLITE_LOG(INFO) << "    Normalized Right: "
+                         << bounding_box.normalized_right();
+      }
+      TFLITE_LOG(INFO)
+          << "======================================================\n";
     }
   }
 
   // Write metrics to file.
-  EvaluationStageMetrics metrics = eval.LatestMetrics();
+  EvaluationStageMetrics latest_metrics = eval.LatestMetrics();
   if (ground_truth_proto_file.empty()) {
     // mAP metrics are meaningless for no ground truth.
-    metrics.mutable_process_metrics()
+    latest_metrics.mutable_process_metrics()
         ->mutable_object_detection_metrics()
         ->clear_average_precision_metrics();
   }
-  ofile << metrics.DebugString();
-  ofile.close();
+  if (!output_file_path.empty()) {
+    std::ofstream metrics_ofile;
+    metrics_ofile.open(output_file_path, std::ios::out);
+    metrics_ofile << latest_metrics.SerializeAsString();
+    metrics_ofile.close();
+  }
+  TFLITE_LOG(INFO) << "Num evaluation runs: " << latest_metrics.num_runs();
+  const auto object_detection_metrics =
+      latest_metrics.process_metrics().object_detection_metrics();
+  const auto& preprocessing_latency =
+      object_detection_metrics.pre_processing_latency();
+  TFLITE_LOG(INFO) << "Preprocessing latency: avg="
+                   << preprocessing_latency.avg_us() << "(us), std_dev="
+                   << preprocessing_latency.std_deviation_us() << "(us)";
+  const auto& inference_latency = object_detection_metrics.inference_latency();
+  TFLITE_LOG(INFO) << "Inference latency: avg=" << inference_latency.avg_us()
+                   << "(us), std_dev=" << inference_latency.std_deviation_us()
+                   << "(us)";
+  const auto& precision_metrics =
+      object_detection_metrics.average_precision_metrics();
+  for (int i = 0; i < precision_metrics.individual_average_precisions_size();
+       ++i) {
+    const auto ap_metric = precision_metrics.individual_average_precisions(i);
+    TFLITE_LOG(INFO) << "Average Precision [IOU Threshold="
+                     << ap_metric.iou_threshold()
+                     << "]: " << ap_metric.average_precision();
+  }
+  TFLITE_LOG(INFO) << "Overall mAP: "
+                   << precision_metrics.overall_mean_average_precision();
 
   return true;
 }
@@ -167,14 +206,14 @@ int Main(int argc, char* argv[]) {
 
   std::vector<std::string> model_labels;
   if (!ReadFileLines(model_output_labels_path, &model_labels)) {
-    LOG(ERROR) << "Could not read model output labels file";
+    TFLITE_LOG(ERROR) << "Could not read model output labels file";
     return EXIT_FAILURE;
   }
 
   if (!EvaluateModel(model_file_path, model_labels, image_paths,
                      ground_truth_proto_file, delegate, output_file_path,
                      num_interpreter_threads, debug_mode, delegate_providers)) {
-    LOG(ERROR) << "Could not evaluate model";
+    TFLITE_LOG(ERROR) << "Could not evaluate model";
     return EXIT_FAILURE;
   }
 

tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification/BUILD

@@ -22,9 +22,9 @@ cc_binary(
     copts = tflite_copts(),
     linkopts = common_linkopts,
     deps = [
-        "//tensorflow/core:tflite_portable_logging",
         "//tensorflow/lite/c:common",
         "//tensorflow/lite/tools:command_line_flags",
+        "//tensorflow/lite/tools:logging",
         "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
         "//tensorflow/lite/tools/evaluation:evaluation_stage",
         "//tensorflow/lite/tools/evaluation:utils",