Fixed standard PSNR/SSIM test.

BUG=1103

Review URL: https://webrtc-codereview.appspot.com/971005

git-svn-id: http://webrtc.googlecode.com/svn/trunk/webrtc@3197 4adac7df-926f-26a2-2b94-8c16560cd09d

common_video/libyuv/include/webrtc_libyuv.h

@@ -43,6 +43,9 @@ enum VideoType {
   kBGRA,
 };
 
+// This is the max PSNR value our algorithms can return.
+const double kInfinitePSNR = 48.0f;
+
 // Conversion between the RawVideoType and the LibYuv videoType.
 // TODO(wu): Consolidate types into one type throughout WebRtc.
 VideoType RawVideoTypeToCommonVideoVideoType(RawVideoType type);
@@ -163,6 +166,7 @@ int MirrorI420UpDown(const I420VideoFrame* src_frame,
                      I420VideoFrame* dst_frame);
 
 // Compute PSNR for an I420 frame (all planes).
+// Returns the PSNR in decibel, to a maximum of kInfinitePSNR.
 double I420PSNR(const I420VideoFrame* ref_frame,
                 const I420VideoFrame* test_frame);
 // Compute SSIM for an I420 frame (all planes).
@@ -171,6 +175,7 @@ double I420SSIM(const I420VideoFrame* ref_frame,
 
 // TODO(mikhal): Remove these functions and keep only the above functionality.
 // Compute PSNR for an I420 buffer (all planes).
+// Returns the PSNR in decibel, to a maximum of kInfinitePSNR.
 double I420PSNR(const uint8_t* ref_frame,
                 const uint8_t* test_frame,
                 int width, int height);

common_video/libyuv/webrtc_libyuv.cc

@@ -352,9 +352,9 @@ double I420PSNR(const I420VideoFrame* ref_frame,
                                  test_frame->buffer(kVPlane),
                                  test_frame->stride(kVPlane),
                                  test_frame->width(), test_frame->height());
-  // LibYuv sets the max psnr value to 128, we restrict it to 48.
+  // LibYuv sets the max psnr value to 128, we restrict it here.
   // In case of 0 mse in one frame, 128 can skew the results significantly.
-  return (psnr > 48.0) ? 48.0 : psnr;
+  return (psnr > kInfinitePSNR) ? kInfinitePSNR : psnr;
 }
 
 // Compute SSIM for an I420 frame (all planes)
@@ -407,9 +407,9 @@ double I420PSNR(const uint8_t* ref_frame,
                                  src_u_b, half_width,
                                  src_v_b, half_width,
                                  width, height);
-  // LibYuv sets the max psnr value to 128, we restrict it to 48.
+  // LibYuv sets the max psnr value to 128, we restrict it here.
   // In case of 0 mse in one frame, 128 can skew the results significantly.
-  return (psnr > 48.0) ? 48.0 : psnr;
+  return (psnr > kInfinitePSNR) ? kInfinitePSNR : psnr;
 }
 // Compute SSIM for an I420 frame (all planes)
 double I420SSIM(const uint8_t* ref_frame,

modules/video_coding/main/source/media_optimization.cc

@@ -249,6 +249,7 @@ VCMMediaOptimization::DropFrame()
 {
     // leak appropriate number of bytes
     _frameDropper->Leak((WebRtc_UWord32)(InputFrameRate() + 0.5f));
+
     return _frameDropper->DropFrame();
 }
 

test/testsupport/metrics/video_metrics.cc

@@ -20,6 +20,9 @@
 namespace webrtc {
 namespace test {
 
+// Copy here so our callers won't need to include libyuv for this constant.
+double kMetricsInfinitePSNR = kInfinitePSNR;
+
 // Used for calculating min and max values.
 static bool LessForFrameResultValue (const FrameResult& s1,
                                      const FrameResult& s2) {

test/testsupport/metrics/video_metrics.h

@@ -17,6 +17,9 @@
 namespace webrtc {
 namespace test {
 
+// The highest PSNR value our algorithms will return.
+extern double kMetricsInfinitePSNR;
+
 // Contains video quality metrics result for a single frame.
 struct FrameResult {
   int frame_number;
@@ -43,10 +46,13 @@ struct QualityMetricsResult {
 
 // Calculates PSNR and SSIM values for the reference and test video files
 // (must be in I420 format). All calculated values are filled into the
-// QualityMetricsResult stucts.
+// QualityMetricsResult structs.
+//
 // PSNR values have the unit decibel (dB) where a high value means the test file
-// is similar to the reference file. The higher value, the more similar.
-// For more info about PSNR, see http://en.wikipedia.org/wiki/PSNR
+// is similar to the reference file. The higher value, the more similar. The
+// maximum PSNR value is kMetricsInfinitePSNR. For more info about PSNR, see
+// http://en.wikipedia.org/wiki/PSNR.
+//
 // SSIM values range between -1.0 and 1.0, where 1.0 means the files are
 // identical. For more info about SSIM, see http://en.wikipedia.org/wiki/SSIM
 // This function only compares video frames up to the point when the shortest
@@ -67,11 +73,14 @@ int I420MetricsFromFiles(const char* ref_filename,
 // Calculates PSNR values for the reference and test video files (must be in
 // I420 format). All calculated values are filled into the QualityMetricsResult
 // struct.
+//
 // PSNR values have the unit decibel (dB) where a high value means the test file
-// is similar to the reference file. The higher value, the more similar.
+// is similar to the reference file. The higher value, the more similar. The
+// maximum PSNR value is kMetricsInfinitePSNR. For more info about PSNR, see
+// http://en.wikipedia.org/wiki/PSNR.
+//
 // This function only compares video frames up to the point when the shortest
 // video ends.
-// For more info about PSNR, see http://en.wikipedia.org/wiki/PSNR
 //
 // Return value:
 //  0 if successful, negative on errors:

video_engine/test/auto_test/automated/vie_video_verification_test.cc

@@ -25,7 +25,6 @@ namespace {
 // wouldn't like scaling, so this will work when we compare with the original.
 const int kInputWidth = 176;
 const int kInputHeight = 144;
-const int kVerifyingTestMaxNumAttempts = 3;
 
 class ViEVideoVerificationTest : public testing::Test {
  protected:
@@ -72,7 +71,8 @@ class ViEVideoVerificationTest : public testing::Test {
     EXPECT_EQ(0, error) << "SSIM routine failed - output files missing?";
     *ssim_result = ssim.average;
 
-    ViETest::Log("Results: PSNR is %f (dB), SSIM is %f (1 is perfect)",
+    ViETest::Log("Results: PSNR is %f (dB; 48 is max), "
+                 "SSIM is %f (1 is perfect)",
                  psnr.average, ssim.average);
   }
 
@@ -140,41 +140,40 @@ class ParameterizedFullStackTest : public ViEVideoVerificationTest,
   TestParameters parameter_table_[2];
 };
 
-// TODO(phoglund): Needs to be rewritten to use external transport. Currently
-// the new memory-safe decoder is too slow with I420 with the default packet
-// engine. See http://code.google.com/p/webrtc/issues/detail?id=1103.
-TEST_F(ViEVideoVerificationTest, DISABLED_RunsBaseStandardTestWithoutErrors) {
-  // The I420 test should give pretty good values since it's a lossless codec
-  // running on the default bitrate. It should average about 30 dB but there
-  // may be cases where it dips as low as 26 under adverse conditions. That's
-  // why we have a retrying mechanism in place for this test.
-  const double kExpectedMinimumPSNR = 30;
-  const double kExpectedMinimumSSIM = 0.95;
-
-  for (int attempt = 0; attempt < kVerifyingTestMaxNumAttempts; attempt++) {
+TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
+  // I420 is lossless, so the I420 test should obviously get perfect results -
+  // the local preview and remote output files should be bit-exact. This test
+  // runs on external transport to ensure we do not drop packets.
+  // However, it's hard to make 100% stringent requirements on the video engine
+  // since for instance the jitter buffer has non-deterministic elements. If it
+  // breaks five times in a row though, you probably introduced a bug.
+  const int kNumAttempts = 5;
+  for (int attempt = 0; attempt < kNumAttempts; ++attempt) {
     InitializeFileRenderers();
     ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight,
                                      local_file_renderer_,
                                      remote_file_renderer_));
-    std::string output_file = remote_file_renderer_->GetFullOutputPath();
+    std::string remote_file = remote_file_renderer_->GetFullOutputPath();
+    std::string local_preview = local_file_renderer_->GetFullOutputPath();
     StopRenderers();
 
     double actual_psnr = 0;
     double actual_ssim = 0;
-    CompareFiles(input_file_, output_file, &actual_psnr, &actual_ssim);
+    CompareFiles(local_preview, remote_file, &actual_psnr, &actual_ssim);
 
     TearDownFileRenderers();
 
-    if (actual_psnr >= kExpectedMinimumPSNR &&
-        actual_ssim >= kExpectedMinimumSSIM) {
-      // Test succeeded!
+    if (actual_psnr == webrtc::test::kMetricsInfinitePSNR &&
+        actual_ssim == 1.0f) {
+      // Test successful.
       return;
+    } else {
+      ViETest::Log("Retrying; attempt %d of %d.", attempt + 1, kNumAttempts);
     }
   }
 
-  ADD_FAILURE() << "Failed to achieve PSNR " << kExpectedMinimumPSNR <<
-      " and SSIM " << kExpectedMinimumSSIM << " after " <<
-      kVerifyingTestMaxNumAttempts << " attempts.";
+  FAIL() << "Failed to achieve perfect PSNR and SSIM results after " <<
+      kNumAttempts << " attempts.";
 }
 
 // Runs a whole stack processing with tracking of which frames are dropped

video_engine/test/auto_test/primitives/base_primitives.cc

@@ -8,60 +8,66 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#include "base_primitives.h"
+#include "webrtc/video_engine/test/auto_test/primitives/base_primitives.h"
 
-#include "vie_autotest.h"
-#include "vie_autotest_defines.h"
 #include "webrtc/modules/video_capture/include/video_capture_factory.h"
+#include "webrtc/video_engine/test/auto_test/interface/vie_autotest.h"
+#include "webrtc/video_engine/test/auto_test/interface/vie_autotest_defines.h"
+#include "webrtc/video_engine/test/libvietest/include/tb_external_transport.h"
 
-void TestI420CallSetup(webrtc::ViECodec* codec_interface,
-                       webrtc::VideoEngine* video_engine,
-                       webrtc::ViEBase* base_interface,
-                       webrtc::ViENetwork* network_interface,
-                       int video_channel,
-                       const char* device_name) {
-  webrtc::VideoCodec video_codec;
-  memset(&video_codec, 0, sizeof(webrtc::VideoCodec));
-
+static void ConfigureCodecsToI420(int video_channel,
+                                  webrtc::VideoCodec video_codec,
+                                  webrtc::ViECodec* codec_interface) {
   // Set up the codec interface with all known receive codecs and with
   // I420 as the send codec.
   for (int i = 0; i < codec_interface->NumberOfCodecs(); i++) {
     EXPECT_EQ(0, codec_interface->GetCodec(i, video_codec));
 
-    // Try to keep the test frame size small when I420.
+    // Try to keep the test frame size small and bit rate generous when I420.
     if (video_codec.codecType == webrtc::kVideoCodecI420) {
       video_codec.width = 176;
       video_codec.height = 144;
+      video_codec.maxBitrate = 32000;
+      video_codec.startBitrate = 32000;
       EXPECT_EQ(0, codec_interface->SetSendCodec(video_channel, video_codec));
     }
 
     EXPECT_EQ(0, codec_interface->SetReceiveCodec(video_channel, video_codec));
   }
-
   // Verify that we really found the I420 codec.
   EXPECT_EQ(0, codec_interface->GetSendCodec(video_channel, video_codec));
   EXPECT_EQ(webrtc::kVideoCodecI420, video_codec.codecType);
+}
 
-  // Set up senders and receivers.
-  char version[1024] = "";
-  EXPECT_EQ(0, base_interface->GetVersion(version));
-  ViETest::Log("\nUsing WebRTC Video Engine version: %s", version);
+void TestI420CallSetup(webrtc::ViECodec* codec_interface,
+                       webrtc::VideoEngine* video_engine,
+                       webrtc::ViEBase* base_interface,
+                       webrtc::ViENetwork* network_interface,
+                       int video_channel,
+                       const char* device_name) {
+  webrtc::VideoCodec video_codec;
+  memset(&video_codec, 0, sizeof(webrtc::VideoCodec));
+
+  ConfigureCodecsToI420(video_channel, video_codec, codec_interface);
 
-  const char *ipAddress = "127.0.0.1";
-  WebRtc_UWord16 rtpPortListen = 6100;
-  WebRtc_UWord16 rtpPortSend = 6100;
-  EXPECT_EQ(0, network_interface->SetLocalReceiver(video_channel,
-                                                   rtpPortListen));
+  TbExternalTransport external_transport(
+      *network_interface, video_channel, NULL);
+  EXPECT_EQ(0, network_interface->RegisterSendTransport(
+      video_channel, external_transport));
   EXPECT_EQ(0, base_interface->StartReceive(video_channel));
-  EXPECT_EQ(0, network_interface->SetSendDestination(video_channel, ipAddress,
-                                                     rtpPortSend));
   EXPECT_EQ(0, base_interface->StartSend(video_channel));
 
-  // Call started.
+  // Let the call run for a while.
   ViETest::Log("Call started");
-
   AutoTestSleep(KAutoTestSleepTimeMs);
 
-  // Done.
+  // Stop the call.
+  ViETest::Log("Stopping call.");
   EXPECT_EQ(0, base_interface->StopSend(video_channel));
+
+  // Make sure we receive all packets.
+  AutoTestSleep(1000);
+
+  EXPECT_EQ(0, base_interface->StopReceive(video_channel));
+  EXPECT_EQ(0, network_interface->DeregisterSendTransport(video_channel));
 }

video_engine/test/auto_test/primitives/framedrop_primitives.cc

@@ -22,41 +22,30 @@
 #include "video_engine/test/auto_test/primitives/general_primitives.h"
 #include "video_engine/test/libvietest/include/tb_interfaces.h"
 #include "video_engine/test/libvietest/include/tb_external_transport.h"
+#include "video_engine/test/libvietest/include/vie_external_render_filter.h"
 #include "video_engine/test/libvietest/include/vie_to_file_renderer.h"
 
 enum { kWaitTimeForFinalDecodeMs = 100 };
 
 // Writes the frames to be encoded to file and tracks which frames are sent in
 // external transport on the local side and reports them to the
 // FrameDropDetector class.
-class LocalRendererEffectFilter : public webrtc::ViEEffectFilter {
+class LocalRendererEffectFilter : public webrtc::ExternalRendererEffectFilter {
  public:
-  explicit LocalRendererEffectFilter(FrameDropDetector* frame_drop_detector,
-                                     webrtc::ExternalRenderer* renderer)
-      : width_(0), height_(0), frame_drop_detector_(frame_drop_detector),
-        renderer_(renderer) {}
-  virtual ~LocalRendererEffectFilter() {}
-  virtual int Transform(int size, unsigned char* frameBuffer,
-                        unsigned int timeStamp90KHz, unsigned int width,
-                        unsigned int height) {
-    if (width != width_ || height_ != height) {
-      renderer_->FrameSizeChange(width, height, 1);
-      width_ = width;
-      height_ = height;
-    }
+  LocalRendererEffectFilter(webrtc::ExternalRenderer* renderer,
+                            FrameDropDetector* frame_drop_detector)
+      : ExternalRendererEffectFilter(renderer),
+        frame_drop_detector_(frame_drop_detector) {}
+  int Transform(int size, unsigned char* frameBuffer,
+                unsigned int timeStamp90KHz, unsigned int width,
+                unsigned int height) {
     frame_drop_detector_->ReportFrameState(FrameDropDetector::kCreated,
                                            timeStamp90KHz);
-    return renderer_->DeliverFrame(frameBuffer,
-                                   size,
-                                   timeStamp90KHz,
-                                   webrtc::TickTime::MillisecondTimestamp());
+    return webrtc::ExternalRendererEffectFilter::Transform(
+        size, frameBuffer, timeStamp90KHz, width, height);
   }
-
  private:
-  unsigned int width_;
-  unsigned int height_;
   FrameDropDetector* frame_drop_detector_;
-  webrtc::ExternalRenderer* renderer_;
 };
 
 // Tracks which frames are sent in external transport on the local side
@@ -175,8 +164,8 @@ void TestFullStack(const TbInterfaces& interfaces,
   // Setup the effect filters.
   // Local rendering at the send-side is done in an effect filter to avoid
   // synchronization issues with the remote renderer.
-  LocalRendererEffectFilter local_renderer_filter(frame_drop_detector,
-                                                  local_file_renderer);
+  LocalRendererEffectFilter local_renderer_filter(local_file_renderer,
+                                                  frame_drop_detector);
   EXPECT_EQ(0, image_process->RegisterSendEffectFilter(video_channel,
                                                        local_renderer_filter));
   DecodedTimestampEffectFilter decode_filter(frame_drop_detector);

video_engine/test/auto_test/primitives/general_primitives.cc

@@ -73,15 +73,6 @@ void RenderToFile(webrtc::ViERender* renderer_interface,
   EXPECT_EQ(0, renderer_interface->StartRender(frame_provider_id));
 }
 
-void StopAndRemoveRenderers(webrtc::ViEBase* base_interface,
-                            webrtc::ViERender* render_interface,
-                            int channel_id,
-                            int capture_id) {
-  EXPECT_EQ(0, render_interface->StopRender(channel_id));
-  EXPECT_EQ(0, render_interface->RemoveRenderer(channel_id));
-  EXPECT_EQ(0, render_interface->RemoveRenderer(capture_id));
-}
-
 void ConfigureRtpRtcp(webrtc::ViERTP_RTCP* rtcp_interface,
                       int video_channel) {
   EXPECT_EQ(0, rtcp_interface->SetRTCPStatus(video_channel,

video_engine/test/auto_test/primitives/general_primitives.h

@@ -55,13 +55,6 @@ void RenderToFile(webrtc::ViERender* renderer_interface,
                   int frame_provider_id,
                   ViEToFileRenderer* to_file_renderer);
 
-// Stops all rendering given the normal case that we have a capture device
-// and a video channel set up for rendering.
-void StopAndRemoveRenderers(webrtc::ViEBase* base_interface,
-                            webrtc::ViERender* render_interface,
-                            int channel_id,
-                            int capture_id);
-
 // Configures RTP-RTCP.
 void ConfigureRtpRtcp(webrtc::ViERTP_RTCP* rtcp_interface,
                       int video_channel);

video_engine/test/auto_test/source/vie_autotest_base.cc

@@ -75,8 +75,9 @@ void ViEAutoTest::ViEBaseStandardTest() {
   EXPECT_EQ(0, capture_interface->StopCapture(capture_id));
   EXPECT_EQ(0, base_interface->StopReceive(video_channel));
 
-  StopAndRemoveRenderers(base_interface, render_interface, video_channel,
-                         capture_id);
+  EXPECT_EQ(0, render_interface->StopRender(video_channel));
+  EXPECT_EQ(0, render_interface->RemoveRenderer(video_channel));
+  EXPECT_EQ(0, render_interface->RemoveRenderer(capture_id));
 
   EXPECT_EQ(0, render_interface->DeRegisterVideoRenderModule(*_vrm1));
   EXPECT_EQ(0, render_interface->DeRegisterVideoRenderModule(*_vrm2));