Add new FFmpeg regression tests. Add flaky test wrapper.

Adds the files from the issue and introduces a new flaky test type
which only ensures that the file loads and runs without crashing.

Changes are in preparation for running ffmpeg_regression_tests on
bots.

BUG=42122
TEST=ffmpeg_regression_tests!

Review URL: https://chromiumcodereview.appspot.com/10518006

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@140902 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 107 insertions, 42 deletions

diff --git a/media/ffmpeg/ffmpeg_regression_tests.cc b/media/ffmpeg/ffmpeg_regression_tests.cc
index 17ca268..6bdde35 100644
--- a/media/ffmpeg/ffmpeg_regression_tests.cc
+++ b/media/ffmpeg/ffmpeg_regression_tests.cc
@@ -58,11 +58,26 @@ struct RegressionTestData {
   PipelineStatus end_status;
 };
 
+// Used for tests which just need to run without crashing or tooling errors, but
+// which may have undefined behavior for hashing, etc.
+struct FlakyRegressionTestData {
+  FlakyRegressionTestData(const char* filename)
+      : filename(filename) {
+  }
+
+  const char* filename;
+};
+
 class FFmpegRegressionTest
     : public testing::TestWithParam<RegressionTestData>,
       public PipelineIntegrationTestBase {
 };
 
+class FlakyFFmpegRegressionTest
+    : public testing::TestWithParam<FlakyRegressionTestData>,
+      public PipelineIntegrationTestBase {
+};
+
 #define FFMPEG_TEST_CASE(name, fn, init_status, end_status, video_md5, \
                          audio_md5) \
     INSTANTIATE_TEST_CASE_P(name, FFmpegRegressionTest, \
@@ -72,6 +87,10 @@ class FFmpegRegressionTest
                                                                video_md5, \
                                                                audio_md5)));
 
+#define FLAKY_FFMPEG_TEST_CASE(name, fn) \
+    INSTANTIATE_TEST_CASE_P(FLAKY_##name, FlakyFFmpegRegressionTest, \
+                            testing::Values(FlakyRegressionTestData(fn)));
+
 // Test cases from issues.
 FFMPEG_TEST_CASE(Cr47325, "security/47325.mp4", PIPELINE_OK, PIPELINE_OK,
                  "2a7a938c6b5979621cec998f02d9bbb6",
@@ -145,6 +164,13 @@ FFMPEG_TEST_CASE(MP4_8, "security/h264.705767.mp4",
 FFMPEG_TEST_CASE(MP4_9, "security/smclockmp4aac_1_0.mp4",
                  DEMUXER_ERROR_COULD_NOT_OPEN, DEMUXER_ERROR_COULD_NOT_OPEN,
                  kNullHash, kNullHash);
+FFMPEG_TEST_CASE(MP4_10, "security/null1.m4a", PIPELINE_OK, PIPELINE_OK,
+                 kNullHash, "287b6f06b6b45ac9e2839f4f397036af");
+FFMPEG_TEST_CASE(MP4_11, "security/null1.mp4", PIPELINE_OK, PIPELINE_OK,
+                 kNullHash, "f20676c5de9b3c7c174c141762afb957");
+FFMPEG_TEST_CASE(MP4_16, "security/looping2.mov",
+                 DEMUXER_ERROR_COULD_NOT_OPEN, DEMUXER_ERROR_COULD_NOT_OPEN,
+                 kNullHash, kNullHash);
 
 // General OGV test cases.
 FFMPEG_TEST_CASE(OGV_1, "security/out.163.ogv", DECODER_ERROR_NOT_SUPPORTED,
@@ -186,6 +212,14 @@ FFMPEG_TEST_CASE(OGV_17, "security/vorbis.482086.ogv",
                  kNullHash, kNullHash);
 FFMPEG_TEST_CASE(OGV_18, "security/wav.711.ogv", DECODER_ERROR_NOT_SUPPORTED,
                  DECODER_ERROR_NOT_SUPPORTED, kNullHash, kNullHash);
+FFMPEG_TEST_CASE(OGV_19, "security/null1.ogv", DECODER_ERROR_NOT_SUPPORTED,
+                 DECODER_ERROR_NOT_SUPPORTED, kNullHash, kNullHash);
+FFMPEG_TEST_CASE(OGV_20, "security/null2.ogv", DECODER_ERROR_NOT_SUPPORTED,
+                 DECODER_ERROR_NOT_SUPPORTED, kNullHash, kNullHash);
+FFMPEG_TEST_CASE(OGV_21, "security/assert1.ogv", DECODER_ERROR_NOT_SUPPORTED,
+                 DECODER_ERROR_NOT_SUPPORTED, kNullHash, kNullHash);
+FFMPEG_TEST_CASE(OGV_22, "security/assert2.ogv", DECODER_ERROR_NOT_SUPPORTED,
+                 DECODER_ERROR_NOT_SUPPORTED, kNullHash, kNullHash);
 
 // General WebM test cases.
 FFMPEG_TEST_CASE(WEBM_1, "security/no-bug.webm", PIPELINE_OK, PIPELINE_OK,
@@ -203,43 +237,6 @@ FFMPEG_TEST_CASE(WEBM_5, "content/frame_size_change.webm", PIPELINE_OK,
 FFMPEG_TEST_CASE(WEBM_6, "security/117912.webm", DEMUXER_ERROR_COULD_NOT_OPEN,
                  DEMUXER_ERROR_COULD_NOT_OPEN, kNullHash, kNullHash);
 
-// Flaky under threading, maybe larger issues.  Values were set with
-// --video-threads=1 under the hope that they may one day pass with threading.
-FFMPEG_TEST_CASE(FLAKY_Cr99652, "security/99652.webm", PIPELINE_OK,
-                 PIPELINE_ERROR_DECODE, "97956dca8897484fbd0dd1169578adbf",
-                 kNullHash);
-FFMPEG_TEST_CASE(FLAKY_Cr100464, "security/100464.webm", PIPELINE_OK,
-                 PIPELINE_OK, "a32ddb4ac5ca89429e1a3c968e876ce8",
-                 "0cd51b7ff41c1c760719d1b3a06c5ce8");
-FFMPEG_TEST_CASE(FLAKY_OGV_0, "security/big_dims.ogv", PIPELINE_ERROR_DECODE,
-                 PIPELINE_ERROR_DECODE, kNullHash, kNullHash);
-FFMPEG_TEST_CASE(FLAKY_OGV_3, "security/smclock_1_0.ogv", PIPELINE_OK,
-                 PIPELINE_OK, "2f7aa71ce789e47a1dde98ecdd774679",
-                 "cc7b61321df57874dba5ff780786a04d");
-FFMPEG_TEST_CASE(FLAKY_OGV_4, "security/smclock.ogv.1.0.ogv",
-                 PIPELINE_OK, PIPELINE_OK, "2f7aa71ce789e47a1dde98ecdd774679",
-                 "cc7b61321df57874dba5ff780786a04d");
-FFMPEG_TEST_CASE(FLAKY_OGV_6, "security/smclocktheora_1_10000.ogv",
-                 PIPELINE_ERROR_DECODE, PIPELINE_ERROR_DECODE, kNullHash,
-                 kNullHash);
-FFMPEG_TEST_CASE(FLAKY_OGV_13, "security/smclocktheora_1_790.ogv",
-                 PIPELINE_OK, PIPELINE_OK, "2f7aa71ce789e47a1dde98ecdd774679",
-                 "cc7b61321df57874dba5ff780786a04d");
-FFMPEG_TEST_CASE(FLAKY_MP4_3, "security/clockh264aac_300413969.mp4",
-                 PIPELINE_OK, PIPELINE_ERROR_DECODE,
-                 "b7659b127be88829a68d8a9aa625795e", kNullHash);
-FFMPEG_TEST_CASE(FLAKY_MP4_4, "security/clockh264aac_301350139.mp4",
-                 PIPELINE_OK, PIPELINE_OK, "bf1dbeb4ee6edb1b6c78742f4943c162",
-                 "98731537a560cda29b7f46f2fb5d7f0b");
-
-// Flaky due to other non-threading related reasons.
-FFMPEG_TEST_CASE(FLAKY_Cr111342, "security/111342.ogm", PIPELINE_OK,
-                 PIPELINE_ERROR_DECODE, "8a14eb9ce0671194a0b04fee8b0b5368",
-                 "aa348fa026a6ce22b781daf94ece20c0");
-
-// Hangs. http://crbug.com/117038
-// FFMPEG_TEST_CASE(WEBM_0, "security/memcpy.webm", PIPELINE_OK, PIPELINE_OK);
-
 // Audio Functional Tests
 FFMPEG_TEST_CASE(AUDIO_GAMING_0, "content/gaming/a_220_00.mp3", PIPELINE_OK,
                  PIPELINE_OK, kNullHash, "5e60018ab1b30a82d53dccb759ab25b2");
@@ -282,6 +279,43 @@ FFMPEG_TEST_CASE(AUDIO_GAMING_18, "content/gaming/rocket_launcher.mp3",
                  PIPELINE_OK, PIPELINE_OK, kNullHash,
                  "05289a50acd15fa1357fe6234f82c3fe");
 
+// Allocate gigabytes of memory, likely can't be run on 32bit machines.
+FFMPEG_TEST_CASE(BIG_MEM_1, "security/bigmem1.mov",
+                 DEMUXER_ERROR_COULD_NOT_OPEN, DEMUXER_ERROR_COULD_NOT_OPEN,
+                 kNullHash, kNullHash);
+FFMPEG_TEST_CASE(BIG_MEM_2, "security/looping1.mov",
+                 DEMUXER_ERROR_COULD_NOT_PARSE, DEMUXER_ERROR_COULD_NOT_PARSE,
+                 kNullHash, kNullHash);
+FFMPEG_TEST_CASE(BIG_MEM_5, "security/looping5.mov",
+                 DEMUXER_ERROR_COULD_NOT_PARSE, DEMUXER_ERROR_COULD_NOT_PARSE,
+                 kNullHash, kNullHash);
+
+// Flaky under threading or for other reasons.  Per rbultje, most of these will
+// never be reliable since FFmpeg does not guarantee consistency in error cases.
+// We only really care that these don't cause crashes or errors under tooling.
+FLAKY_FFMPEG_TEST_CASE(Cr99652, "security/99652.webm");
+FLAKY_FFMPEG_TEST_CASE(Cr100464, "security/100464.webm");
+FLAKY_FFMPEG_TEST_CASE(Cr111342, "security/111342.ogm");
+FLAKY_FFMPEG_TEST_CASE(OGV_0, "security/big_dims.ogv");
+FLAKY_FFMPEG_TEST_CASE(OGV_3, "security/smclock_1_0.ogv");
+FLAKY_FFMPEG_TEST_CASE(OGV_4, "security/smclock.ogv.1.0.ogv");
+FLAKY_FFMPEG_TEST_CASE(OGV_6, "security/smclocktheora_1_10000.ogv");
+FLAKY_FFMPEG_TEST_CASE(OGV_13, "security/smclocktheora_1_790.ogv");
+FLAKY_FFMPEG_TEST_CASE(MP4_3, "security/clockh264aac_300413969.mp4");
+FLAKY_FFMPEG_TEST_CASE(MP4_4, "security/clockh264aac_301350139.mp4");
+FLAKY_FFMPEG_TEST_CASE(MP4_12, "security/assert1.mov");
+FLAKY_FFMPEG_TEST_CASE(BIG_MEM_3, "security/looping3.mov");
+FLAKY_FFMPEG_TEST_CASE(BIG_MEM_4, "security/looping4.mov");
+
+// Videos with massive gaps between frame timestamps that result in long hangs
+// with our pipeline.  Should be uncommented when we support clockless playback.
+// FFMPEG_TEST_CASE(WEBM_0, "security/memcpy.webm", PIPELINE_OK, PIPELINE_OK,
+//                  kNullHash, kNullHash);
+// FFMPEG_TEST_CASE(MP4_17, "security/assert2.mov", PIPELINE_OK, PIPELINE_OK,
+//                  kNullHash, kNullHash);
+// FFMPEG_TEST_CASE(OGV_23, "security/assert2.ogv", PIPELINE_OK, PIPELINE_OK,
+//                  kNullHash, kNullHash);
+
 TEST_P(FFmpegRegressionTest, BasicPlayback) {
   if (GetParam().init_status == PIPELINE_OK) {
     ASSERT_TRUE(Start(GetTestDataURL(GetParam().filename),
@@ -306,4 +340,11 @@ TEST_P(FFmpegRegressionTest, BasicPlayback) {
   }
 }
 
+TEST_P(FlakyFFmpegRegressionTest, BasicPlayback) {
+  if (Start(GetTestDataURL(GetParam().filename))) {
+    Play();
+    WaitUntilEndedOrError();
+  }
+}
+
 }  // namespace media
diff --git a/media/filters/pipeline_integration_test_base.cc b/media/filters/pipeline_integration_test_base.cc
index e995565..e6a58f5 100644
--- a/media/filters/pipeline_integration_test_base.cc
+++ b/media/filters/pipeline_integration_test_base.cc
@@ -38,16 +38,21 @@ PipelineIntegrationTestBase::~PipelineIntegrationTestBase() {
 }
 
 void PipelineIntegrationTestBase::OnStatusCallback(
-    PipelineStatus expected_status,
     PipelineStatus status) {
-  EXPECT_EQ(status, expected_status);
   pipeline_status_ = status;
   message_loop_.PostTask(FROM_HERE, MessageLoop::QuitClosure());
 }
 
+void PipelineIntegrationTestBase::OnStatusCallbackChecked(
+    PipelineStatus expected_status,
+    PipelineStatus status) {
+  EXPECT_EQ(status, expected_status);
+  OnStatusCallback(status);
+}
+
 PipelineStatusCB PipelineIntegrationTestBase::QuitOnStatusCB(
     PipelineStatus expected_status) {
-  return base::Bind(&PipelineIntegrationTestBase::OnStatusCallback,
+  return base::Bind(&PipelineIntegrationTestBase::OnStatusCallbackChecked,
                     base::Unretained(this),
                     expected_status);
 }
@@ -69,6 +74,8 @@ bool PipelineIntegrationTestBase::WaitUntilOnEnded() {
 }
 
 PipelineStatus PipelineIntegrationTestBase::WaitUntilEndedOrError() {
+  if (ended_ || pipeline_status_ != PIPELINE_OK)
+    return pipeline_status_;
   message_loop_.Run();
   return pipeline_status_;
 }
@@ -98,6 +105,18 @@ bool PipelineIntegrationTestBase::Start(const std::string& url,
   return Start(url, expected_status);
 }
 
+bool PipelineIntegrationTestBase::Start(const std::string& url) {
+  pipeline_->Start(
+      CreateFilterCollection(url),
+      base::Bind(&PipelineIntegrationTestBase::OnEnded, base::Unretained(this)),
+      base::Bind(&PipelineIntegrationTestBase::OnError, base::Unretained(this)),
+      NetworkEventCB(),
+      base::Bind(&PipelineIntegrationTestBase::OnStatusCallback,
+                 base::Unretained(this)));
+  message_loop_.Run();
+  return (pipeline_status_ == PIPELINE_OK);
+}
+
 void PipelineIntegrationTestBase::Play() {
   pipeline_->SetPlaybackRate(1);
 }
diff --git a/media/filters/pipeline_integration_test_base.h b/media/filters/pipeline_integration_test_base.h
index a617211..4fcb33b 100644
--- a/media/filters/pipeline_integration_test_base.h
+++ b/media/filters/pipeline_integration_test_base.h
@@ -44,6 +44,10 @@ class PipelineIntegrationTestBase {
   // audio underflow will be disabled to ensure consistent hashes.
   bool Start(const std::string& url, PipelineStatus expected_status,
              bool hashing_enabled);
+  // Initialize the pipeline and ignore any status updates.  Useful for testing
+  // invalid audio/video clips which don't have deterministic results.
+  bool Start(const std::string& url);
+
   void Play();
   void Pause();
   bool Seek(base::TimeDelta seek_time);
@@ -77,8 +81,9 @@ class PipelineIntegrationTestBase {
   bool ended_;
   PipelineStatus pipeline_status_;
 
-  void OnStatusCallback(PipelineStatus expected_status,
-                        PipelineStatus status);
+  void OnStatusCallbackChecked(PipelineStatus expected_status,
+                               PipelineStatus status);
+  void OnStatusCallback(PipelineStatus status);
   PipelineStatusCB QuitOnStatusCB(PipelineStatus expected_status);
   void OnEnded(PipelineStatus status);
   void OnError(PipelineStatus status);