On Mac, check the default resolution and aspect ratio to avoid distorted HD.

This CL introduces a heuristic that seems to avoid distorted HD on some
common consumer webcams.  The heuristic uses QTKit's default resolution
and reported pixel aspect ratio (if available) to determine whether
HD video is likely to be distorted on this capture device.

With this change, video capture begins immediately during Allocate(), but
OnFrameInfo doesn't fire until the resolution has been checked, and possibly
altered.  The camera is first opened at its default resolution.  If this is
less than HD, then HD capture is disabled.  Otherwise, if a returned HD stream
has nonsquare PAR, then HD capture is also disabled.  OnFrameInfo is only
fired once the camera has reached its final resolution.

BUG=234803

Review URL: https://chromiumcodereview.appspot.com/23889016

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@223663 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 145 insertions, 41 deletions

diff --git a/media/video/capture/mac/video_capture_device_mac.h b/media/video/capture/mac/video_capture_device_mac.h
index bb43c4d..1284337 100644
--- a/media/video/capture/mac/video_capture_device_mac.h
+++ b/media/video/capture/mac/video_capture_device_mac.h
@@ -39,12 +39,14 @@ class VideoCaptureDeviceMac : public VideoCaptureDevice1 {
 
   // Called to deliver captured video frames.
   void ReceiveFrame(const uint8* video_frame, int video_frame_length,
-                    const VideoCaptureCapability& frame_info);
+                    const VideoCaptureCapability& frame_info,
+                    int aspect_numerator, int aspect_denominator);
 
   void ReceiveError(const std::string& reason);
 
  private:
   void SetErrorState(const std::string& reason);
+  bool UpdateCaptureResolution();
 
   // Flag indicating the internal state.
   enum InternalState {
@@ -58,6 +60,9 @@ class VideoCaptureDeviceMac : public VideoCaptureDevice1 {
   Name device_name_;
   VideoCaptureDevice::EventHandler* observer_;
 
+  VideoCaptureCapability current_settings_;
+  bool sent_frame_info_ = false;
+
   // Only read and write state_ from inside this loop.
   const scoped_refptr<base::MessageLoopProxy> loop_proxy_;
   InternalState state_;
diff --git a/media/video/capture/mac/video_capture_device_mac.mm b/media/video/capture/mac/video_capture_device_mac.mm
index 7614fd3..cb7447b 100644
--- a/media/video/capture/mac/video_capture_device_mac.mm
+++ b/media/video/capture/mac/video_capture_device_mac.mm
@@ -25,10 +25,14 @@ struct Resolution {
   int height;
 };
 
+const Resolution kQVGA = { 320, 240 },
+                 kVGA = { 640, 480 },
+                 kHD = { 1280, 720 };
+
 const Resolution kWellSupportedResolutions[] = {
-   { 320, 240 },
-   { 640, 480 },
-   { 1280, 720 },
+   kQVGA,
+   kVGA,
+   kHD,
 };
 
 // TODO(ronghuawu): Replace this with CapabilityList::GetBestMatchedCapability.
@@ -140,33 +144,46 @@ void VideoCaptureDeviceMac::Allocate(
   else if (frame_rate > kMaxFrameRate)
     frame_rate = kMaxFrameRate;
 
-  if (![capture_device_ setCaptureHeight:height
-                                   width:width
-                               frameRate:frame_rate]) {
-    SetErrorState("Could not configure capture device.");
+  current_settings_.color = PIXEL_FORMAT_UYVY;
+  current_settings_.width = width;
+  current_settings_.height = height;
+  current_settings_.frame_rate = frame_rate;
+  current_settings_.expected_capture_delay = 0;
+  current_settings_.interlaced = false;
+
+  if (width != kHD.width || height != kHD.height) {
+    // If the resolution is VGA or QVGA, set the capture resolution to the
+    // target size.  For most cameras (though not all), at these resolutions
+    // QTKit produces frames with square pixels.
+    if (!UpdateCaptureResolution())
+      return;
+
+    sent_frame_info_ = true;
+    observer_->OnFrameInfo(current_settings_);
+  }
+
+  // If the resolution is HD, start capturing without setting a resolution.
+  // QTKit will produce frames at the native resolution, allowing us to
+  // identify cameras whose native resolution is too low for HD.  This
+  // additional information comes at a cost in startup latency, because the
+  // webcam will need to be reopened if its default resolution is not HD or VGA.
+
+  if (![capture_device_ startCapture]) {
+    SetErrorState("Could not start capture device.");
     return;
   }
 
   state_ = kAllocated;
-  VideoCaptureCapability current_settings;
-  current_settings.color = PIXEL_FORMAT_UYVY;
-  current_settings.width = width;
-  current_settings.height = height;
-  current_settings.frame_rate = frame_rate;
-  current_settings.expected_capture_delay = 0;
-  current_settings.interlaced = false;
-
-  observer_->OnFrameInfo(current_settings);
 }
 
 void VideoCaptureDeviceMac::Start() {
   DCHECK_EQ(loop_proxy_, base::MessageLoopProxy::current());
   DCHECK_EQ(state_, kAllocated);
-  if (![capture_device_ startCapture]) {
-    SetErrorState("Could not start capture device.");
-    return;
-  }
   state_ = kCapturing;
+
+  // This method no longer has any effect.  Capturing is triggered by
+  // the call to Allocate.
+  // TODO(bemasc, ncarter): Remove this method.
 }
 
 void VideoCaptureDeviceMac::Stop() {
@@ -216,9 +233,52 @@ bool VideoCaptureDeviceMac::Init() {
 void VideoCaptureDeviceMac::ReceiveFrame(
     const uint8* video_frame,
     int video_frame_length,
-    const VideoCaptureCapability& frame_info) {
+    const VideoCaptureCapability& frame_info,
+    int aspect_numerator,
+    int aspect_denominator) {
   // This method is safe to call from a device capture thread,
   // i.e. any thread controlled by QTKit.
+
+  if (!sent_frame_info_) {
+    if (current_settings_.width == kHD.width &&
+        current_settings_.height == kHD.height) {
+      bool changeToVga = false;
+      if (frame_info.width < kHD.width || frame_info.height < kHD.height) {
+        // These are the default capture settings, not yet configured to match
+        // |current_settings_|.
+        DCHECK(frame_info.frame_rate == 0);
+        DVLOG(1) << "Switching to VGA because the default resolution is " <<
+            frame_info.width << "x" << frame_info.height;
+        changeToVga = true;
+      }
+      if (frame_info.width == kHD.width && frame_info.height == kHD.height &&
+          aspect_numerator != aspect_denominator) {
+        DVLOG(1) << "Switching to VGA because HD has nonsquare pixel " <<
+            "aspect ratio " << aspect_numerator << ":" << aspect_denominator;
+        changeToVga = true;
+      }
+
+      if (changeToVga) {
+        current_settings_.width = kVGA.width;
+        current_settings_.height = kVGA.height;
+      }
+    }
+
+    if (current_settings_.width == frame_info.width &&
+        current_settings_.height == frame_info.height) {
+      sent_frame_info_ = true;
+      observer_->OnFrameInfo(current_settings_);
+    } else {
+      UpdateCaptureResolution();
+      // The current frame does not have the right width and height, so it
+      // must not be passed to |observer_|.
+      return;
+    }
+  }
+
+  DCHECK(current_settings_.width == frame_info.width &&
+         current_settings_.height == frame_info.height);
+
   observer_->OnIncomingCapturedFrame(
       video_frame, video_frame_length, base::Time::Now(), 0, false, false);
 }
@@ -236,4 +296,14 @@ void VideoCaptureDeviceMac::SetErrorState(const std::string& reason) {
   observer_->OnError();
 }
 
+bool VideoCaptureDeviceMac::UpdateCaptureResolution() {
+ if (![capture_device_ setCaptureHeight:current_settings_.height
+                                  width:current_settings_.width
+                              frameRate:current_settings_.frame_rate]) {
+   ReceiveError("Could not configure capture device.");
+   return false;
+ }
+ return true;
+}
+
 } // namespace media
diff --git a/media/video/capture/mac/video_capture_device_qtkit_mac.h b/media/video/capture/mac/video_capture_device_qtkit_mac.h
index a2d6f26..d032ef0 100644
--- a/media/video/capture/mac/video_capture_device_qtkit_mac.h
+++ b/media/video/capture/mac/video_capture_device_qtkit_mac.h
@@ -23,8 +23,6 @@ namespace media {
  @private
   // Settings.
   int frameRate_;
-  int frameWidth_;
-  int frameHeight_;
 
   NSLock *lock_;
   media::VideoCaptureDeviceMac *frameReceiver_;
diff --git a/media/video/capture/mac/video_capture_device_qtkit_mac.mm b/media/video/capture/mac/video_capture_device_qtkit_mac.mm
index 9f9ea1c..2b7e28e 100644
--- a/media/video/capture/mac/video_capture_device_qtkit_mac.mm
+++ b/media/video/capture/mac/video_capture_device_qtkit_mac.mm
@@ -108,6 +108,13 @@
     // particular crash.
     base::debug::SetCrashKeyValue("VideoCaptureDeviceQTKit", "OpenedDevice");
 
+    // Set the video pixel format to 2VUY (a.k.a UYVY, packed 4:2:2).
+    NSDictionary *captureDictionary = [NSDictionary
+        dictionaryWithObject:
+            [NSNumber numberWithUnsignedInt:kCVPixelFormatType_422YpCbCr8]
+                      forKey:(id)kCVPixelBufferPixelFormatTypeKey];
+    [captureDecompressedOutput setPixelBufferAttributes:captureDictionary];
+
     return YES;
   } else {
     // Remove the previously set capture device.
@@ -160,25 +167,29 @@
     return NO;
   }
 
-  frameWidth_ = width;
-  frameHeight_ = height;
   frameRate_ = frameRate;
 
+  QTCaptureDecompressedVideoOutput *output =
+      [[captureSession_ outputs] objectAtIndex:0];
+
+  // The old capture dictionary is used to retrieve the initial pixel
+  // format, which must be maintained.
+  NSDictionary *oldCaptureDictionary = [output pixelBufferAttributes];
+
   // Set up desired output properties.
   NSDictionary *captureDictionary =
       [NSDictionary dictionaryWithObjectsAndKeys:
-          [NSNumber numberWithDouble:frameWidth_],
+          [NSNumber numberWithDouble:width],
           (id)kCVPixelBufferWidthKey,
-          [NSNumber numberWithDouble:frameHeight_],
+          [NSNumber numberWithDouble:height],
           (id)kCVPixelBufferHeightKey,
-          [NSNumber numberWithUnsignedInt:kCVPixelFormatType_422YpCbCr8],
+          [oldCaptureDictionary
+              valueForKey:(id)kCVPixelBufferPixelFormatTypeKey],
           (id)kCVPixelBufferPixelFormatTypeKey,
           nil];
-  [[[captureSession_ outputs] objectAtIndex:0]
-      setPixelBufferAttributes:captureDictionary];
+  [output setPixelBufferAttributes:captureDictionary];
 
-  [[[captureSession_ outputs] objectAtIndex:0]
-      setMinimumVideoFrameInterval:(NSTimeInterval)1/(float)frameRate];
+  [output setMinimumVideoFrameInterval:(NSTimeInterval)1/(float)frameRate];
   return YES;
 }
 
@@ -232,10 +243,11 @@
       == kCVReturnSuccess) {
     void *baseAddress = CVPixelBufferGetBaseAddress(videoFrame);
     size_t bytesPerRow = CVPixelBufferGetBytesPerRow(videoFrame);
-    int frameHeight = CVPixelBufferGetHeight(videoFrame);
-    int frameSize = bytesPerRow * frameHeight;
+    size_t frameWidth = CVPixelBufferGetWidth(videoFrame);
+    size_t frameHeight = CVPixelBufferGetHeight(videoFrame);
+    size_t frameSize = bytesPerRow * frameHeight;
 
-    // TODO(shess): bytesPerRow may not correspond to frameWidth_*4,
+    // TODO(shess): bytesPerRow may not correspond to frameWidth_*2,
     // but VideoCaptureController::OnIncomingCapturedFrame() requires
     // it to do so.  Plumbing things through is intrusive, for now
     // just deliver an adjusted buffer.
@@ -243,7 +255,8 @@
     // VideoCaptureController::OnIncomingCapturedVideoFrame, which supports
     // pitches.
     UInt8* addressToPass = static_cast<UInt8*>(baseAddress);
-    size_t expectedBytesPerRow = frameWidth_ * 4;
+    // UYVY is 2 bytes per pixel.
+    size_t expectedBytesPerRow = frameWidth * 2;
     if (bytesPerRow > expectedBytesPerRow) {
       // TODO(shess): frameHeight and frameHeight_ are not the same,
       // try to do what the surrounding code seems to assume.
@@ -253,7 +266,7 @@
       // std::vector is contiguous according to standard.
       UInt8* adjustedAddress = &adjustedFrame_[0];
 
-      for (int y = 0; y < frameHeight; ++y) {
+      for (size_t y = 0; y < frameHeight; ++y) {
         memcpy(adjustedAddress + y * expectedBytesPerRow,
                addressToPass + y * bytesPerRow,
                expectedBytesPerRow);
@@ -263,15 +276,33 @@
       frameSize = frameHeight * expectedBytesPerRow;
     }
     media::VideoCaptureCapability captureCapability;
-    captureCapability.width = frameWidth_;
-    captureCapability.height = frameHeight_;
+    captureCapability.width = frameWidth;
+    captureCapability.height = frameHeight;
     captureCapability.frame_rate = frameRate_;
     captureCapability.color = media::PIXEL_FORMAT_UYVY;
     captureCapability.expected_capture_delay = 0;
     captureCapability.interlaced = false;
 
+    // The aspect ratio dictionary is often missing, in which case we report
+    // a pixel aspect ratio of 0:0.
+    int aspectNumerator = 0, aspectDenominator = 0;
+    CFDictionaryRef aspectRatioDict = (CFDictionaryRef)CVBufferGetAttachment(
+        videoFrame, kCVImageBufferPixelAspectRatioKey, NULL);
+    if (aspectRatioDict) {
+      CFNumberRef aspectNumeratorRef = (CFNumberRef)CFDictionaryGetValue(
+          aspectRatioDict, kCVImageBufferPixelAspectRatioHorizontalSpacingKey);
+      CFNumberRef aspectDenominatorRef = (CFNumberRef)CFDictionaryGetValue(
+          aspectRatioDict, kCVImageBufferPixelAspectRatioVerticalSpacingKey);
+      DCHECK(aspectNumeratorRef && aspectDenominatorRef) <<
+          "Aspect Ratio dictionary missing its entries.";
+      CFNumberGetValue(aspectNumeratorRef, kCFNumberIntType, &aspectNumerator);
+      CFNumberGetValue(
+          aspectDenominatorRef, kCFNumberIntType, &aspectDenominator);
+    }
+
     // Deliver the captured video frame.
-    frameReceiver_->ReceiveFrame(addressToPass, frameSize, captureCapability);
+    frameReceiver_->ReceiveFrame(addressToPass, frameSize, captureCapability,
+        aspectNumerator, aspectDenominator);
 
     CVPixelBufferUnlockBaseAddress(videoFrame, kLockFlags);
   }