Attachment 349197 Details for Bug 189427 – Patch for landing

[patch] Patch for landing

bug-189427-20180907144822.patch (text/plain), 9.29 KB, created by youenn fablet on 2018-09-07 14:48:23 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: youenn fablet

Created: 2018-09-07 14:48:23 PDT

Size: 9.29 KB

patch

obsolete

>Subversion Revision: 235788
>diff --git a/Source/WebCore/ChangeLog b/Source/WebCore/ChangeLog
>index 71e1bdc3c7c5212cca4a0c8818675762a298ad36..f8f0b4815324d70e9f224193326abeb802361320 100644
>--- a/Source/WebCore/ChangeLog
>+++ b/Source/WebCore/ChangeLog
>@@ -1,3 +1,24 @@
>+2018-09-07  Youenn Fablet  <youenn@apple.com>
>+
>+        RealtimeOutgoingVideoSourceCocoa should use VTImageRotationSession to rotate CVPixelBuffers
>+        https://bugs.webkit.org/show_bug.cgi?id=189427
>+
>+        Reviewed by Eric Carlson.
>+
>+        Previously, we were relying on libwebrtc utils to do the rotation.
>+        This is inefficient compared to VTImageRotateSession and also induces additional memory cost
>+        since libwebrtc is rotating using its own buffers and the encoder will convert this buffer back to a CVPixelBuffer.
>+
>+        Instead use VTImageRotationSession when rotation must be done at sending side.
>+        Covered by webrtc/video-rotation.html.
>+
>+        * platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp:
>+        (WebCore::RealtimeOutgoingVideoSourceCocoa::sampleBufferUpdated):
>+        * platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h:
>+        * platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.mm:
>+        (WebCore::computeRotatedWidthAndHeight):
>+        (WebCore::RealtimeOutgoingVideoSourceCocoa::rotatePixelBuffer):
>+
> 2018-09-07  Zalan Bujtas  <zalan@apple.com>
> 
>         [LFC] Replace "computed" value with "used" value to match spec language
>diff --git a/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp b/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp
>index ed14c34c31d66c05ced595c656030e713f655787..bc8af39ec88bf1ba320a3bfb10627131fdb41064 100644
>--- a/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp
>+++ b/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp
>@@ -152,23 +152,14 @@ void RealtimeOutgoingVideoSourceCocoa::sampleBufferUpdated(MediaStreamTrackPriva
>     auto pixelBuffer = static_cast<CVPixelBufferRef>(CMSampleBufferGetImageBuffer(sample.platformSample().sample.cmSampleBuffer));
>     auto pixelFormatType = CVPixelBufferGetPixelFormatType(pixelBuffer);
> 
>-    rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer;
>-    RetainPtr<CVPixelBufferRef> convertedBuffer;
>-    if (pixelFormatType == kCVPixelFormatType_420YpCbCr8Planar || pixelFormatType == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
>-        buffer = webrtc::pixelBufferToFrame(pixelBuffer);
>-    else {
>+    RetainPtr<CVPixelBufferRef> convertedBuffer = pixelBuffer;
>+    if (pixelFormatType != kCVPixelFormatType_420YpCbCr8Planar && pixelFormatType != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
>         convertedBuffer = convertToYUV(pixelBuffer);
>-        buffer = webrtc::pixelBufferToFrame(convertedBuffer.get());
>-    }
> 
>-    if (m_shouldApplyRotation && m_currentRotation != webrtc::kVideoRotation_0) {
>-        // FIXME: We should make AVVideoCaptureSource handle the rotation whenever possible.
>-        // This implementation is inefficient, we should rotate on the CMSampleBuffer directly instead of doing this double allocation.
>-        auto rotatedBuffer = buffer->ToI420();
>-        ASSERT(rotatedBuffer);
>-        buffer = webrtc::I420Buffer::Rotate(*rotatedBuffer, m_currentRotation);
>-    }
>-    sendFrame(WTFMove(buffer));
>+    if (m_shouldApplyRotation && m_currentRotation != webrtc::kVideoRotation_0)
>+        convertedBuffer = rotatePixelBuffer(convertedBuffer.get(), m_currentRotation);
>+
>+    sendFrame(webrtc::pixelBufferToFrame(convertedBuffer.get()));
> }
> 
> 
>diff --git a/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h b/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h
>index 961d512d7b42a9dda13ee8de3b10246d2f4353fc..fc3aacd4a653103d56eeecc870523f15f8304fc6 100644
>--- a/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h
>+++ b/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h
>@@ -29,6 +29,10 @@
> 
> #include "RealtimeOutgoingVideoSource.h"
> #include "PixelBufferConformerCV.h"
>+#include <webrtc/api/video/video_rotation.h>
>+
>+typedef struct OpaqueVTImageRotationSession* VTImageRotationSessionRef;
>+typedef struct __CVPixelBufferPool* CVPixelBufferPoolRef;
> 
> namespace WebCore {
> 
>@@ -43,8 +47,15 @@ private:
>     void sampleBufferUpdated(MediaStreamTrackPrivate&, MediaSample&) final;
> 
>     RetainPtr<CVPixelBufferRef> convertToYUV(CVPixelBufferRef);
>+    RetainPtr<CVPixelBufferRef> rotatePixelBuffer(CVPixelBufferRef, webrtc::VideoRotation);
> 
>     std::unique_ptr<PixelBufferConformerCV> m_pixelBufferConformer;
>+    RetainPtr<VTImageRotationSessionRef> m_rotationSession;
>+    RetainPtr<CVPixelBufferPoolRef> m_rotationPool;
>+    webrtc::VideoRotation m_currentRotationAngle { webrtc::kVideoRotation_0 };
>+    size_t m_rotatedWidth { 0 };
>+    size_t m_rotatedHeight { 0 };
>+    OSType m_rotatedFormat;
> 
> #if !RELEASE_LOG_DISABLED
>     size_t m_numberOfFrames { 0 };
>diff --git a/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.mm b/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.mm
>index 881f5711fa01ff4db396162a76870d2e26bf476b..96a4585648d2e87e79dc43c04833ddf99eb39537 100644
>--- a/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.mm
>+++ b/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.mm
>@@ -28,9 +28,12 @@
> 
> #if USE(LIBWEBRTC)
> 
>+#import "Logging.h"
>+#import "MediaSample.h"
> #import "PixelBufferConformerCV.h"
> #import <pal/cf/CoreMediaSoftLink.h>
> #import "CoreVideoSoftLink.h"
>+#import "VideoToolboxSoftLink.h"
> 
> namespace WebCore {
> 
>@@ -45,6 +48,85 @@ RetainPtr<CVPixelBufferRef> RealtimeOutgoingVideoSourceCocoa::convertToYUV(CVPix
>     return m_pixelBufferConformer->convert(pixelBuffer);
> }
> 
>+static inline void computeRotatedWidthAndHeight(CVPixelBufferRef pixelBuffer, webrtc::VideoRotation rotation, size_t& width, size_t& height)
>+{
>+    switch (rotation) {
>+    case webrtc::kVideoRotation_0:
>+    case webrtc::kVideoRotation_180:
>+        width = CVPixelBufferGetWidth(pixelBuffer);
>+        height = CVPixelBufferGetHeight(pixelBuffer);
>+        return;
>+    case webrtc::kVideoRotation_90:
>+    case webrtc::kVideoRotation_270:
>+        width = CVPixelBufferGetHeight(pixelBuffer);
>+        height = CVPixelBufferGetWidth(pixelBuffer);
>+        return;
>+    }
>+}
>+
>+RetainPtr<CVPixelBufferRef> RealtimeOutgoingVideoSourceCocoa::rotatePixelBuffer(CVPixelBufferRef pixelBuffer, webrtc::VideoRotation rotation)
>+{
>+    ASSERT(rotation);
>+    if (!rotation)
>+        return pixelBuffer;
>+
>+    if (!m_rotationSession || rotation != m_currentRotation) {
>+        VTImageRotationSessionRef rawRotationSession = nullptr;
>+        auto status = VTImageRotationSessionCreate(kCFAllocatorDefault, rotation, &rawRotationSession);
>+        if (status != noErr) {
>+            RELEASE_LOG(MediaStream, "RealtimeOutgoingVideoSourceCocoa::rotatePixelBuffer failed creating a rotation session with error %d", status);
>+            return nullptr;
>+        }
>+
>+        m_rotationSession = adoptCF(rawRotationSession);
>+        m_currentRotation = rotation;
>+
>+        VTImageRotationSessionSetProperty(rawRotationSession, kVTImageRotationPropertyKey_EnableHighSpeedTransfer, kCFBooleanTrue);
>+    }
>+
>+    size_t rotatedWidth, rotatedHeight;
>+    computeRotatedWidthAndHeight(pixelBuffer, rotation, rotatedWidth, rotatedHeight);
>+    auto format = CVPixelBufferGetPixelFormatType(pixelBuffer);
>+    if (!m_rotationPool || rotatedWidth != m_rotatedWidth || rotatedHeight != m_rotatedHeight || format != m_rotatedFormat) {
>+        auto pixelAttributes = @{
>+            (__bridge NSString *)kCVPixelBufferWidthKey: @(rotatedWidth),
>+            (__bridge NSString *)kCVPixelBufferHeightKey: @(rotatedHeight),
>+            (__bridge NSString *)kCVPixelBufferPixelFormatTypeKey: @(format),
>+            (__bridge NSString *)kCVPixelBufferCGImageCompatibilityKey: @NO,
>+        };
>+
>+        CVPixelBufferPoolRef pool = nullptr;
>+        auto status = CVPixelBufferPoolCreate(kCFAllocatorDefault, nullptr, (__bridge CFDictionaryRef)pixelAttributes, &pool);
>+
>+        if (status != kCVReturnSuccess) {
>+            RELEASE_LOG(MediaStream, "RealtimeOutgoingVideoSourceCocoa::rotatePixelBuffer failed creating a pixel buffer pool with error %d", status);
>+            return nullptr;
>+        }
>+        m_rotationPool = adoptCF(pool);
>+
>+        m_rotatedWidth = rotatedWidth;
>+        m_rotatedHeight = rotatedHeight;
>+        m_rotatedFormat = format;
>+    }
>+
>+    CVPixelBufferRef rawRotatedBuffer = nullptr;
>+    auto status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, m_rotationPool.get(), &rawRotatedBuffer);
>+
>+    if (status != kCVReturnSuccess) {
>+        RELEASE_LOG(MediaStream, "RealtimeOutgoingVideoSourceCocoa::rotatePixelBuffer failed creating a pixel buffer with error %d", status);
>+        return nullptr;
>+    }
>+    RetainPtr<CVPixelBufferRef> rotatedBuffer = adoptCF(rawRotatedBuffer);
>+
>+    status = VTImageRotationSessionTransferImage(m_rotationSession.get(), pixelBuffer, rotatedBuffer.get());
>+
>+    if (status != noErr) {
>+        RELEASE_LOG(MediaStream, "RealtimeOutgoingVideoSourceCocoa::rotatePixelBuffer failed rotating with error %d", status);
>+        return nullptr;
>+    }
>+    return rotatedBuffer;
>+}
>+
> } // namespace WebCore
> 
> #endif // USE(LIBWEBRTC)

Actions: View | Formatted Diff | Diff

Attachments on bug 189427: 349177 | 349197