WebKit Bugzilla
New
Browse
Search+
Log In
×
Sign in with GitHub
or
Remember my login
Create Account
·
Forgot Password
Forgotten password account recovery
[patch]
Patch
bug-237706-20220318093633.patch (text/plain), 32.41 KB, created by
youenn fablet
on 2022-03-18 01:36:35 PDT
(
hide
)
Description:
Patch
Filename:
MIME Type:
Creator:
youenn fablet
Created:
2022-03-18 01:36:35 PDT
Size:
32.41 KB
patch
obsolete
>Subversion Revision: 291468 >diff --git a/Source/WebCore/ChangeLog b/Source/WebCore/ChangeLog >index 92b6e988db3f05c7e588861ceccfcabe4e28f99d..872778c81387da642aa43b8aa12382f12c5ecc41 100644 >--- a/Source/WebCore/ChangeLog >+++ b/Source/WebCore/ChangeLog >@@ -1,3 +1,29 @@ >+2022-03-18 Youenn Fablet <youenn@apple.com> >+ >+ Remove use of MediaSampleAVFObjC from WebRTC pipelines >+ https://bugs.webkit.org/show_bug.cgi?id=237706 >+ <rdar://problem/90425391> >+ >+ Reviewed by NOBODY (OOPS!). >+ >+ Replace MediaSampleAVFObjC by VideoFrameCV when handling CVPixelBuffers. >+ >+ Covered by existing tests. >+ >+ * platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h: >+ * platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm: >+ * platform/graphics/cv/ImageRotationSessionVT.mm: >+ * platform/graphics/cv/ImageTransferSessionVT.h: >+ * platform/graphics/cv/ImageTransferSessionVT.mm: >+ * platform/graphics/cv/VideoFrameCV.h: >+ * platform/graphics/cv/VideoFrameCV.mm: >+ * platform/mediastream/RealtimeVideoSource.cpp: >+ * platform/mediastream/RealtimeVideoSource.h: >+ * platform/mediastream/mac/AVVideoCaptureSource.mm: >+ * platform/mediastream/mac/MockRealtimeVideoSourceMac.mm: >+ * platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h: >+ * platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm: >+ > 2022-03-18 Youenn Fablet <youenn@apple.com> > > Remove MediaSample usage from canvas capture code >diff --git a/Source/WebKit/ChangeLog b/Source/WebKit/ChangeLog >index 68b9442c0959feef6307960b96161624bb61ab5d..3c3149d3d5e6b313ace9856d3083f2e65a5c8d07 100644 >--- a/Source/WebKit/ChangeLog >+++ b/Source/WebKit/ChangeLog >@@ -1,3 +1,16 @@ >+2022-03-18 Youenn Fablet <youenn@apple.com> >+ >+ Remove use of MediaSampleAVFObjC from WebRTC pipelines >+ https://bugs.webkit.org/show_bug.cgi?id=237706 >+ <rdar://problem/90425391> >+ >+ Reviewed by NOBODY (OOPS!). >+ >+ * GPUProcess/webrtc/LibWebRTCCodecsProxy.mm: >+ * GPUProcess/webrtc/RemoteMediaRecorder.cpp: >+ * GPUProcess/webrtc/RemoteSampleBufferDisplayLayer.cpp: >+ * UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp: >+ > 2022-03-18 Youenn Fablet <youenn@apple.com> > > Remove MediaSample usage from canvas capture code >diff --git a/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h b/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h >index 87fac8513f645cc0d125faaf436fc69cb00ef936..5dcfc02da1b8b3fc1646cb221f912f12bd1ed9f2 100644 >--- a/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h >+++ b/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h >@@ -44,11 +44,6 @@ public: > static Ref<MediaSampleAVFObjC> create(CMSampleBufferRef sample, uint64_t trackID) { return adoptRef(*new MediaSampleAVFObjC(sample, trackID)); } > static Ref<MediaSampleAVFObjC> create(CMSampleBufferRef sample, AtomString trackID) { return adoptRef(*new MediaSampleAVFObjC(sample, trackID)); } > static Ref<MediaSampleAVFObjC> create(CMSampleBufferRef sample, VideoRotation rotation = VideoRotation::None, bool mirrored = false) { return adoptRef(*new MediaSampleAVFObjC(sample, rotation, mirrored)); } >- static RefPtr<MediaSampleAVFObjC> createFromPixelBuffer(PixelBuffer&&); >- WEBCORE_EXPORT static RefPtr<MediaSampleAVFObjC> createFromPixelBuffer(RetainPtr<CVPixelBufferRef>&&, VideoRotation, bool mirrored, MediaTime presentationTime = { }, MediaTime decodingTime = { }); >- >- WEBCORE_EXPORT static void setAsDisplayImmediately(MediaSample&); >- static RetainPtr<CMSampleBufferRef> cloneSampleBufferAndSetAsDisplayImmediately(CMSampleBufferRef); > > WEBCORE_EXPORT RefPtr<JSC::Uint8ClampedArray> getRGBAImageData() const override; > >diff --git a/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm b/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm >index babc58d0244da8499d446be817a225404d3d04bc..a045870ed7e687d21f3379140b280466ea31f928 100644 >--- a/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm >+++ b/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm >@@ -75,58 +75,6 @@ MediaSampleAVFObjC::MediaSampleAVFObjC(CMSampleBufferRef sample, VideoRotation r > > MediaSampleAVFObjC::~MediaSampleAVFObjC() = default; > >-RefPtr<MediaSampleAVFObjC> MediaSampleAVFObjC::createFromPixelBuffer(PixelBuffer&& pixelBuffer) >-{ >- auto size = pixelBuffer.size(); >- auto width = size.width(); >- auto height = size.height(); >- >- auto data = pixelBuffer.takeData(); >- auto dataBaseAddress = data->data(); >- auto leakedData = &data.leakRef(); >- >- auto derefBuffer = [] (void* context, const void*) { >- static_cast<JSC::Uint8ClampedArray*>(context)->deref(); >- }; >- >- CVPixelBufferRef cvPixelBufferRaw = nullptr; >- auto status = CVPixelBufferCreateWithBytes(kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA, dataBaseAddress, width * 4, derefBuffer, leakedData, nullptr, &cvPixelBufferRaw); >- >- auto cvPixelBuffer = adoptCF(cvPixelBufferRaw); >- if (!cvPixelBuffer) { >- derefBuffer(leakedData, nullptr); >- return nullptr; >- } >- ASSERT_UNUSED(status, !status); >- return createFromPixelBuffer(WTFMove(cvPixelBuffer), VideoRotation::None, false); >-} >- >-RefPtr<MediaSampleAVFObjC> MediaSampleAVFObjC::createFromPixelBuffer(RetainPtr<CVPixelBufferRef>&& pixelBuffer, VideoRotation rotation, bool mirrored, MediaTime presentationTime, MediaTime decodingTime) >-{ >- CMVideoFormatDescriptionRef formatDescriptionRaw = nullptr; >- auto status = PAL::CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer.get(), &formatDescriptionRaw); >- if (status || !formatDescriptionRaw) { >- ASSERT_NOT_REACHED(); >- return nullptr; >- } >- auto formatDescription = adoptCF(formatDescriptionRaw); >- >- CMSampleTimingInfo sampleTimingInformation = { PAL::kCMTimeInvalid, PAL::toCMTime(presentationTime), PAL::toCMTime(decodingTime) }; >- CMSampleBufferRef sampleBufferRaw = nullptr; >- status = PAL::CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBuffer.get(), formatDescription.get(), &sampleTimingInformation, &sampleBufferRaw); >- if (status || !sampleBufferRaw) { >- ASSERT_NOT_REACHED(); >- return nullptr; >- } >- auto sampleBuffer = adoptCF(sampleBufferRaw); >- CFArrayRef attachmentsArray = PAL::CMSampleBufferGetSampleAttachmentsArray(sampleBuffer.get(), true); >- for (CFIndex i = 0, count = CFArrayGetCount(attachmentsArray); i < count; ++i) { >- CFMutableDictionaryRef attachments = checked_cf_cast<CFMutableDictionaryRef>(CFArrayGetValueAtIndex(attachmentsArray, i)); >- CFDictionarySetValue(attachments, PAL::kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue); >- } >- return create(sampleBuffer.get(), rotation, mirrored); >-} >- > MediaTime MediaSampleAVFObjC::presentationTime() const > { > auto timeStamp = PAL::CMSampleBufferGetOutputPresentationTimeStamp(m_sample.get()); >@@ -397,11 +345,6 @@ static inline void setSampleBufferAsDisplayImmediately(CMSampleBufferRef sampleB > } > } > >-void MediaSampleAVFObjC::setAsDisplayImmediately(MediaSample& sample) >-{ >- setSampleBufferAsDisplayImmediately(sample.platformSample().sample.cmSampleBuffer); >-} >- > bool MediaSampleAVFObjC::isHomogeneous() const > { > CFArrayRef attachmentsArray = PAL::CMSampleBufferGetSampleAttachmentsArray(m_sample.get(), true); >@@ -472,39 +415,6 @@ Vector<Ref<MediaSampleAVFObjC>> MediaSampleAVFObjC::divideIntoHomogeneousSamples > return samples; > } > >-RetainPtr<CMSampleBufferRef> MediaSampleAVFObjC::cloneSampleBufferAndSetAsDisplayImmediately(CMSampleBufferRef sample) >-{ >- auto pixelBuffer = static_cast<CVImageBufferRef>(PAL::CMSampleBufferGetImageBuffer(sample)); >- if (!pixelBuffer) >- return nullptr; >- >- CMVideoFormatDescriptionRef formatDescription = nullptr; >- auto status = PAL::CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &formatDescription); >- if (status) >- return nullptr; >- auto retainedFormatDescription = adoptCF(formatDescription); >- >- CMItemCount itemCount = 0; >- status = PAL::CMSampleBufferGetSampleTimingInfoArray(sample, 0, nullptr, &itemCount); >- if (status) >- return nullptr; >- >- Vector<CMSampleTimingInfo> timingInfoArray; >- timingInfoArray.grow(itemCount); >- status = PAL::CMSampleBufferGetSampleTimingInfoArray(sample, itemCount, timingInfoArray.data(), nullptr); >- if (status) >- return nullptr; >- >- CMSampleBufferRef newSampleBuffer; >- status = PAL::CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBuffer, formatDescription, timingInfoArray.data(), &newSampleBuffer); >- if (status) >- return nullptr; >- >- setSampleBufferAsDisplayImmediately(newSampleBuffer); >- >- return adoptCF(newSampleBuffer); >-} >- > CVPixelBufferRef MediaSampleAVFObjC::pixelBuffer() const > { > return static_cast<CVPixelBufferRef>(PAL::CMSampleBufferGetImageBuffer(m_sample.get())); >diff --git a/Source/WebCore/platform/graphics/cv/ImageRotationSessionVT.mm b/Source/WebCore/platform/graphics/cv/ImageRotationSessionVT.mm >index 626cd83861eaec73cc8cc9c855fcc18958c6d11c..3eab0689e90451ad458cef389694ff9e69b06f76 100644 >--- a/Source/WebCore/platform/graphics/cv/ImageRotationSessionVT.mm >+++ b/Source/WebCore/platform/graphics/cv/ImageRotationSessionVT.mm >@@ -127,9 +127,9 @@ RetainPtr<CVPixelBufferRef> ImageRotationSessionVT::rotate(CVPixelBufferRef pixe > return result; > } > >-RetainPtr<CVPixelBufferRef> ImageRotationSessionVT::rotate(MediaSample& sample, const RotationProperties& rotation, IsCGImageCompatible cgImageCompatible) >+RetainPtr<CVPixelBufferRef> ImageRotationSessionVT::rotate(MediaSample& videoFrame, const RotationProperties& rotation, IsCGImageCompatible cgImageCompatible) > { >- auto pixelBuffer = static_cast<CVPixelBufferRef>(PAL::CMSampleBufferGetImageBuffer(sample.platformSample().sample.cmSampleBuffer)); >+ auto pixelBuffer = videoFrame.pixelBuffer(); > ASSERT(pixelBuffer); > if (!pixelBuffer) > return nullptr; >diff --git a/Source/WebCore/platform/graphics/cv/ImageTransferSessionVT.h b/Source/WebCore/platform/graphics/cv/ImageTransferSessionVT.h >index ee02adaadbdd757d6adb53c3443053cb1f3f31ab..ba8104d69814b4d81a2c59409e5e2dff7a3549c7 100644 >--- a/Source/WebCore/platform/graphics/cv/ImageTransferSessionVT.h >+++ b/Source/WebCore/platform/graphics/cv/ImageTransferSessionVT.h >@@ -26,7 +26,7 @@ > #pragma once > > #include "IntSize.h" >-#include "MediaSample.h" >+#include "VideoFrame.h" > #include <wtf/RetainPtr.h> > > typedef struct CGImage *CGImageRef; >@@ -45,7 +45,7 @@ public: > return std::unique_ptr<ImageTransferSessionVT>(new ImageTransferSessionVT(pixelFormat, shouldUseIOSurface)); > } > >- RefPtr<MediaSample> convertMediaSample(MediaSample&, const IntSize&); >+ RefPtr<MediaSample> convertVideoFrame(MediaSample&, const IntSize&); > RefPtr<MediaSample> createMediaSample(CGImageRef, const MediaTime&, const IntSize&, MediaSample::VideoRotation = MediaSample::VideoRotation::None, bool mirrored = false); > RefPtr<MediaSample> createMediaSample(CMSampleBufferRef, const MediaTime&, const IntSize&, MediaSample::VideoRotation = MediaSample::VideoRotation::None, bool mirrored = false); > >diff --git a/Source/WebCore/platform/graphics/cv/ImageTransferSessionVT.mm b/Source/WebCore/platform/graphics/cv/ImageTransferSessionVT.mm >index e3171d3537278fd3f1640fbc956fa01ca6a865c2..8b6d79455ab7332c343992814933c605f979cc6d 100644 >--- a/Source/WebCore/platform/graphics/cv/ImageTransferSessionVT.mm >+++ b/Source/WebCore/platform/graphics/cv/ImageTransferSessionVT.mm >@@ -29,9 +29,10 @@ > #import "CVUtilities.h" > #import "GraphicsContextCG.h" > #import "Logging.h" >-#import "MediaSampleAVFObjC.h" >+#import "VideoFrameCV.h" > #import <CoreMedia/CMFormatDescription.h> > #import <CoreMedia/CMSampleBuffer.h> >+#import <pal/avfoundation/MediaTimeAVFoundation.h> > > #if !PLATFORM(MACCATALYST) > #import <pal/spi/cocoa/IOSurfaceSPI.h> >@@ -254,18 +255,16 @@ RetainPtr<CMSampleBufferRef> ImageTransferSessionVT::createCMSampleBuffer(IOSurf > } > #endif > >-RefPtr<MediaSample> ImageTransferSessionVT::convertMediaSample(MediaSample& sample, const IntSize& size) >+RefPtr<MediaSample> ImageTransferSessionVT::convertVideoFrame(MediaSample& videoFrame, const IntSize& size) > { >- ASSERT(sample.platformSample().type == PlatformSample::CMSampleBufferType); >+ if (size == expandedIntSize(videoFrame.presentationSize())) >+ return &videoFrame; > >- if (size == expandedIntSize(sample.presentationSize())) >- return &sample; >- >- auto resizedBuffer = convertCMSampleBuffer(sample.platformSample().sample.cmSampleBuffer, size); >+ auto resizedBuffer = convertPixelBuffer(videoFrame.pixelBuffer(), size); > if (!resizedBuffer) > return nullptr; > >- return MediaSampleAVFObjC::create(resizedBuffer.get(), sample.videoRotation(), sample.videoMirrored()); >+ return VideoFrameCV::create(videoFrame.presentationTime(), videoFrame.videoMirrored(), videoFrame.videoRotation(), WTFMove(resizedBuffer)); > } > > #if !PLATFORM(MACCATALYST) >@@ -275,7 +274,7 @@ RefPtr<MediaSample> ImageTransferSessionVT::createMediaSample(IOSurfaceRef surfa > if (!sampleBuffer) > return nullptr; > >- return MediaSampleAVFObjC::create(sampleBuffer.get(), rotation, mirrored); >+ return VideoFrameCV::create(sampleBuffer.get(), mirrored, rotation); > } > #endif > >@@ -285,7 +284,7 @@ RefPtr<MediaSample> ImageTransferSessionVT::createMediaSample(CGImageRef image, > if (!sampleBuffer) > return nullptr; > >- return MediaSampleAVFObjC::create(sampleBuffer.get(), rotation, mirrored); >+ return VideoFrameCV::create(sampleBuffer.get(), mirrored, rotation); > } > > RefPtr<MediaSample> ImageTransferSessionVT::createMediaSample(CMSampleBufferRef buffer, const MediaTime& sampleTime, const IntSize& size, MediaSample::VideoRotation rotation, bool mirrored) >@@ -294,7 +293,7 @@ RefPtr<MediaSample> ImageTransferSessionVT::createMediaSample(CMSampleBufferRef > if (!sampleBuffer) > return nullptr; > >- return MediaSampleAVFObjC::create(sampleBuffer.get(), rotation, mirrored); >+ return VideoFrameCV::create(sampleBuffer.get(), mirrored, rotation); > } > > } // namespace WebCore >diff --git a/Source/WebCore/platform/graphics/cv/VideoFrameCV.h b/Source/WebCore/platform/graphics/cv/VideoFrameCV.h >index 15f3186dc767e7bd24b33d9efb018ed2b939009a..93a3e7157e61f5091052bf3f5f404cf092681c84 100644 >--- a/Source/WebCore/platform/graphics/cv/VideoFrameCV.h >+++ b/Source/WebCore/platform/graphics/cv/VideoFrameCV.h >@@ -40,6 +40,7 @@ class PixelBuffer; > class VideoFrameCV : public VideoFrame { > public: > WEBCORE_EXPORT static Ref<VideoFrameCV> create(MediaTime presentationTime, bool isMirrored, VideoRotation, RetainPtr<CVPixelBufferRef>&&); >+ WEBCORE_EXPORT static Ref<VideoFrameCV> create(CMSampleBufferRef, bool isMirrored, VideoRotation); > static RefPtr<VideoFrameCV> createFromPixelBuffer(PixelBuffer&&); > WEBCORE_EXPORT ~VideoFrameCV(); > >diff --git a/Source/WebCore/platform/graphics/cv/VideoFrameCV.mm b/Source/WebCore/platform/graphics/cv/VideoFrameCV.mm >index f20d385d86e2b11807311299915ff1708dcdb01b..abd7e2fc897b071bb0b5ff3976d99f0c19e1446c 100644 >--- a/Source/WebCore/platform/graphics/cv/VideoFrameCV.mm >+++ b/Source/WebCore/platform/graphics/cv/VideoFrameCV.mm >@@ -28,11 +28,22 @@ > > #if ENABLE(VIDEO) && USE(AVFOUNDATION) > #include "CVUtilities.h" >+#include "PixelBuffer.h" > #include "ProcessIdentity.h" > #include "CoreVideoSoftLink.h" > > namespace WebCore { > >+Ref<VideoFrameCV> VideoFrameCV::create(CMSampleBufferRef sampleBuffer, bool isMirrored, VideoRotation rotation) >+{ >+ auto pixelBuffer = static_cast<CVPixelBufferRef>(PAL::CMSampleBufferGetImageBuffer(sampleBuffer)); >+ auto timeStamp = PAL::CMSampleBufferGetOutputPresentationTimeStamp(sampleBuffer); >+ if (CMTIME_IS_INVALID(timeStamp)) >+ timeStamp = PAL::CMSampleBufferGetPresentationTimeStamp(sampleBuffer); >+ >+ return VideoFrameCV::create(PAL::toMediaTime(timeStamp), isMirrored, rotation, pixelBuffer); >+} >+ > Ref<VideoFrameCV> VideoFrameCV::create(MediaTime presentationTime, bool isMirrored, VideoRotation rotation, RetainPtr<CVPixelBufferRef>&& pixelBuffer) > { > ASSERT(pixelBuffer); >diff --git a/Source/WebCore/platform/mediastream/RealtimeVideoSource.cpp b/Source/WebCore/platform/mediastream/RealtimeVideoSource.cpp >index 5b85bde4d75f8ada1a20310afb80259a452fac81..a235d423f29104c59492854f0f757af07f924a28 100644 >--- a/Source/WebCore/platform/mediastream/RealtimeVideoSource.cpp >+++ b/Source/WebCore/platform/mediastream/RealtimeVideoSource.cpp >@@ -30,7 +30,7 @@ > > #if PLATFORM(COCOA) > #include "ImageTransferSessionVT.h" >-#include "MediaSampleAVFObjC.h" >+#include "VideoFrameCV.h" > #endif > > namespace WebCore { >@@ -172,31 +172,23 @@ void RealtimeVideoSource::sourceStopped() > } > > #if PLATFORM(COCOA) >-RefPtr<MediaSample> RealtimeVideoSource::adaptVideoSample(MediaSample& sample) >-{ >- if (sample.platformSample().type != PlatformSample::CMSampleBufferType) { >- // FIXME: Support more efficiently downsampling of remote video frames by downsampling in GPUProcess. >- auto newSample = MediaSampleAVFObjC::createFromPixelBuffer(sample.pixelBuffer(), sample.videoRotation(), sample.videoMirrored(), sample.presentationTime(), { }); >- if (!newSample) >- return nullptr; >- return adaptVideoSample(*newSample); >- } >- ASSERT(sample.platformSample().type == PlatformSample::CMSampleBufferType); >- if (!m_imageTransferSession || m_imageTransferSession->pixelFormat() != sample.videoPixelFormat()) >- m_imageTransferSession = ImageTransferSessionVT::create(sample.videoPixelFormat(), m_shouldUseIOSurface); >+RefPtr<MediaSample> RealtimeVideoSource::adaptVideoFrame(MediaSample& videoFrame) >+{ >+ if (!m_imageTransferSession || m_imageTransferSession->pixelFormat() != videoFrame.videoPixelFormat()) >+ m_imageTransferSession = ImageTransferSessionVT::create(videoFrame.videoPixelFormat(), m_shouldUseIOSurface); > > ASSERT(m_imageTransferSession); > if (!m_imageTransferSession) > return nullptr; > >- auto mediaSample = m_imageTransferSession->convertMediaSample(sample, size()); >- ASSERT(mediaSample); >+ auto newVideoFrame = m_imageTransferSession->convertVideoFrame(videoFrame, size()); >+ ASSERT(newVideoFrame); > >- return mediaSample; >+ return newVideoFrame; > } > #endif > >-void RealtimeVideoSource::videoSampleAvailable(MediaSample& sample, VideoFrameTimeMetadata metadata) >+void RealtimeVideoSource::videoSampleAvailable(MediaSample& videoFrame, VideoFrameTimeMetadata metadata) > { > if (m_frameDecimation > 1 && ++m_frameDecimationCounter % m_frameDecimation) > return; >@@ -208,15 +200,15 @@ void RealtimeVideoSource::videoSampleAvailable(MediaSample& sample, VideoFrameTi > > #if PLATFORM(COCOA) > auto size = this->size(); >- if (!size.isEmpty() && size != expandedIntSize(sample.presentationSize())) { >- if (auto mediaSample = adaptVideoSample(sample)) { >- RealtimeMediaSource::videoSampleAvailable(*mediaSample, metadata); >+ if (!size.isEmpty() && size != expandedIntSize(videoFrame.presentationSize())) { >+ if (auto newVideoFrame = adaptVideoFrame(videoFrame)) { >+ RealtimeMediaSource::videoSampleAvailable(*newVideoFrame, metadata); > return; > } > } > #endif > >- RealtimeMediaSource::videoSampleAvailable(sample, metadata); >+ RealtimeMediaSource::videoSampleAvailable(videoFrame, metadata); > } > > Ref<RealtimeMediaSource> RealtimeVideoSource::clone() >diff --git a/Source/WebCore/platform/mediastream/RealtimeVideoSource.h b/Source/WebCore/platform/mediastream/RealtimeVideoSource.h >index 1336802e089bd3bf1737e77be653514a4b27be90..a71949789e4e05a90a89fc1403bb8088cffbf8a8 100644 >--- a/Source/WebCore/platform/mediastream/RealtimeVideoSource.h >+++ b/Source/WebCore/platform/mediastream/RealtimeVideoSource.h >@@ -75,7 +75,7 @@ private: > void videoSampleAvailable(MediaSample&, VideoFrameTimeMetadata) final; > > #if PLATFORM(COCOA) >- RefPtr<MediaSample> adaptVideoSample(MediaSample&); >+ RefPtr<MediaSample> adaptVideoFrame(MediaSample&); > #endif > > #if !RELEASE_LOG_DISABLED >diff --git a/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm b/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm >index 330bfeb8476651b9a40508cc6c3e5078dae276e5..378c9af97d7525cdfc2855fe1840a96861adc17e 100644 >--- a/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm >+++ b/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm >@@ -33,18 +33,19 @@ > #import "IntRect.h" > #import "Logging.h" > #import "MediaConstraints.h" >-#import "MediaSampleAVFObjC.h" > #import "PlatformLayer.h" > #import "RealtimeMediaSourceCenter.h" > #import "RealtimeMediaSourceSettings.h" > #import "RealtimeVideoSource.h" > #import "RealtimeVideoUtilities.h" >+#import "VideoFrameCV.h" > #import <AVFoundation/AVCaptureDevice.h> > #import <AVFoundation/AVCaptureInput.h> > #import <AVFoundation/AVCaptureOutput.h> > #import <AVFoundation/AVCaptureSession.h> > #import <AVFoundation/AVError.h> > #import <objc/runtime.h> >+#import <pal/avfoundation/MediaTimeAVFoundation.h> > #import <pal/spi/cocoa/AVFoundationSPI.h> > > #import "CoreVideoSoftLink.h" >@@ -560,12 +561,12 @@ void AVVideoCaptureSource::captureOutputDidOutputSampleBufferFromConnection(AVCa > if (++m_framesCount <= framesToDropWhenStarting) > return; > >- auto sample = MediaSampleAVFObjC::create(sampleBuffer, m_sampleRotation, [captureConnection isVideoMirrored]); >- m_buffer = &sample.get(); >- setIntrinsicSize(expandedIntSize(sample->presentationSize())); >+ auto videoFrame = VideoFrameCV::create(sampleBuffer, [captureConnection isVideoMirrored], m_sampleRotation); >+ m_buffer = &videoFrame.get(); >+ setIntrinsicSize(expandedIntSize(videoFrame->presentationSize())); > VideoFrameTimeMetadata metadata; > metadata.captureTime = MonotonicTime::now().secondsSinceEpoch(); >- dispatchMediaSampleToObservers(WTFMove(sample), metadata); >+ dispatchMediaSampleToObservers(WTFMove(videoFrame), metadata); > } > > void AVVideoCaptureSource::captureSessionIsRunningDidChange(bool state) >diff --git a/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm b/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm >index a74227ab467fd29d2aa186a58664769043748f2d..0353258ab61f42da423c1657541c0870353bddd4 100644 >--- a/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm >+++ b/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm >@@ -36,7 +36,6 @@ > #import "ImageBuffer.h" > #import "ImageTransferSessionVT.h" > #import "MediaConstraints.h" >-#import "MediaSampleAVFObjC.h" > #import "MockRealtimeMediaSourceCenter.h" > #import "NotImplemented.h" > #import "PlatformLayer.h" >diff --git a/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h b/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h >index 8a0252962136a223ed342f7807725af848d1b8b5..92c2fec682dd6ae764fbb2904cac8fefa496734f 100644 >--- a/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h >+++ b/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h >@@ -29,10 +29,9 @@ > > #if USE(LIBWEBRTC) > >-#include "MediaSample.h" > #include "RealtimeIncomingVideoSource.h" >+#include "VideoFrame.h" > >-using CMSampleBufferRef = struct opaqueCMSampleBuffer*; > using CVPixelBufferPoolRef = struct __CVPixelBufferPool*; > using CVPixelBufferRef = struct __CVBuffer*; > >@@ -50,8 +49,8 @@ private: > RealtimeIncomingVideoSourceCocoa(rtc::scoped_refptr<webrtc::VideoTrackInterface>&&, String&&); > RetainPtr<CVPixelBufferRef> pixelBufferFromVideoFrame(const webrtc::VideoFrame&); > CVPixelBufferPoolRef pixelBufferPool(size_t width, size_t height, webrtc::BufferType); >- RefPtr<MediaSample> toVideoFrame(const webrtc::VideoFrame&, MediaSample::VideoRotation); >- RefPtr<MediaSample> createMediaSampleFromCVPixelBuffer(CVPixelBufferRef, MediaSample::VideoRotation, int64_t); >+ RefPtr<VideoFrame> toVideoFrame(const webrtc::VideoFrame&, MediaSample::VideoRotation); >+ Ref<VideoFrame> createVideoSampleFromCVPixelBuffer(CVPixelBufferRef, MediaSample::VideoRotation, int64_t); > > // rtc::VideoSinkInterface > void OnFrame(const webrtc::VideoFrame&) final; >diff --git a/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm b/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm >index 51854fb2b9c539797d5c767956a8253a95749172..ee2823c1ea7385d9a1840196c7dcbb6b31041b05 100644 >--- a/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm >+++ b/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm >@@ -32,7 +32,7 @@ > > #import "CVUtilities.h" > #import "Logging.h" >-#import "MediaSampleAVFObjC.h" >+#import "VideoFrameCV.h" > #import "VideoFrameLibWebRTC.h" > #import <wtf/cf/TypeCastsCF.h> > >@@ -91,33 +91,12 @@ CVPixelBufferPoolRef RealtimeIncomingVideoSourceCocoa::pixelBufferPool(size_t wi > return m_pixelBufferPool.get(); > } > >-RefPtr<MediaSample> RealtimeIncomingVideoSourceCocoa::createMediaSampleFromCVPixelBuffer(CVPixelBufferRef pixelBuffer, MediaSample::VideoRotation rotation, int64_t timeStamp) >+Ref<VideoFrame> RealtimeIncomingVideoSourceCocoa::createVideoSampleFromCVPixelBuffer(CVPixelBufferRef pixelBuffer, MediaSample::VideoRotation rotation, int64_t timeStamp) > { >- CMSampleTimingInfo timingInfo; >- timingInfo.presentationTimeStamp = PAL::CMTimeMake(timeStamp, 1000000); >- timingInfo.decodeTimeStamp = PAL::kCMTimeInvalid; >- timingInfo.duration = PAL::kCMTimeInvalid; >- >- CMVideoFormatDescriptionRef formatDescription; >- OSStatus ostatus = PAL::CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, (CVImageBufferRef)pixelBuffer, &formatDescription); >- if (ostatus != noErr) { >- ERROR_LOG_IF(loggerPtr(), LOGIDENTIFIER, "Failed to initialize CMVideoFormatDescription with error ", static_cast<int>(ostatus)); >- return nullptr; >- } >- >- CMSampleBufferRef sampleBuffer; >- ostatus = PAL::CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, (CVImageBufferRef)pixelBuffer, formatDescription, &timingInfo, &sampleBuffer); >- CFRelease(formatDescription); >- if (ostatus != noErr) { >- ERROR_LOG_IF(loggerPtr(), LOGIDENTIFIER, "Failed to create the sample buffer with error ", static_cast<int>(ostatus)); >- return nullptr; >- } >- >- auto sample = adoptCF(sampleBuffer); >- return MediaSampleAVFObjC::create(sample.get(), rotation); >+ return VideoFrameCV::create(MediaTime(timeStamp, 1000000), false, rotation, pixelBuffer); > } > >-RefPtr<MediaSample> RealtimeIncomingVideoSourceCocoa::toVideoFrame(const webrtc::VideoFrame& frame, MediaSample::VideoRotation rotation) >+RefPtr<VideoFrame> RealtimeIncomingVideoSourceCocoa::toVideoFrame(const webrtc::VideoFrame& frame, MediaSample::VideoRotation rotation) > { > if (muted()) { > if (!m_blackFrame || m_blackFrameWidth != frame.width() || m_blackFrameHeight != frame.height()) { >@@ -125,7 +104,7 @@ RefPtr<MediaSample> RealtimeIncomingVideoSourceCocoa::toVideoFrame(const webrtc: > m_blackFrameHeight = frame.height(); > m_blackFrame = createBlackPixelBuffer(m_blackFrameWidth, m_blackFrameHeight); > } >- return createMediaSampleFromCVPixelBuffer(m_blackFrame.get(), rotation, frame.timestamp_us()); >+ return createVideoSampleFromCVPixelBuffer(m_blackFrame.get(), rotation, frame.timestamp_us()); > } > > if (auto* provider = videoFrameBufferProvider(frame)) { >@@ -137,7 +116,7 @@ RefPtr<MediaSample> RealtimeIncomingVideoSourceCocoa::toVideoFrame(const webrtc: > > // If we already have a CVPixelBufferRef, use it directly. > if (auto pixelBuffer = webrtc::pixelBufferFromFrame(frame)) >- return createMediaSampleFromCVPixelBuffer(pixelBuffer, rotation, frame.timestamp_us()); >+ return createVideoSampleFromCVPixelBuffer(pixelBuffer, rotation, frame.timestamp_us()); > > // In case of in memory libwebrtc samples, we have non interleaved YUV data, let's lazily create CVPixelBuffers if needed. > return VideoFrameLibWebRTC::create(MediaTime(frame.timestamp_us(), 1000000), false, rotation, frame.video_frame_buffer(), [protectedThis = Ref { *this }, this](auto& buffer) { >diff --git a/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm b/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm >index d8a46d7907d6559820c843aa5204ebe63ca62e8c..d8a29226616b2647f11f8456029f3dedee23e214 100644 >--- a/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm >+++ b/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm >@@ -39,7 +39,7 @@ > #import "WebCoreArgumentCoders.h" > #import <WebCore/CVUtilities.h> > #import <WebCore/LibWebRTCProvider.h> >-#import <WebCore/MediaSampleAVFObjC.h> >+#import <WebCore/VideoFrameCV.h> > #import <webrtc/sdk/WebKit/WebKitDecoder.h> > #import <webrtc/sdk/WebKit/WebKitEncoder.h> > #import <wtf/BlockPtr.h> >@@ -97,13 +97,11 @@ auto LibWebRTCCodecsProxy::createDecoderCallback(RTCDecoderIdentifier identifier > if (useRemoteFrames) > videoFrameObjectHeap = m_videoFrameObjectHeap.ptr(); > return [identifier, connection = m_connection, resourceOwner = m_resourceOwner, videoFrameObjectHeap = WTFMove(videoFrameObjectHeap)] (CVPixelBufferRef pixelBuffer, uint32_t timeStampNs, uint32_t timeStamp) mutable { >- auto sample = WebCore::MediaSampleAVFObjC::createFromPixelBuffer(pixelBuffer, WebCore::MediaSample::VideoRotation::None, false, MediaTime(timeStampNs, 1), { }); >- if (!sample) >- return; >+ auto videoFrame = WebCore::VideoFrameCV::create(MediaTime(timeStampNs, 1), false, WebCore::MediaSample::VideoRotation::None, pixelBuffer); > if (resourceOwner) >- sample->setOwnershipIdentity(resourceOwner); >+ videoFrame->setOwnershipIdentity(resourceOwner); > if (videoFrameObjectHeap) { >- auto properties = videoFrameObjectHeap->add(sample.releaseNonNull()); >+ auto properties = videoFrameObjectHeap->add(WTFMove(videoFrame)); > connection->send(Messages::LibWebRTCCodecs::CompletedDecoding { identifier, timeStamp, timeStampNs, WTFMove(properties) }, 0); > return; > } >diff --git a/Source/WebKit/GPUProcess/webrtc/RemoteMediaRecorder.cpp b/Source/WebKit/GPUProcess/webrtc/RemoteMediaRecorder.cpp >index bd1c2d125ba9a771ec62a9955027d22c3f166697..52f618a386c30eaf6dbe0880164f4c9f32698b11 100644 >--- a/Source/WebKit/GPUProcess/webrtc/RemoteMediaRecorder.cpp >+++ b/Source/WebKit/GPUProcess/webrtc/RemoteMediaRecorder.cpp >@@ -33,7 +33,6 @@ > #include "RemoteVideoFrameObjectHeap.h" > #include "SharedRingBufferStorage.h" > #include <WebCore/CARingBuffer.h> >-#include <WebCore/MediaSampleAVFObjC.h> > #include <WebCore/WebAudioBufferList.h> > #include <wtf/CompletionHandler.h> > >diff --git a/Source/WebKit/GPUProcess/webrtc/RemoteSampleBufferDisplayLayer.cpp b/Source/WebKit/GPUProcess/webrtc/RemoteSampleBufferDisplayLayer.cpp >index 94026327d004522b5c98b7d1062e4155e09f9a81..8d04f2cc5a789a7a4e958740c7b68b5393e5ba4b 100644 >--- a/Source/WebKit/GPUProcess/webrtc/RemoteSampleBufferDisplayLayer.cpp >+++ b/Source/WebKit/GPUProcess/webrtc/RemoteSampleBufferDisplayLayer.cpp >@@ -33,7 +33,6 @@ > #include "SampleBufferDisplayLayerMessages.h" > #include <WebCore/ImageTransferSessionVT.h> > #include <WebCore/LocalSampleBufferDisplayLayer.h> >-#include <WebCore/MediaSampleAVFObjC.h> > > namespace WebKit { > >diff --git a/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp b/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp >index 3aa0084ec1fbc3e068c9b49da5e07533f61c183a..2d453e4da79c2d82d1a91d042f3451eb8eddc50b 100644 >--- a/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp >+++ b/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp >@@ -41,9 +41,9 @@ > #include <WebCore/CARingBuffer.h> > #include <WebCore/ImageRotationSessionVT.h> > #include <WebCore/MediaConstraints.h> >-#include <WebCore/MediaSampleAVFObjC.h> > #include <WebCore/RealtimeMediaSourceCenter.h> > #include <WebCore/RealtimeVideoSource.h> >+#include <WebCore/VideoFrameCV.h> > #include <WebCore/WebAudioBufferList.h> > #include <wtf/UniqueRef.h> > >@@ -182,7 +182,7 @@ private: > { > if (m_shouldApplyRotation && sample.videoRotation() != MediaSample::VideoRotation::None) { > auto pixelBuffer = rotatePixelBuffer(sample); >- return MediaSampleAVFObjC::createFromPixelBuffer(WTFMove(pixelBuffer), MediaSample::VideoRotation::None, sample.videoMirrored(), sample.presentationTime(), sample.decodeTime()); >+ return VideoFrameCV::create(sample.presentationTime(), sample.videoMirrored(), MediaSample::VideoRotation::None, WTFMove(pixelBuffer)); > } > return &sample; > }
<b>You cannot view the attachment while viewing its details because your browser does not support IFRAMEs. <a href="attachment.cgi?id=455078">View the attachment on a separate page</a>.</b>
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Formatted Diff
|
Diff
Attachments on
bug 237706
:
454683
| 455078