Attachment 357355 Details for Bug 192629 – Patch

[patch] Patch

bug-192629-20181214164145.patch (text/plain), 13.54 KB, created by Eric Carlson on 2018-12-14 16:41:46 PST

(hide)

Description:

Filename:

MIME Type:

Creator: Eric Carlson

Created: 2018-12-14 16:41:46 PST

Size: 13.54 KB

patch

obsolete

>Subversion Revision: 239240
>diff --git a/Source/WebCore/ChangeLog b/Source/WebCore/ChangeLog
>index 9a3a45602b1fc96f804b7f28a2efb2f96ed2dfd8..6616d752748260da9fdd363fe8e8e3d749ef8782 100644
>--- a/Source/WebCore/ChangeLog
>+++ b/Source/WebCore/ChangeLog
>@@ -1,3 +1,29 @@
>+2018-12-14  Eric Carlson  <eric.carlson@apple.com>
>+
>+        [MediaStream] A stream's first video frame should be rendered
>+        https://bugs.webkit.org/show_bug.cgi?id=192629
>+        <rdar://problem/46664353>
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        Test: fast/mediastream/media-stream-renders-first-frame.html
>+
>+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
>+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSample):
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::ensureLayers):
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::currentDisplayMode const):
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::updateDisplayMode):
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::play):
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::currentReadyState):
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::characteristicsChanged):
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::checkSelectedVideoTrack):
>+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::paintCurrentFrameInContext):
>+        * platform/mediastream/RealtimeMediaSource.cpp:
>+        (WebCore::RealtimeMediaSource::size const):
>+        * platform/mediastream/mac/AVVideoCaptureSource.mm:
>+        (WebCore::AVVideoCaptureSource::processNewFrame):
>+
> 2018-12-14  Youenn Fablet  <youenn@apple.com>
> 
>         getSenders/getReceivers() should not return closed transceiver senders/receivers
>diff --git a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h b/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h
>index 6bb1016603c96bea861a2cc15d7bb6e3f0b7121d..e48474fbd4c4751d161df520eef313c7c3095d45 100644
>--- a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h
>+++ b/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h
>@@ -189,6 +189,7 @@ private:
>     enum DisplayMode {
>         None,
>         PaintItBlack,
>+        WaitingForFirstImage,
>         PausedImage,
>         LivePreview,
>     };
>@@ -229,6 +230,8 @@ private:
> 
>     void applicationDidBecomeActive() final;
> 
>+    bool hideBackgroundLayer() const { return (!m_activeVideoTrack || m_waitingForFirstImage) && m_displayMode != PaintItBlack; }
>+
>     MediaPlayer* m_player { nullptr };
>     RefPtr<MediaStreamPrivate> m_mediaStreamPrivate;
>     RefPtr<MediaStreamTrackPrivate> m_activeVideoTrack;
>@@ -274,10 +277,10 @@ private:
>     bool m_ended { false };
>     bool m_hasEverEnqueuedVideoFrame { false };
>     bool m_pendingSelectedTrackCheck { false };
>-    bool m_shouldDisplayFirstVideoFrame { false };
>     bool m_transformIsValid { false };
>     bool m_visible { false };
>     bool m_haveSeenMetadata { false };
>+    int m_waitingForFirstImage { 0 };
> };
>     
> }
>diff --git a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm b/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm
>index da6b8d4d13bd7a81b8a4f864c0ecacfbbb755496..d235b13ac9a6a584336c44f6d98e1b7f74d14e00 100644
>--- a/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm
>+++ b/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm
>@@ -363,7 +363,7 @@ void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSample(MediaStreamTrackPr
>             updateReadyState();
>     }
> 
>-    if (m_displayMode != LivePreview || (m_displayMode == PausedImage && m_imagePainter.mediaSample))
>+    if (m_displayMode != LivePreview && !m_waitingForFirstImage)
>         return;
> 
>     auto videoTrack = m_videoTrackMap.get(track.id());
>@@ -391,6 +391,8 @@ void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSample(MediaStreamTrackPr
>     }
> 
>     enqueueCorrectedVideoSample(sample);
>+    if (m_waitingForFirstImage && !--m_waitingForFirstImage)
>+        updateDisplayMode();
> }
> 
> void MediaPlayerPrivateMediaStreamAVFObjC::requestNotificationWhenReadyForVideoData()
>@@ -481,6 +483,9 @@ void MediaPlayerPrivateMediaStreamAVFObjC::ensureLayers()
>     m_sampleBufferDisplayLayer.get().videoGravity = AVLayerVideoGravityResizeAspectFill;
> 
>     m_backgroundLayer = adoptNS([[CALayer alloc] init]);
>+    auto hidden = hideBackgroundLayer();
>+    m_backgroundLayer.get().hidden = hidden;
>+
>     m_backgroundLayer.get().backgroundColor = cachedCGColor(Color::black);
>     m_backgroundLayer.get().needsDisplayOnBoundsChange = YES;
> 
>@@ -601,6 +606,9 @@ MediaPlayerPrivateMediaStreamAVFObjC::DisplayMode MediaPlayerPrivateMediaStreamA
>             return PaintItBlack;
>     }
> 
>+    if (m_waitingForFirstImage)
>+        return WaitingForFirstImage;
>+
>     if (playing() && !m_ended) {
>         if (!m_mediaStreamPrivate->isProducingData())
>             return PausedImage;
>@@ -623,9 +631,16 @@ bool MediaPlayerPrivateMediaStreamAVFObjC::updateDisplayMode()
>     INFO_LOG(LOGIDENTIFIER, "updated to ", static_cast<int>(displayMode));
>     m_displayMode = displayMode;
> 
>-    if (m_sampleBufferDisplayLayer) {
>-        runWithoutAnimations([this] {
>-            m_sampleBufferDisplayLayer.get().hidden = m_displayMode < PausedImage;
>+    auto hidden = m_displayMode < PausedImage;
>+    if (m_sampleBufferDisplayLayer && m_sampleBufferDisplayLayer.get().hidden != hidden) {
>+        runWithoutAnimations([this, hidden] {
>+            m_sampleBufferDisplayLayer.get().hidden = hidden;
>+        });
>+    }
>+    hidden = hideBackgroundLayer();
>+    if (m_backgroundLayer && m_backgroundLayer.get().hidden != hidden) {
>+        runWithoutAnimations([this, hidden] {
>+            m_backgroundLayer.get().hidden = hidden;
>         });
>     }
> 
>@@ -646,7 +661,6 @@ void MediaPlayerPrivateMediaStreamAVFObjC::play()
>     for (const auto& track : m_audioTrackMap.values())
>         track->play();
> 
>-    m_shouldDisplayFirstVideoFrame = true;
>     updateDisplayMode();
> 
>     scheduleDeferredTask([this] {
>@@ -767,6 +781,8 @@ MediaPlayer::ReadyState MediaPlayerPrivateMediaStreamAVFObjC::currentReadyState(
>         if (track == m_mediaStreamPrivate->activeVideoTrack() && !m_imagePainter.mediaSample) {
>             if (!m_haveSeenMetadata)
>                 return MediaPlayer::ReadyState::HaveNothing;
>+            if (m_waitingForFirstImage)
>+                return MediaPlayer::ReadyState::HaveMetadata;
>             allTracksAreLive = false;
>         }
>     }
>@@ -828,6 +844,8 @@ void MediaPlayerPrivateMediaStreamAVFObjC::characteristicsChanged()
>     if (intrinsicSize.height() != m_intrinsicSize.height() || intrinsicSize.width() != m_intrinsicSize.width()) {
>         m_intrinsicSize = intrinsicSize;
>         sizeChanged = true;
>+        m_imagePainter.reset();
>+        m_playbackState = PlaybackState::Paused;
>     }
> 
>     updateTracks();
>@@ -979,10 +997,16 @@ void MediaPlayerPrivateMediaStreamAVFObjC::checkSelectedVideoTrack()
>             }
>         }
> 
>-        if (oldVideoTrack != m_activeVideoTrack)
>+        if (oldVideoTrack != m_activeVideoTrack) {
>             m_imagePainter.reset();
>+            if (m_displayMode < PausedImage)
>+                m_waitingForFirstImage = 6;
>+        }
>         ensureLayers();
>         m_sampleBufferDisplayLayer.get().hidden = hideVideoLayer || m_displayMode < PausedImage;
>+        auto hidden = hideBackgroundLayer();
>+        m_backgroundLayer.get().hidden = hidden;
>+
>         m_pendingSelectedTrackCheck = false;
>         updateDisplayMode();
>     });
>@@ -1075,11 +1099,14 @@ void MediaPlayerPrivateMediaStreamAVFObjC::paintCurrentFrameInContext(GraphicsCo
>         updateCurrentFrameImage();
> 
>     GraphicsContextStateSaver stateSaver(context);
>-    if (m_displayMode == PaintItBlack || !m_imagePainter.cgImage || !m_imagePainter.mediaSample) {
>+    if (m_displayMode == PaintItBlack) {
>         context.fillRect(IntRect(IntPoint(), IntSize(destRect.width(), destRect.height())), Color::black);
>         return;
>     }
> 
>+    if (!m_imagePainter.cgImage || !m_imagePainter.mediaSample)
>+        return;
>+
>     auto image = m_imagePainter.cgImage.get();
>     FloatRect imageRect(0, 0, CGImageGetWidth(image), CGImageGetHeight(image));
>     AffineTransform videoTransform = videoTransformationMatrix(*m_imagePainter.mediaSample);
>diff --git a/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp b/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp
>index 3bac9336a4eb7a4b6c8962d53357ee40969eb4c2..2e7f731dc83f9a51698cadda73ee644d5e22ef4c 100644
>--- a/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp
>+++ b/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp
>@@ -874,7 +874,9 @@ const IntSize RealtimeMediaSource::size() const
>     auto size = m_size;
> 
>     if (size.isEmpty() && !m_intrinsicSize.isEmpty()) {
>-        if (size.width())
>+        if (size.isZero())
>+            size = m_intrinsicSize;
>+        else if (size.width())
>             size.setHeight(size.width() * (m_intrinsicSize.height() / static_cast<double>(m_intrinsicSize.width())));
>         else if (size.height())
>             size.setWidth(size.height() * (m_intrinsicSize.width() / static_cast<double>(m_intrinsicSize.height())));
>diff --git a/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm b/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm
>index 9410587736c69c9dcf1de146d5feb8dcfd8c80b5..702dec12888ac9b8004a6386512a015918d8ea13 100644
>--- a/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm
>+++ b/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm
>@@ -524,6 +524,7 @@ void AVVideoCaptureSource::processNewFrame(Ref<MediaSample>&& sample)
>         return;
> 
>     m_buffer = &sample.get();
>+    setIntrinsicSize(expandedIntSize(sample->presentationSize()));
>     dispatchMediaSampleToObservers(WTFMove(sample));
> }
> 
>diff --git a/LayoutTests/ChangeLog b/LayoutTests/ChangeLog
>index 8642cbfca317aa207b825ec1f75912645f8559a6..87dd5a8a3c942f9deae582b1d9cf6de67cdd3142 100644
>--- a/LayoutTests/ChangeLog
>+++ b/LayoutTests/ChangeLog
>@@ -1,3 +1,14 @@
>+2018-12-14  Eric Carlson  <eric.carlson@apple.com>
>+
>+        [MediaStream] A stream's first video frame should be rendered
>+        https://bugs.webkit.org/show_bug.cgi?id=192629
>+        <rdar://problem/46664353>
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        * fast/mediastream/media-stream-renders-first-frame-expected.txt: Added.
>+        * fast/mediastream/media-stream-renders-first-frame.html: Added.
>+
> 2018-12-14  Matt Baker  <mattbaker@apple.com>
> 
>         Web Inspector: Cookies view should use model objects instead of raw payload data
>diff --git a/LayoutTests/fast/mediastream/media-stream-renders-first-frame-expected.txt b/LayoutTests/fast/mediastream/media-stream-renders-first-frame-expected.txt
>new file mode 100644
>index 0000000000000000000000000000000000000000..f92f3ab023094f8560f7538ae6f4bb5b207e8d7a
>--- /dev/null
>+++ b/LayoutTests/fast/mediastream/media-stream-renders-first-frame-expected.txt
>@@ -0,0 +1,4 @@
>+
>+
>+PASS A frame from the camera is shown before playback begins. 
>+
>diff --git a/LayoutTests/fast/mediastream/media-stream-renders-first-frame.html b/LayoutTests/fast/mediastream/media-stream-renders-first-frame.html
>new file mode 100644
>index 0000000000000000000000000000000000000000..fa7d39f0a026f99dbe2fb3a091ec696ee76b7db5
>--- /dev/null
>+++ b/LayoutTests/fast/mediastream/media-stream-renders-first-frame.html
>@@ -0,0 +1,62 @@
>+<!DOCTYPE html>
>+<html>
>+    <head>
>+        <video id="video" width=480px height=480px controls ></video>
>+        <canvas id="canvas" width=640px height=480px></canvas>
>+        <script src="../../resources/testharness.js"></script>
>+        <script src="../../resources/testharnessreport.js"></script>
>+        <script>
>+
>+const canvas = document.getElementById("canvas");
>+const video = document.getElementById("video");
>+
>+function isPixelBlack(pixel)
>+{
>+    return pixel[0] === 0 && pixel[1] === 0 && pixel[2] === 0 && pixel[3] === 255;
>+}
>+
>+function logPixel(name, pixel)
>+{
>+    console.log(`${name}: ${pixel[0]}, ${pixel[1]}, ${pixel[2]}, ${pixel[3]}`);
>+}
>+
>+function checkCanvas(canvas, stream)
>+{
>+    return new Promise((resolve, reject) => {
>+        video.srcObject = stream;
>+        video.oncanplay = () => {
>+            const ctx = canvas.getContext("2d");
>+            ctx.drawImage(video, 0 ,0);
>+
>+            try {
>+                setTimeout(() => {
>+                    assert_false(isPixelBlack(ctx.getImageData(5, 5, 1, 1).data), "Pixel at 5x5 is not black.");
>+                    assert_false(isPixelBlack(ctx.getImageData(50, 200, 1, 1).data), "Pixel at 50x200 is not black.");
>+                    resolve();
>+                }, 500);
>+            } catch(err) {
>+                reject(err);
>+                return;
>+            }
>+        }
>+    });
>+}
>+
>+promise_test(async () => {
>+    let stream = await navigator.mediaDevices.getUserMedia({ video: true });
>+    stream = null;
>+
>+    const devices = await navigator.mediaDevices.enumerateDevices();
>+    let cameraID = undefined;
>+    devices.forEach(device => { if (device.label == "Mock video device 2") cameraID = device.deviceId; });
>+    assert_true(cameraID !== undefined, "Found camera2");
>+    
>+    stream = await navigator.mediaDevices.getUserMedia({ video: { deviceId: { exact: cameraID } } });
>+    
>+    return checkCanvas(canvas, stream);
>+
>+}, "A frame from the camera is shown before playback begins.");
>+
>+        </script>
>+    </head>
>+</html>

Actions: View | Formatted Diff | Diff

Attachments on bug 192629: 357355 | 357366 | 357373 | 357475 | 357484 | 357509