Attachment 356893 Details for Bug 192528 – Patch

[patch] Patch

bug-192528-20181208170101.patch (text/plain), 14.90 KB, created by Eric Carlson on 2018-12-08 17:01:02 PST

(hide)

Description:

Filename:

MIME Type:

Creator: Eric Carlson

Created: 2018-12-08 17:01:02 PST

Size: 14.90 KB

patch

obsolete

>Subversion Revision: 239009
>diff --git a/Source/WebCore/ChangeLog b/Source/WebCore/ChangeLog
>index e440931452070d9d5047ad434965cfbd7acb1e27..4cb5e74838172afbe0be01fa0dc3b9c91f5e6f75 100644
>--- a/Source/WebCore/ChangeLog
>+++ b/Source/WebCore/ChangeLog
>@@ -1,3 +1,27 @@
>+2018-12-08  Eric Carlson  <eric.carlson@apple.com>
>+
>+        [MediaStream] Scaled video frames should be resized in letterbox mode
>+        https://bugs.webkit.org/show_bug.cgi?id=192528
>+        <rdar://problem/46576638>
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        Test: fast/mediastream/resize-letterbox.html
>+
>+        * platform/graphics/cv/ImageTransferSessionVT.mm:
>+        (WebCore::ImageTransferSessionVT::ImageTransferSessionVT): Use letterbox resize mode, not trim.
>+
>+        * platform/mock/MockRealtimeVideoSource.cpp:
>+        (WebCore::MockRealtimeVideoSource::captureSize const): "Capture" at the preset size, not 
>+        necessarily at the requested size to be more like a physical camera.
>+        (WebCore::MockRealtimeVideoSource::settingsDidChange):
>+        (WebCore::MockRealtimeVideoSource::drawAnimation):
>+        (WebCore::MockRealtimeVideoSource::drawBoxes):
>+        (WebCore::MockRealtimeVideoSource::drawText):
>+        (WebCore::MockRealtimeVideoSource::generateFrame):
>+        (WebCore::MockRealtimeVideoSource::imageBuffer const):
>+        * platform/mock/MockRealtimeVideoSource.h:
>+
> 2018-12-07  Eric Carlson  <eric.carlson@apple.com>
> 
>         [MediaStream] 'devicechange' event should not fire in frames that can't access capture devices
>diff --git a/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.cpp b/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.cpp
>index eaf78b45d05287a796e6a2a05cd0e8f0a8994d3f..b0a22d8c3d46504ff1e65afdb5916d299234eca4 100644
>--- a/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.cpp
>+++ b/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.cpp
>@@ -58,7 +58,7 @@ SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTPixelTransferSessionCreat
> SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTPixelTransferSessionTransferImage, OSStatus, (VTPixelTransferSessionRef session, CVPixelBufferRef sourceBuffer, CVPixelBufferRef destinationBuffer), (session, sourceBuffer, destinationBuffer))
> SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTSessionSetProperty, OSStatus, (VTSessionRef session, CFStringRef propertyKey, CFTypeRef propertyValue), (session, propertyKey, propertyValue))
> SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTPixelTransferPropertyKey_ScalingMode, CFStringRef)
>-SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTScalingMode_Trim, CFStringRef)
>+SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTScalingMode_Letterbox, CFStringRef)
> SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTPixelTransferPropertyKey_EnableHardwareAcceleratedTransfer, CFStringRef)
> SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTPixelTransferPropertyKey_EnableHighSpeedTransfer, CFStringRef)
> SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTPixelTransferPropertyKey_RealTime, CFStringRef)
>diff --git a/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.h b/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.h
>index 071b711ed04d3b814822892a150d7ae5fde6f855..9f345dd8a433d0024e16b5b9b485fb607e1c8647 100644
>--- a/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.h
>+++ b/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.h
>@@ -83,8 +83,8 @@ SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, VideoToolbox, VTSessionSetProperty, OSSta
> 
> SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTPixelTransferPropertyKey_ScalingMode, CFStringRef)
> #define kVTPixelTransferPropertyKey_ScalingMode get_VideoToolbox_kVTPixelTransferPropertyKey_ScalingMode()
>-SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTScalingMode_Trim, CFStringRef)
>-#define kVTScalingMode_Trim get_VideoToolbox_kVTScalingMode_Trim()
>+SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTScalingMode_Letterbox, CFStringRef)
>+#define kVTScalingMode_Letterbox get_VideoToolbox_kVTScalingMode_Letterbox()
> SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTPixelTransferPropertyKey_EnableHardwareAcceleratedTransfer, CFStringRef)
> #define kVTPixelTransferPropertyKey_EnableHardwareAcceleratedTransfer get_VideoToolbox_kVTPixelTransferPropertyKey_EnableHardwareAcceleratedTransfer()
> SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTPixelTransferPropertyKey_EnableHighSpeedTransfer, CFStringRef)
>diff --git a/Source/WebCore/platform/graphics/cv/ImageTransferSessionVT.mm b/Source/WebCore/platform/graphics/cv/ImageTransferSessionVT.mm
>index 24aa48c41c932233b53484d4c74e7937fa8c5e69..9c21be71081267b2cd922a692cc76e481b332d25 100644
>--- a/Source/WebCore/platform/graphics/cv/ImageTransferSessionVT.mm
>+++ b/Source/WebCore/platform/graphics/cv/ImageTransferSessionVT.mm
>@@ -59,7 +59,7 @@ ImageTransferSessionVT::ImageTransferSessionVT(uint32_t pixelFormat)
>     ASSERT(transferSession);
>     m_transferSession = adoptCF(transferSession);
> 
>-    auto status = VTSessionSetProperty(transferSession, kVTPixelTransferPropertyKey_ScalingMode, kVTScalingMode_Trim);
>+    auto status = VTSessionSetProperty(transferSession, kVTPixelTransferPropertyKey_ScalingMode, kVTScalingMode_Letterbox);
>     if (status != kCVReturnSuccess)
>         RELEASE_LOG(Media, "ImageTransferSessionVT::ImageTransferSessionVT: VTSessionSetProperty(kVTPixelTransferPropertyKey_ScalingMode) failed with error %d", static_cast<int>(status));
> 
>diff --git a/Source/WebCore/platform/mock/MockRealtimeVideoSource.cpp b/Source/WebCore/platform/mock/MockRealtimeVideoSource.cpp
>index dcffabdcb306757ec3b203d406016b9c52ea6ae6..b12dd4a3d1ab73f7dd19610facd7dd8d6925bce3 100644
>--- a/Source/WebCore/platform/mock/MockRealtimeVideoSource.cpp
>+++ b/Source/WebCore/platform/mock/MockRealtimeVideoSource.cpp
>@@ -178,11 +178,16 @@ const RealtimeMediaSourceSettings& MockRealtimeVideoSource::settings()
>     return m_currentSettings.value();
> }
> 
>+IntSize MockRealtimeVideoSource::captureSize() const
>+{
>+    return m_preset ? m_preset->size : this->size();
>+}
>+
> void MockRealtimeVideoSource::settingsDidChange(OptionSet<RealtimeMediaSourceSettings::Flag> settings)
> {
>     m_currentSettings = std::nullopt;
>     if (settings.containsAny({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height })) {
>-        m_baseFontSize = size().height() * .08;
>+        m_baseFontSize = captureSize().height() * .08;
>         m_bipBopFontSize = m_baseFontSize * 2.5;
>         m_statsFontSize = m_baseFontSize * .5;
>         m_imageBuffer = nullptr;
>@@ -218,8 +223,9 @@ Seconds MockRealtimeVideoSource::elapsedTime()
> 
> void MockRealtimeVideoSource::drawAnimation(GraphicsContext& context)
> {
>-    float radius = size().width() * .09;
>-    FloatPoint location(size().width() * .8, size().height() * .3);
>+    auto size = captureSize();
>+    float radius = size.width() * .09;
>+    FloatPoint location(size.width() * .8, size.height() * .3);
> 
>     m_path.clear();
>     m_path.moveTo(location);
>@@ -248,7 +254,7 @@ void MockRealtimeVideoSource::drawBoxes(GraphicsContext& context)
>     static const RGBA32 green = 0xff008000;
>     static const RGBA32 cyan = 0xFF00FFFF;
> 
>-    IntSize size = this->size();
>+    IntSize size = captureSize();
>     float boxSize = size.width() * .035;
>     float boxTop = size.height() * .6;
> 
>@@ -330,8 +336,8 @@ void MockRealtimeVideoSource::drawText(GraphicsContext& context)
>     FontCascade statsFont { WTFMove(fontDescription), 0, 0 };
>     statsFont.update(nullptr);
> 
>-    IntSize size = this->size();
>-    FloatPoint timeLocation(size.width() * .05, size.height() * .15);
>+    IntSize captureSize = this->captureSize();
>+    FloatPoint timeLocation(captureSize.width() * .05, captureSize.height() * .15);
>     context.setFillColor(Color::white);
>     context.setTextDrawingMode(TextModeFill);
>     String string = String::format("%02u:%02u:%02u.%03u", hours, minutes, seconds, milliseconds % 1000);
>@@ -341,7 +347,7 @@ void MockRealtimeVideoSource::drawText(GraphicsContext& context)
>     timeLocation.move(0, m_baseFontSize);
>     context.drawText(timeFont, TextRun((StringView(string))), timeLocation);
> 
>-    FloatPoint statsLocation(size.width() * .45, size.height() * .75);
>+    FloatPoint statsLocation(captureSize.width() * .45, captureSize.height() * .75);
>     string = String::format("Requested frame rate: %.1f fps", frameRate());
>     context.drawText(statsFont, TextRun((StringView(string))), statsLocation);
> 
>@@ -349,10 +355,15 @@ void MockRealtimeVideoSource::drawText(GraphicsContext& context)
>     string = String::format("Observed frame rate: %.1f fps", observedFrameRate());
>     context.drawText(statsFont, TextRun((StringView(string))), statsLocation);
> 
>+    auto size = this->size();
>     statsLocation.move(0, m_statsFontSize);
>     string = String::format("Size: %u x %u", size.width(), size.height());
>     context.drawText(statsFont, TextRun((StringView(string))), statsLocation);
> 
>+    statsLocation.move(0, m_statsFontSize);
>+    string = String::format("Preset size: %u x %u", captureSize.width(), captureSize.height());
>+    context.drawText(statsFont, TextRun((StringView(string))), statsLocation);
>+
>     if (mockCamera()) {
>         const char* camera;
>         switch (facingMode()) {
>@@ -380,7 +391,7 @@ void MockRealtimeVideoSource::drawText(GraphicsContext& context)
>         context.drawText(statsFont, TextRun { name() }, statsLocation);
>     }
> 
>-    FloatPoint bipBopLocation(size.width() * .6, size.height() * .6);
>+    FloatPoint bipBopLocation(captureSize.width() * .6, captureSize.height() * .6);
>     unsigned frameMod = m_frameNumber % 60;
>     if (frameMod <= 15) {
>         context.setFillColor(Color::cyan);
>@@ -413,7 +424,7 @@ void MockRealtimeVideoSource::generateFrame()
>     GraphicsContext& context = buffer->context();
>     GraphicsContextStateSaver stateSaver(context);
> 
>-    auto& size = this->size();
>+    auto size = captureSize();
>     FloatRect frameRect(FloatPoint(), size);
> 
>     context.fillRect(FloatRect(FloatPoint(), size), m_fillColor);
>@@ -432,7 +443,7 @@ ImageBuffer* MockRealtimeVideoSource::imageBuffer() const
>     if (m_imageBuffer)
>         return m_imageBuffer.get();
> 
>-    m_imageBuffer = ImageBuffer::create(size(), Unaccelerated);
>+    m_imageBuffer = ImageBuffer::create(captureSize(), Unaccelerated);
>     if (!m_imageBuffer)
>         return nullptr;
> 
>diff --git a/Source/WebCore/platform/mock/MockRealtimeVideoSource.h b/Source/WebCore/platform/mock/MockRealtimeVideoSource.h
>index bd3c3e4519af7e85fccb269839e5db6e8930cf5a..075c0efc25ccef79ec281c8c2a009c2722f6bb4a 100644
>--- a/Source/WebCore/platform/mock/MockRealtimeVideoSource.h
>+++ b/Source/WebCore/platform/mock/MockRealtimeVideoSource.h
>@@ -70,6 +70,8 @@ private:
>     bool isCaptureSource() const final { return true; }
>     bool supportsSizeAndFrameRate(std::optional<int> width, std::optional<int> height, std::optional<double>) final;
>     void setSizeAndFrameRate(std::optional<int> width, std::optional<int> height, std::optional<double>) final;
>+    void setSizeAndFrameRateWithPreset(IntSize, double, RefPtr<VideoPreset> preset) final { m_preset = preset; }
>+    IntSize captureSize() const;
> 
>     void generatePresets() final;
> 
>@@ -108,6 +110,7 @@ private:
>     RealtimeMediaSourceSupportedConstraints m_supportedConstraints;
>     Color m_fillColor { Color::black };
>     MockMediaDevice m_device;
>+    RefPtr<VideoPreset> m_preset;
> };
> 
> } // namespace WebCore
>diff --git a/LayoutTests/ChangeLog b/LayoutTests/ChangeLog
>index c7e2389ed9d98fb484cc1e1e982f3c91d1703879..04ca661baf9d7f1c8c4f5ac8a5fe5dde53d87122 100644
>--- a/LayoutTests/ChangeLog
>+++ b/LayoutTests/ChangeLog
>@@ -1,3 +1,14 @@
>+2018-12-08  Eric Carlson  <eric.carlson@apple.com>
>+
>+        [MediaStream] Scaled video frames should be resized in letterbox mode
>+        https://bugs.webkit.org/show_bug.cgi?id=192528
>+        <rdar://problem/46576638>
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        * fast/mediastream/resize-letterbox-expected.txt: Added.
>+        * fast/mediastream/resize-letterbox.html: Added.
>+
> 2018-12-07  Eric Carlson  <eric.carlson@apple.com>
> 
>         [MediaStream] 'devicechange' event should not fire in frames that can't access capture devices
>diff --git a/LayoutTests/fast/mediastream/resize-letterbox-expected.txt b/LayoutTests/fast/mediastream/resize-letterbox-expected.txt
>new file mode 100644
>index 0000000000000000000000000000000000000000..4c018584a49045f2f6035dfb82e2be2cc89102bd
>--- /dev/null
>+++ b/LayoutTests/fast/mediastream/resize-letterbox-expected.txt
>@@ -0,0 +1,4 @@
>+
>+
>+PASS Video frames are resized in letterbox-mode when captured at non-native size. 
>+
>diff --git a/LayoutTests/fast/mediastream/resize-letterbox.html b/LayoutTests/fast/mediastream/resize-letterbox.html
>new file mode 100644
>index 0000000000000000000000000000000000000000..c61d10665b41923a85c0f00b318843493c141b36
>--- /dev/null
>+++ b/LayoutTests/fast/mediastream/resize-letterbox.html
>@@ -0,0 +1,67 @@
>+<!DOCTYPE html>
>+<html>
>+    <head>
>+        <video id="video" autoplay width=480px height=480px controls ></video>
>+        <canvas id="canvas" width=480px height=480px></canvas>
>+        <script src="../../resources/testharness.js"></script>
>+        <script src="../../resources/testharnessreport.js"></script>
>+        <script>
>+
>+const canvas = document.getElementById("canvas");
>+const video = document.getElementById("video");
>+
>+function isPixelBlack(pixel)
>+{
>+    return pixel[0] === 0 && pixel[1] === 0 && pixel[2] === 0 && pixel[3] === 255;
>+}
>+
>+function isPixelGray(pixel)
>+{
>+    return pixel[0] === 128 && pixel[1] === 128 && pixel[2] === 128 && pixel[3] === 255;
>+}
>+
>+function logPixel(name, pixel)
>+{
>+    console.log(`${name}: ${pixel[0]}, ${pixel[1]}, ${pixel[2]}, ${pixel[3]}`);
>+}
>+
>+function checkCanvas(canvas, stream)
>+{
>+    return new Promise((resolve, reject) => {
>+        video.srcObject = stream;
>+        video.onplay = () => {
>+            const ctx = canvas.getContext("2d");
>+            ctx.drawImage(video, 0 ,0);
>+
>+            try {
>+                setTimeout(() => {
>+                    assert_true(isPixelBlack(ctx.getImageData(5, 5, 1, 1).data), "Pixel at 5x5 is NOT from camera.");
>+                    assert_true(isPixelGray(ctx.getImageData(50, 200, 1, 1).data), "Pixel at 50x200 is from camera.");
>+                    resolve();
>+                }, 500);
>+            } catch(err) {
>+                reject(err);
>+                return;
>+            }
>+        }
>+    });
>+}
>+
>+promise_test(async () => {
>+    let stream = await navigator.mediaDevices.getUserMedia({ video: true });
>+    stream = null;
>+
>+    const devices = await navigator.mediaDevices.enumerateDevices();
>+    let cameraID = undefined;
>+    devices.forEach(device => { if (device.label == "Mock video device 2") cameraID = device.deviceId; });
>+    assert_true(cameraID !== undefined, "Found camera2");
>+    
>+    stream = await navigator.mediaDevices.getUserMedia({ video: { deviceId: { exact: cameraID }, width: 480, height: 480 } });
>+    
>+    return checkCanvas(canvas, stream);
>+
>+}, "Video frames are resized in letterbox-mode when captured at non-native size.");
>+
>+        </script>
>+    </head>
>+</html>

Actions: View | Formatted Diff | Diff

Attachments on bug 192528: 356886 | 356893