Attachment #441039 for bug #231425

View Details Raw Unified Return to Bug 231425



2021-10-12  Kimmo Kinnunen  <kkinnunen@apple.com>

        WebGL video texture upload is very slow due to excessive transfer of the video pixel buffer
        https://bugs.webkit.org/show_bug.cgi?id=231425

        Reviewed by Youenn Fablet.

        Typical WebGL content requests the videos to be uploaded to a texture
        once per render loop update, even though the video has not changed.
        The video pixel buffer is slow to transfer across IPC. It should be transferred
        only when it has changed.

        MediaPlayerPrivateAVFoundationObjC and MediaPlayerPrivateMediaSourceAVFObjC hold
        the last requested pixel buffer ref. They will update it or discard it only
        when pixelBufferForCurrentTime() is called.

        Cache the pixel buffer to MediaPlayerPrivateRemote (GPUP side proxy)
        Cache the pixel buffer to RemoteMediaPlayerProxy (WebP side proxy)

        The caches increase the total memory use only during the duration
        of the call, since after the ref has been updated, it refers
        to the one always held in the original objects (MediaPlayerPrivateAVFoundationObjC
        and MediaPlayerPrivateMediaSourceAVFObjC)

        No new tests, fixes a perf regression wrt GPUP media.

        * GPUProcess/media/RemoteMediaPlayerProxy.cpp:
        (WebKit::RemoteMediaPlayerProxy::invalidate):
        * GPUProcess/media/RemoteMediaPlayerProxy.h:
        * GPUProcess/media/RemoteMediaPlayerProxy.messages.in:
        * GPUProcess/media/cocoa/RemoteMediaPlayerProxyCocoa.mm:
        (WebKit::RemoteMediaPlayerProxy::nativeImageForCurrentTime):
        (WebKit::RemoteMediaPlayerProxy::pixelBufferForCurrentTimeIfChanged):
        (WebKit::RemoteMediaPlayerProxy::pixelBufferForCurrentTime): Deleted.
        * WebProcess/GPU/media/MediaPlayerPrivateRemote.h:
        * WebProcess/GPU/media/cocoa/MediaPlayerPrivateRemoteCocoa.mm:
        (WebKit::MediaPlayerPrivateRemote::pixelBufferForCurrentTime):

2021-10-12  Per Arne Vollan <pvollan@apple.com>

        [iOS][GPUP] Remove sandbox access to com.apple.audio.AudioComponentRegistrar

Lines 105-110 void RemoteMediaPlayerProxy::invalidate() a/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.cpp_sec1
- a/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.cpp +3 lines
105	m_sandboxExtension = nullptr;	105	m_sandboxExtension = nullptr;
106	}	106	}
107	m_renderingResourcesRequest = { };	107	m_renderingResourcesRequest = { };
		108	#if USE(AVFOUNDATION)
		109	m_pixelBufferForCurrentTime = nullptr;
		110	#endif
108	}	111	}
109		112
110	void RemoteMediaPlayerProxy::getConfiguration(RemoteMediaPlayerConfiguration& configuration)	113	void RemoteMediaPlayerProxy::getConfiguration(RemoteMediaPlayerConfiguration& configuration)

Lines 314-320 private: a/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.h_sec1
- a/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.h -2 / +4 lines
314	void nativeImageForCurrentTime(CompletionHandler<void(std::optional<WTF::MachSendRight>&&)>&&);	314	void nativeImageForCurrentTime(CompletionHandler<void(std::optional<WTF::MachSendRight>&&)>&&);
315	#endif	315	#endif
316	#if USE(AVFOUNDATION)	316	#if USE(AVFOUNDATION)
317	void pixelBufferForCurrentTime(CompletionHandler<void(RetainPtr<CVPixelBufferRef>&&)>&&);	317	void pixelBufferForCurrentTimeIfChanged(CompletionHandler<void(std::optional<RetainPtr<CVPixelBufferRef>>&&)>&&);
318	#endif	318	#endif
319		319
320	#if !RELEASE_LOG_DISABLED	320	#if !RELEASE_LOG_DISABLED
Lines 369-375 private: a/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.h_sec2
369	ScopedRenderingResourcesRequest m_renderingResourcesRequest;	369	ScopedRenderingResourcesRequest m_renderingResourcesRequest;
370		370
371	bool m_observingTimeChanges { false };	371	bool m_observingTimeChanges { false };
372		372	#if USE(AVFOUNDATION)
		373	RetainPtr<CVPixelBufferRef> m_pixelBufferForCurrentTime;
		374	#endif
373	#if !RELEASE_LOG_DISABLED	375	#if !RELEASE_LOG_DISABLED
374	const Logger& m_logger;	376	const Logger& m_logger;
375	#endif	377	#endif

Lines 126-132 messages -> RemoteMediaPlayerProxy NotRefCounted { a/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.messages.in_sec1
- a/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.messages.in -1 / +1 lines
126	NativeImageForCurrentTime() -> (std::optional<MachSendRight> sendRight) Synchronous	126	NativeImageForCurrentTime() -> (std::optional<MachSendRight> sendRight) Synchronous
127	#endif	127	#endif
128	#if USE(AVFOUNDATION)	128	#if USE(AVFOUNDATION)
129	PixelBufferForCurrentTime() -> (RetainPtr<CVPixelBufferRef> pixelBuffer) Synchronous	129	PixelBufferForCurrentTimeIfChanged() -> (std::optional<RetainPtr<CVPixelBufferRef>> pixelBuffer) Synchronous
130	#endif	130	#endif
131		131
132	PlayAtHostTime(MonotonicTime time)	132	PlayAtHostTime(MonotonicTime time)



    completionHandler(surface->createSendRight());
}

#if USE(AVFOUNDATION)
void RemoteMediaPlayerProxy::pixelBufferForCurrentTimeIfChanged(CompletionHandler<void(std::optional<RetainPtr<CVPixelBufferRef>>&&)>&& completionHandler)
{
    std::optional<RetainPtr<CVPixelBufferRef>> result;
    RetainPtr<CVPixelBufferRef> pixelBuffer;
    if (m_player)
        pixelBuffer = m_player->pixelBufferForCurrentTime();
    if (m_pixelBufferForCurrentTime != pixelBuffer) {
        result = pixelBuffer;
        m_pixelBufferForCurrentTime = WTFMove(pixelBuffer);
    }
    completionHandler(WTFMove(result));
}
#endif

} // namespace WebKit


Lines 445-450 private: a/Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.h_sec1
- a/Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.h +3 lines
445	bool m_waitingForKey { false };	445	bool m_waitingForKey { false };
446	bool m_timeIsProgressing { false };	446	bool m_timeIsProgressing { false };
447	bool m_renderingCanBeAccelerated { false };	447	bool m_renderingCanBeAccelerated { false };
		448	#if USE(AVFOUNDATION)
		449	RetainPtr<CVPixelBufferRef> m_pixelBufferForCurrentTime;
		450	#endif
448	};	451	};
449		452
450	} // namespace WebKit	453	} // namespace WebKit



    return NativeImage::create(WTFMove(platformImage));
}

#if USE(AVFOUNDATION)
RetainPtr<CVPixelBufferRef> MediaPlayerPrivateRemote::pixelBufferForCurrentTime()
{
    std::optional<RetainPtr<CVPixelBufferRef>> result;
    if (!connection().sendSync(Messages::RemoteMediaPlayerProxy::PixelBufferForCurrentTimeIfChanged(), Messages::RemoteMediaPlayerProxy::PixelBufferForCurrentTimeIfChanged::Reply(result), m_id))
    if (!connection().sendSync(Messages::RemoteMediaPlayerProxy::PixelBufferForCurrentTime(), Messages::RemoteMediaPlayerProxy::PixelBufferForCurrentTime::Reply(result), m_id))
        return nullptr;
    if (result)
        m_pixelBufferForCurrentTime = WTFMove(*result);
    return m_pixelBufferForCurrentTime;
}
#endif

} // namespace WebKit


Return to Bug 231425

Lines 1-3 a/Source/WebKit/ChangeLog_sec1
- a/Source/WebKit/ChangeLog +38 lines
		1	2021-10-12 Kimmo Kinnunen <kkinnunen@apple.com>
		2
		3	WebGL video texture upload is very slow due to excessive transfer of the video pixel buffer
		4	https://bugs.webkit.org/show_bug.cgi?id=231425
		5
		6	Reviewed by Youenn Fablet.
		7
		8	Typical WebGL content requests the videos to be uploaded to a texture
		9	once per render loop update, even though the video has not changed.
		10	The video pixel buffer is slow to transfer across IPC. It should be transferred
		11	only when it has changed.
		12
		13	MediaPlayerPrivateAVFoundationObjC and MediaPlayerPrivateMediaSourceAVFObjC hold
		14	the last requested pixel buffer ref. They will update it or discard it only
		15	when pixelBufferForCurrentTime() is called.
		16
		17	Cache the pixel buffer to MediaPlayerPrivateRemote (GPUP side proxy)
		18	Cache the pixel buffer to RemoteMediaPlayerProxy (WebP side proxy)
		19
		20	The caches increase the total memory use only during the duration
		21	of the call, since after the ref has been updated, it refers
		22	to the one always held in the original objects (MediaPlayerPrivateAVFoundationObjC
		23	and MediaPlayerPrivateMediaSourceAVFObjC)
		24
		25	No new tests, fixes a perf regression wrt GPUP media.
		26
		27	* GPUProcess/media/RemoteMediaPlayerProxy.cpp:
		28	(WebKit::RemoteMediaPlayerProxy::invalidate):
		29	* GPUProcess/media/RemoteMediaPlayerProxy.h:
		30	* GPUProcess/media/RemoteMediaPlayerProxy.messages.in:
		31	* GPUProcess/media/cocoa/RemoteMediaPlayerProxyCocoa.mm:
		32	(WebKit::RemoteMediaPlayerProxy::nativeImageForCurrentTime):
		33	(WebKit::RemoteMediaPlayerProxy::pixelBufferForCurrentTimeIfChanged):
		34	(WebKit::RemoteMediaPlayerProxy::pixelBufferForCurrentTime): Deleted.
		35	* WebProcess/GPU/media/MediaPlayerPrivateRemote.h:
		36	* WebProcess/GPU/media/cocoa/MediaPlayerPrivateRemoteCocoa.mm:
		37	(WebKit::MediaPlayerPrivateRemote::pixelBufferForCurrentTime):
		38
1	2021-10-12 Per Arne Vollan <pvollan@apple.com>	39	2021-10-12 Per Arne Vollan <pvollan@apple.com>
2		40
3	[iOS][GPUP] Remove sandbox access to com.apple.audio.AudioComponentRegistrar	41	[iOS][GPUP] Remove sandbox access to com.apple.audio.AudioComponentRegistrar

Lines 111-123 void RemoteMediaPlayerProxy::nativeImageForCurrentTime(CompletionHandler<void(st a/Source/WebKit/GPUProcess/media/cocoa/RemoteMediaPlayerProxyCocoa.mm_sec1
- a/Source/WebKit/GPUProcess/media/cocoa/RemoteMediaPlayerProxyCocoa.mm -3 / +10 lines
111	completionHandler(surface->createSendRight());	111	completionHandler(surface->createSendRight());
112	}	112	}
113		113
114	void RemoteMediaPlayerProxy::pixelBufferForCurrentTime(CompletionHandler<void(RetainPtr<CVPixelBufferRef>&&)>&& completionHandler)	114	#if USE(AVFOUNDATION)
		115	void RemoteMediaPlayerProxy::pixelBufferForCurrentTimeIfChanged(CompletionHandler<void(std::optional<RetainPtr<CVPixelBufferRef>>&&)>&& completionHandler)
115	{	116	{
116	RetainPtr<CVPixelBufferRef> result;	117	std::optional<RetainPtr<CVPixelBufferRef>> result;
		118	RetainPtr<CVPixelBufferRef> pixelBuffer;
117	if (m_player)	119	if (m_player)
118	result = m_player->pixelBufferForCurrentTime();	120	pixelBuffer = m_player->pixelBufferForCurrentTime();
		121	if (m_pixelBufferForCurrentTime != pixelBuffer) {
		122	result = pixelBuffer;
		123	m_pixelBufferForCurrentTime = WTFMove(pixelBuffer);
		124	}
119	completionHandler(WTFMove(result));	125	completionHandler(WTFMove(result));
120	}	126	}
		127	#endif
121		128
122	} // namespace WebKit	129	} // namespace WebKit
123		130

Lines 66-79 RefPtr<NativeImage> MediaPlayerPrivateRemote::nativeImageForCurrentTime() a/Source/WebKit/WebProcess/GPU/media/cocoa/MediaPlayerPrivateRemoteCocoa.mm_sec1
- a/Source/WebKit/WebProcess/GPU/media/cocoa/MediaPlayerPrivateRemoteCocoa.mm -4 / +7 lines
66	return NativeImage::create(WTFMove(platformImage));	66	return NativeImage::create(WTFMove(platformImage));
67	}	67	}
68		68
		69	#if USE(AVFOUNDATION)
69	RetainPtr<CVPixelBufferRef> MediaPlayerPrivateRemote::pixelBufferForCurrentTime()	70	RetainPtr<CVPixelBufferRef> MediaPlayerPrivateRemote::pixelBufferForCurrentTime()
70	{	71	{
71		72	std::optional<RetainPtr<CVPixelBufferRef>> result;
72	RetainPtr<CVPixelBufferRef> result;	73	if (!connection().sendSync(Messages::RemoteMediaPlayerProxy::PixelBufferForCurrentTimeIfChanged(), Messages::RemoteMediaPlayerProxy::PixelBufferForCurrentTimeIfChanged::Reply(result), m_id))
73	if (!connection().sendSync(Messages::RemoteMediaPlayerProxy::PixelBufferForCurrentTime(), Messages::RemoteMediaPlayerProxy::PixelBufferForCurrentTime::Reply(result), m_id))
74	return nullptr;	74	return nullptr;
75	return result;	75	if (result)
		76	m_pixelBufferForCurrentTime = WTFMove(*result);
		77	return m_pixelBufferForCurrentTime;
76	}	78	}
		79	#endif
77		80
78	} // namespace WebKit	81	} // namespace WebKit
79		82