From d2097d044997e4a45dde86e2c9cd012d5c91f66c Mon Sep 17 00:00:00 2001 From: Robert Mader Date: Mon, 9 Oct 2023 23:45:46 +0200 Subject: [PATCH] gstsink: Add support for DMABuf import and graphics offload By implementing support for `GdkDmabufTextureBuilder` and `GstVideoInfoDmaDrm`. This allows zero-copy video playback on Wayland when paired with hardware video decoding. Can be tested with `gtk4-demo --run=video_player` --- modules/media/gtkgstpaintable.c | 3 + modules/media/gtkgstsink.c | 163 +++++++++++++++++++++++++++++- modules/media/gtkgstsinkprivate.h | 5 + modules/media/meson.build | 11 +- 4 files changed, 178 insertions(+), 4 deletions(-) diff --git a/modules/media/gtkgstpaintable.c b/modules/media/gtkgstpaintable.c index 8abf4a2a6e..bd2b8408b0 100644 --- a/modules/media/gtkgstpaintable.c +++ b/modules/media/gtkgstpaintable.c @@ -115,12 +115,15 @@ gtk_gst_paintable_video_renderer_create_video_sink (GstPlayerVideoRenderer *rend GstPlayer *player) { GtkGstPaintable *self = GTK_GST_PAINTABLE (renderer); + GdkDmabufFormats *dmabuf_formats; GstElement *sink; GdkGLContext *ctx; + dmabuf_formats = gdk_display_get_dmabuf_formats (gdk_display_get_default ()); sink = g_object_new (GTK_TYPE_GST_SINK, "paintable", self, "gl-context", self->context, + "dmabuf-formats", dmabuf_formats, NULL); if (self->context != NULL) diff --git a/modules/media/gtkgstsink.c b/modules/media/gtkgstsink.c index 9190ec6244..131c0de38d 100644 --- a/modules/media/gtkgstsink.c +++ b/modules/media/gtkgstsink.c @@ -52,10 +52,16 @@ #include +#ifdef HAVE_GSTREAMER_DRM +#include +#include +#endif + enum { PROP_0, PROP_PAINTABLE, PROP_GL_CONTEXT, + PROP_DMABUF_FORMATS, N_PROPS, }; @@ -71,7 +77,11 @@ static GstStaticPadTemplate gtk_gst_sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, - GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), " + GST_STATIC_CAPS ( +#ifdef HAVE_GSTREAMER_DRM + GST_VIDEO_DMA_DRM_CAPS_MAKE "; " +#endif + "video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), " "format = (string) RGBA, " "width = " GST_VIDEO_SIZE_RANGE ", " "height = " GST_VIDEO_SIZE_RANGE ", " @@ -116,6 +126,42 @@ gtk_gst_sink_get_times (GstBaseSink *bsink, } } +#ifdef HAVE_GSTREAMER_DRM +static void +add_drm_formats_and_modifiers (GstCaps *caps, + GdkDmabufFormats *dmabuf_formats) +{ + GValue dmabuf_list = G_VALUE_INIT; + size_t i; + + g_value_init (&dmabuf_list, GST_TYPE_LIST); + + for (i = 0; i < gdk_dmabuf_formats_get_n_formats (dmabuf_formats); i++) + { + GValue value = G_VALUE_INIT; + gchar *drm_format_string; + guint32 fmt; + guint64 mod; + + gdk_dmabuf_formats_get_format (dmabuf_formats, i, &fmt, &mod); + + if (mod == DRM_FORMAT_MOD_INVALID) + continue; + + drm_format_string = gst_video_dma_drm_fourcc_to_string (fmt, mod); + if (!drm_format_string) + continue; + + g_value_init (&value, G_TYPE_STRING); + g_value_take_string (&value, drm_format_string); + gst_value_list_append_and_take_value (&dmabuf_list, &value); + } + + gst_structure_take_value (gst_caps_get_structure (caps, 0), "drm-format", + &dmabuf_list); +} +#endif + static GstCaps * gtk_gst_sink_get_caps (GstBaseSink *bsink, GstCaps *filter) @@ -127,6 +173,13 @@ gtk_gst_sink_get_caps (GstBaseSink *bsink, if (self->gst_context) { tmp = gst_pad_get_pad_template_caps (GST_BASE_SINK_PAD (bsink)); +#ifdef HAVE_GSTREAMER_DRM + if (self->dmabuf_formats) + { + tmp = gst_caps_make_writable (tmp); + add_drm_formats_and_modifiers (tmp, self->dmabuf_formats); + } +#endif } else { @@ -159,8 +212,24 @@ gtk_gst_sink_set_caps (GstBaseSink *bsink, GST_DEBUG_OBJECT (self, "set caps with %" GST_PTR_FORMAT, caps); - if (!gst_video_info_from_caps (&self->v_info, caps)) - return FALSE; +#ifdef HAVE_GSTREAMER_DRM + if (gst_video_is_dma_drm_caps (caps)) { + if (!gst_video_info_dma_drm_from_caps (&self->drm_info, caps)) + return FALSE; + + if (!gst_video_info_dma_drm_to_video_info (&self->drm_info, &self->v_info)) + return FALSE; + + GST_INFO_OBJECT (self, "using DMABuf, passthrough possible"); + } else { + gst_video_info_dma_drm_init (&self->drm_info); +#endif + + if (!gst_video_info_from_caps (&self->v_info, caps)) + return FALSE; +#ifdef HAVE_GSTREAMER_DRM + } +#endif return TRUE; } @@ -202,6 +271,14 @@ gtk_gst_sink_propose_allocation (GstBaseSink *bsink, return FALSE; } +#ifdef HAVE_GSTREAMER_DRM + if (gst_caps_features_contains (gst_caps_get_features (caps, 0), GST_CAPS_FEATURE_MEMORY_DMABUF)) + { + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0); + return TRUE; + } +#endif + if (!gst_caps_features_contains (gst_caps_get_features (caps, 0), GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) return FALSE; @@ -287,6 +364,66 @@ gtk_gst_sink_texture_from_buffer (GtkGstSink *self, GstVideoFrame *frame = g_new (GstVideoFrame, 1); GdkTexture *texture; +#ifdef HAVE_GSTREAMER_DRM + if (gst_is_dmabuf_memory (gst_buffer_peek_memory (buffer, 0))) + { + g_autoptr (GdkDmabufTextureBuilder) builder = NULL; + const GstVideoMeta *vmeta = gst_buffer_get_video_meta (buffer); + GError *error = NULL; + int i; + + /* We don't map dmabufs */ + g_clear_pointer (&frame, g_free); + + g_return_val_if_fail (vmeta, NULL); + g_return_val_if_fail (self->gdk_context, NULL); + g_return_val_if_fail (self->drm_info.drm_fourcc != DRM_FORMAT_INVALID, NULL); + + builder = gdk_dmabuf_texture_builder_new (); + gdk_dmabuf_texture_builder_set_display (builder, gdk_gl_context_get_display (self->gdk_context)); + gdk_dmabuf_texture_builder_set_fourcc (builder, self->drm_info.drm_fourcc); + gdk_dmabuf_texture_builder_set_modifier (builder, self->drm_info.drm_modifier); + // Padded width/height is set into the vmeta, perhaps we should import using these ? + gdk_dmabuf_texture_builder_set_width (builder, GST_VIDEO_INFO_WIDTH (&self->v_info)); + gdk_dmabuf_texture_builder_set_height (builder, GST_VIDEO_INFO_HEIGHT (&self->v_info)); + gdk_dmabuf_texture_builder_set_n_planes (builder, vmeta->n_planes); + + for (i = 0; i < vmeta->n_planes; i++) + { + GstMemory *mem; + guint mem_idx, length; + gsize skip; + + if (!gst_buffer_find_memory (buffer, + vmeta->offset[i], + 1, + &mem_idx, + &length, + &skip)) + { + GST_ERROR_OBJECT (self, "Buffer data is bogus"); + return NULL; + } + + mem = gst_buffer_peek_memory (buffer, mem_idx); + + gdk_dmabuf_texture_builder_set_fd (builder, i, gst_dmabuf_memory_get_fd (mem)); + gdk_dmabuf_texture_builder_set_offset (builder, i, mem->offset + skip); + gdk_dmabuf_texture_builder_set_stride (builder, i, vmeta->stride[i]); + } + + texture = gdk_dmabuf_texture_builder_build (builder, + (GDestroyNotify) gst_buffer_unref, + gst_buffer_ref (buffer), + &error); + if (!texture) + GST_ERROR_OBJECT (self, "Failed to create dmabuf texture: %s", error->message); + + *pixel_aspect_ratio = ((double) GST_VIDEO_INFO_PAR_N (&self->v_info) / + (double) GST_VIDEO_INFO_PAR_D (&self->v_info)); + } + else +#endif if (self->gdk_context && gst_video_frame_map (frame, &self->v_info, buffer, GST_MAP_READ | GST_MAP_GL)) { @@ -593,6 +730,10 @@ gtk_gst_sink_set_property (GObject *object, g_clear_object (&self->gdk_context); break; + case PROP_DMABUF_FORMATS: + self->dmabuf_formats = g_value_get_boxed (value); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -615,6 +756,9 @@ gtk_gst_sink_get_property (GObject *object, case PROP_GL_CONTEXT: g_value_set_object (value, self->gdk_context); break; + case PROP_DMABUF_FORMATS: + g_value_set_boxed (value, self->dmabuf_formats); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -675,6 +819,19 @@ gtk_gst_sink_class_init (GtkGstSinkClass * klass) GDK_TYPE_GL_CONTEXT, G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS); + /** + * GtkGstSink:dmabuf-formats: + * + * The #GdkDmabufFormats that are supported by the #GdkDisplay and can be used + * with #GdkDmabufTextureBuilder. + * + * Since: 4.14 + */ + properties[PROP_DMABUF_FORMATS] = + g_param_spec_boxed ("dmabuf-formats", NULL, NULL, + GDK_TYPE_DMABUF_FORMATS, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS); + g_object_class_install_properties (gobject_class, N_PROPS, properties); gst_element_class_set_metadata (gstelement_class, diff --git a/modules/media/gtkgstsinkprivate.h b/modules/media/gtkgstsinkprivate.h index b52d6183f6..f291fca664 100644 --- a/modules/media/gtkgstsinkprivate.h +++ b/modules/media/gtkgstsinkprivate.h @@ -47,11 +47,16 @@ struct _GtkGstSink GstVideoSink parent; GstVideoInfo v_info; +#ifdef HAVE_GSTREAMER_DRM + GstVideoInfoDmaDrm drm_info; +#endif + GtkGstPaintable * paintable; GdkGLContext * gdk_context; GstGLDisplay * gst_display; GstGLContext * gst_gdk_context; GstGLContext * gst_context; + GdkDmabufFormats * dmabuf_formats; }; struct _GtkGstSinkClass diff --git a/modules/media/meson.build b/modules/media/meson.build index 74364e6439..32f31ba01e 100644 --- a/modules/media/meson.build +++ b/modules/media/meson.build @@ -43,6 +43,8 @@ gstplayer_dep = dependency('gstreamer-player-1.0', version: '>= 1.12.3', required: get_option('media-gstreamer')) gstgl_dep = dependency('gstreamer-gl-1.0', version: '>= 1.12.3', required: get_option('media-gstreamer')) +gstdrm_dep = dependency('gstreamer-allocators-1.0', version: '>= 1.23', + required: false) if gstplayer_dep.found() and gstgl_dep.found() extra_win_cflags = [] @@ -54,6 +56,13 @@ if gstplayer_dep.found() and gstgl_dep.found() media_backends += 'gstreamer' cdata.set('HAVE_GSTREAMER', 1) + media_gst_deps = [ libm, libgtk_dep, gstplayer_dep, gstgl_dep ] + + if dmabuf_dep.found() and gstdrm_dep.found() + cdata.set('HAVE_GSTREAMER_DRM', 1) + media_gst_deps += [ gstdrm_dep ] + endif + shared_module('media-gstreamer', sources: [ 'gtkgstmediafile.c', @@ -61,7 +70,7 @@ if gstplayer_dep.found() and gstgl_dep.found() 'gtkgstsink.c', ], c_args: extra_c_args + extra_win_cflags, - dependencies: [ libm, libgtk_dep, gstplayer_dep, gstgl_dep ], + dependencies: media_gst_deps, name_suffix: module_suffix, install_dir: media_install_dir, install: true,