gstreamer: Improve caps handling

Separate out the caps for dmabuf, gl and memory and add them
when appropriate.

modules/media/gtkgstpaintable.c

@@ -37,6 +37,7 @@ struct _GtkGstPaintable
   graphene_rect_t viewport;
 
   GdkGLContext *context;
+  GdkSurface *surface;
 };
 
 struct _GtkGstPaintableClass
@@ -133,27 +134,35 @@ gtk_gst_paintable_video_renderer_create_video_sink (GstPlayerVideoRenderer *rend
 {
   GtkGstPaintable *self = GTK_GST_PAINTABLE (renderer);
   GstElement *sink;
-  GdkGLContext *ctx;
+  GdkGLContext *ctx = NULL;
+  GdkDisplay *display;
+
+  if (self->surface)
+    display = gdk_surface_get_display (self->surface);
+  else
+    display = gdk_display_get_default ();
 
   sink = g_object_new (GTK_TYPE_GST_SINK,
                        "paintable", self,
                        "gl-context", self->context,
+                       "display", display,
                        NULL);
 
-  if (self->context != NULL)
-    g_object_get (GTK_GST_SINK (sink), "gl-context", &ctx, NULL);
+  g_object_get (GTK_GST_SINK (sink), "gl-context", &ctx, NULL);
 
-  if (self->context != NULL && ctx != NULL)
+  if (ctx != NULL)
     {
-      GstElement *glsinkbin = gst_element_factory_make ("glsinkbin", NULL);
+      GstElement *glsinkbin;
 
-      if (!glsinkbin)
-        return NULL;
+      glsinkbin = gst_element_factory_make ("glsinkbin", NULL);
+
+      if (glsinkbin)
+        g_object_set (glsinkbin, "sink", sink, NULL);
+
+      g_object_unref (sink);
+      sink = glsinkbin;
 
-      g_object_set (glsinkbin, "sink", sink, NULL);
       g_object_unref (ctx);
-
-      return glsinkbin;
     }
   else
     {
@@ -164,11 +173,12 @@ gtk_gst_paintable_video_renderer_create_video_sink (GstPlayerVideoRenderer *rend
           g_object_unref (sink);
           sink = g_object_new (GTK_TYPE_GST_SINK,
                                "paintable", self,
+                               "display", display,
                                NULL);
         }
-
-      return sink;
     }
+
+  return sink;
 }
 
 static void
@@ -218,9 +228,11 @@ gtk_gst_paintable_realize (GtkGstPaintable *self,
 {
   GError *error = NULL;
 
-  if (self->context)
+  if (self->surface)
     return;
 
+  self->surface = surface;
+
   self->context = gdk_surface_create_gl_context (surface, &error);
   if (self->context == NULL)
     {
@@ -246,11 +258,12 @@ gtk_gst_paintable_unrealize (GtkGstPaintable *self,
    * - track how often we were realized with that surface
    * - track alternate surfaces
    */
-  if (self->context == NULL)
-    return;
 
-  if (gdk_gl_context_get_surface (self->context) == surface)
-    g_clear_object (&self->context);
+  if (self->surface == surface)
+    {
+      g_clear_object (&self->context);
+      self->surface = NULL;
+    }
 }
 
 static void

modules/media/gtkgstsink.c

@@ -59,6 +59,7 @@ enum {
   PROP_0,
   PROP_PAINTABLE,
   PROP_GL_CONTEXT,
+  PROP_DISPLAY,
 
   N_PROPS,
 };
@@ -68,20 +69,25 @@ GST_DEBUG_CATEGORY (gtk_debug_gst_sink);
 
 #define FORMATS "{ BGRA, ARGB, RGBA, ABGR, RGB, BGR }"
 
-#define NOGL_CAPS GST_VIDEO_CAPS_MAKE (FORMATS)
+#define MEMORY_TEXTURE_CAPS GST_VIDEO_CAPS_MAKE (FORMATS)
+
+#define GL_TEXTURE_CAPS \
+                     "video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), " \
+                     "format = (string) RGBA, " \
+                     "width = " GST_VIDEO_SIZE_RANGE ", " \
+                     "height = " GST_VIDEO_SIZE_RANGE ", " \
+                     "framerate = " GST_VIDEO_FPS_RANGE ", " \
+                     "texture-target = (string) 2D"
+
+#define DMABUF_TEXTURE_CAPS GST_VIDEO_DMA_DRM_CAPS_MAKE
 
 static GstStaticPadTemplate gtk_gst_sink_template =
 GST_STATIC_PAD_TEMPLATE ("sink",
     GST_PAD_SINK,
     GST_PAD_ALWAYS,
-    GST_STATIC_CAPS (GST_VIDEO_DMA_DRM_CAPS_MAKE "; "
-                     "video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), "
-                     "format = (string) RGBA, "
-                     "width = " GST_VIDEO_SIZE_RANGE ", "
-                     "height = " GST_VIDEO_SIZE_RANGE ", "
-                     "framerate = " GST_VIDEO_FPS_RANGE ", "
-                     "texture-target = (string) 2D"
-                     "; " NOGL_CAPS)
+    GST_STATIC_CAPS(DMABUF_TEXTURE_CAPS "; "
+                    GL_TEXTURE_CAPS "; "
+                    MEMORY_TEXTURE_CAPS)
     );
 
 #undef GST_VIDEO_DMA_DRM_CAPS_MAKE_STR
@@ -204,22 +210,31 @@ gtk_gst_sink_get_caps (GstBaseSink *bsink,
                        GstCaps     *filter)
 {
   GtkGstSink *self = GTK_GST_SINK (bsink);
-  GstCaps *tmp;
-  GstCaps *result;
+  GstCaps *tmp, *result;
 
-  if (self->gst_context)
-    {
-      GdkDisplay *display = gdk_gl_context_get_display (self->gdk_context);
-      GdkDmabufFormats *formats = gdk_display_get_dmabuf_formats (display);
+  result = gst_caps_new_empty ();
 
-      tmp = gst_pad_get_pad_template_caps (GST_BASE_SINK_PAD (bsink));
-      tmp = gst_caps_make_writable (tmp);
-      add_drm_formats_and_modifiers (tmp, formats);
-    }
-  else
+  if (self->gdk_display)
     {
-      tmp = gst_caps_from_string (NOGL_CAPS);
+      GdkDmabufFormats *formats = gdk_display_get_dmabuf_formats (self->gdk_display);
+
+      if (formats)
+        {
+          tmp = gst_caps_from_string (DMABUF_TEXTURE_CAPS);
+          add_drm_formats_and_modifiers (tmp, formats);
+          gst_caps_append (result, tmp);
+        }
     }
+
+  if (self->gdk_context)
+    {
+      tmp = gst_caps_from_string (GL_TEXTURE_CAPS);
+      gst_caps_append (result, tmp);
+    }
+
+  tmp = gst_caps_from_string (MEMORY_TEXTURE_CAPS);
+  gst_caps_append (result, tmp);
+
   GST_DEBUG_OBJECT (self, "advertising own caps %" GST_PTR_FORMAT, tmp);
 
   if (filter)
@@ -299,9 +314,6 @@ gtk_gst_sink_propose_allocation (GstBaseSink *bsink,
   gboolean need_pool;
   GstVideoInfo info;
 
-  if (!self->gst_context)
-    return FALSE;
-
   gst_query_parse_allocation (query, &caps, &need_pool);
 
   if (caps == NULL)
@@ -316,47 +328,49 @@ gtk_gst_sink_propose_allocation (GstBaseSink *bsink,
       return TRUE;
     }
 
-  if (!gst_caps_features_contains (gst_caps_get_features (caps, 0), GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
-    return FALSE;
-
-  if (!gst_video_info_from_caps (&info, caps))
+  if (gst_caps_features_contains (gst_caps_get_features (caps, 0), GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
     {
-      GST_DEBUG_OBJECT (self, "invalid caps specified");
-      return FALSE;
-    }
-
-  /* the normal size of a frame */
-  size = info.size;
-
-  if (need_pool)
-    {
-      GST_DEBUG_OBJECT (self, "create new pool");
-      pool = gst_gl_buffer_pool_new (self->gst_context);
-
-      config = gst_buffer_pool_get_config (pool);
-      gst_buffer_pool_config_set_params (config, caps, size, 0, 0);
-      gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_GL_SYNC_META);
-
-      if (!gst_buffer_pool_set_config (pool, config))
+      if (!gst_video_info_from_caps (&info, caps))
         {
-          GST_DEBUG_OBJECT (bsink, "failed setting config");
-          gst_object_unref (pool);
+          GST_DEBUG_OBJECT (self, "invalid caps specified");
           return FALSE;
         }
+
+      /* the normal size of a frame */
+      size = info.size;
+
+      if (need_pool)
+        {
+          GST_DEBUG_OBJECT (self, "create new pool");
+          pool = gst_gl_buffer_pool_new (self->gst_context);
+
+          config = gst_buffer_pool_get_config (pool);
+          gst_buffer_pool_config_set_params (config, caps, size, 0, 0);
+          gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_GL_SYNC_META);
+
+          if (!gst_buffer_pool_set_config (pool, config))
+            {
+              GST_DEBUG_OBJECT (bsink, "failed setting config");
+              gst_object_unref (pool);
+              return FALSE;
+            }
+        }
+
+      /* we need at least 2 buffer because we hold on to the last one */
+      gst_query_add_allocation_pool (query, pool, size, 2, 0);
+      if (pool)
+        gst_object_unref (pool);
+
+      /* we also support various metadata */
+      gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0);
+
+      if (self->gst_context->gl_vtable->FenceSync)
+        gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0);
+
+      return TRUE;
     }
 
-  /* we need at least 2 buffer because we hold on to the last one */
-  gst_query_add_allocation_pool (query, pool, size, 2, 0);
-  if (pool)
-    gst_object_unref (pool);
-
-  /* we also support various metadata */
-  gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0);
-
-  if (self->gst_context->gl_vtable->FenceSync)
-    gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0);
-
-  return TRUE;
+  return FALSE;
 }
 
 static GdkMemoryFormat
@@ -418,11 +432,10 @@ gtk_gst_sink_texture_from_buffer (GtkGstSink      *self,
       g_clear_pointer (&frame, g_free);
 
       g_return_val_if_fail (vmeta, NULL);
-      g_return_val_if_fail (self->gdk_context, NULL);
       g_return_val_if_fail (self->drm_info.drm_fourcc != DRM_FORMAT_INVALID, NULL);
 
       builder = gdk_dmabuf_texture_builder_new ();
-      gdk_dmabuf_texture_builder_set_display (builder, gdk_gl_context_get_display (self->gdk_context));
+      gdk_dmabuf_texture_builder_set_display (builder, self->gdk_display);
       gdk_dmabuf_texture_builder_set_fourcc (builder, self->drm_info.drm_fourcc);
       gdk_dmabuf_texture_builder_set_modifier (builder, self->drm_info.drm_modifier);
       gdk_dmabuf_texture_builder_set_width (builder, vmeta->width);
@@ -768,6 +781,10 @@ gtk_gst_sink_set_property (GObject      *object,
         g_clear_object (&self->gdk_context);
       break;
 
+    case PROP_DISPLAY:
+      self->gdk_display = g_value_dup_object (value);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -787,10 +804,15 @@ gtk_gst_sink_get_property (GObject    *object,
     case PROP_PAINTABLE:
       g_value_set_object (value, self->paintable);
       break;
+
     case PROP_GL_CONTEXT:
       g_value_set_object (value, self->gdk_context);
       break;
 
+    case PROP_DISPLAY:
+      g_value_set_object (value, self->gdk_display);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -807,6 +829,7 @@ gtk_gst_sink_dispose (GObject *object)
   g_clear_object (&self->gst_gdk_context);
   g_clear_object (&self->gst_display);
   g_clear_object (&self->gdk_context);
+  g_clear_object (&self->gdk_display);
 
   G_OBJECT_CLASS (gtk_gst_sink_parent_class)->dispose (object);
 }
@@ -851,6 +874,11 @@ gtk_gst_sink_class_init (GtkGstSinkClass * klass)
                          GDK_TYPE_GL_CONTEXT,
                          G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS);
 
+  properties[PROP_DISPLAY] =
+    g_param_spec_object ("display", NULL, NULL,
+                         GDK_TYPE_DISPLAY,
+                         G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS);
+
   g_object_class_install_properties (gobject_class, N_PROPS, properties);
 
   gst_element_class_set_metadata (gstelement_class,

modules/media/gtkgstsinkprivate.h

@@ -49,12 +49,13 @@ struct _GtkGstSink
   GstVideoInfo         v_info;
   GstVideoInfoDmaDrm   drm_info;
 
-  GtkGstPaintable *    paintable;
-  GdkGLContext *       gdk_context;
-  GstGLDisplay *       gst_display;
-  GstGLContext *       gst_gdk_context;
-  GstGLContext *       gst_context;
-  GdkColorState *      color_state;
+  GtkGstPaintable     *paintable;
+  GdkDisplay          *gdk_display;
+  GdkGLContext        *gdk_context;
+  GstGLDisplay        *gst_display;
+  GstGLContext        *gst_gdk_context;
+  GstGLContext        *gst_context;
+  GdkColorState       *color_state;
 };
 
 struct _GtkGstSinkClass