gstreamer1.0-plugins-bad: Add directviv to glimagesink to improve playback performance

1.Add a physical buffer uploader in glupload plugin and using viv direct
  texture to bind physical continious buffer with texture to avoid memory
  copy from videobuffer to texture to gain good performance.

2.Reduce glimagesink load latency by override glimagesink ALLOCATION query to
  avoid memory copy.

Signed-off-by: Yuqing Zhu <carol.zhu@nxp.com>
Signed-off-by: Otavio Salvador <otavio@ossystems.com.br>

2 files changed, 439 insertions, 0 deletions

diff --git a/recipes-multimedia/gstreamer/gstreamer1.0-plugins-bad/0002-Add-directviv-to-glimagesink-to-improve-playback-per.patch b/recipes-multimedia/gstreamer/gstreamer1.0-plugins-bad/0002-Add-directviv-to-glimagesink-to-improve-playback-per.patch
new file mode 100644
index 00000000..43d89060
--- /dev/null
+++ b/recipes-multimedia/gstreamer/gstreamer1.0-plugins-bad/0002-Add-directviv-to-glimagesink-to-improve-playback-per.patch
@@ -0,0 +1,438 @@
+From 94e6bb069cb5207761f2e4234137f2a748f984db Mon Sep 17 00:00:00 2001
+From: Haihua Hu <b55597@freescale.com>
+Date: Tue, 13 Oct 2015 09:33:54 +0800
+Subject: [PATCH 13/18] Add directviv to glimagesink to improve playback
+ performance
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+1.Add a physical buffer uploader in glupload plugin and using viv direct
+  texture to bind physical continious buffer with texture to avoid memory
+  copy from videobuffer to texture to gain good performance.
+2.Reduce glimagesink load latency by override glimagesink ALLOCATION query to
+  avoid memory copy.
+
+Upstream-Status: Inappropriate [i.MX specific]
+
+Signed-off-by: Haihua Hu <b55597@freescale.com>
+Signed-off-by: Lyon Wang <lyon.wang@freescale.com>
+---
+ ext/gl/gstglimagesink.c                 |   5 ++
+ gst-libs/gst/gl/Makefile.am             |   2 +
+ gst-libs/gst/gl/gstglupload.c           | 141 ++++++++++++++++++++++++++++++-
+ gst-libs/gst/gl/gstglvivdirecttexture.c | 143 ++++++++++++++++++++++++++++++++
+ gst-libs/gst/gl/gstglvivdirecttexture.h |  35 ++++++++
+ 5 files changed, 323 insertions(+), 3 deletions(-)
+ create mode 100644 gst-libs/gst/gl/gstglvivdirecttexture.c
+ create mode 100644 gst-libs/gst/gl/gstglvivdirecttexture.h
+
+diff --git a/ext/gl/gstglimagesink.c b/ext/gl/gstglimagesink.c
+index 3b5e3b5..532ea6a 100644
+--- a/ext/gl/gstglimagesink.c
++++ b/ext/gl/gstglimagesink.c
+@@ -911,6 +911,11 @@ gst_glimage_sink_query (GstBaseSink * bsink, GstQuery * query)
+       res = GST_BASE_SINK_CLASS (parent_class)->query (bsink, query);
+       break;
+     }
++    case GST_QUERY_ALLOCATION:
++    {
++      gst_glimage_sink_propose_allocation(bsink, query);
++      break;
++    }
+     default:
+       res = GST_BASE_SINK_CLASS (parent_class)->query (bsink, query);
+       break;
+diff --git a/gst-libs/gst/gl/Makefile.am b/gst-libs/gst/gl/Makefile.am
+index 4bd6511..c396603 100644
+--- a/gst-libs/gst/gl/Makefile.am
++++ b/gst-libs/gst/gl/Makefile.am
+@@ -33,6 +33,7 @@ libgstgl_@GST_API_VERSION@_la_SOURCES = \
+        gstglviewconvert.c \
+        gstgloverlaycompositor.c \
+        gstglquery.c \
++       gstglvivdirecttexture.c \
+        gstglcontrolbindingproxy.c
+ 
+ libgstgl_@GST_API_VERSION@includedir = $(includedir)/gstreamer-@GST_API_VERSION@/gst/gl
+@@ -66,6 +67,7 @@ libgstgl_@GST_API_VERSION@include_HEADERS = \
+        gstglcontrolbindingproxy.h \
+        gstgl_fwd.h \
+        gstgl_enums.h \
++       gstglvivdirecttexture.h \
+        gl.h
+ 
+ noinst_HEADERS = \
+diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
+index 32e6150..99cc68a 100644
+--- a/gst-libs/gst/gl/gstglupload.c
++++ b/gst-libs/gst/gl/gstglupload.c
+@@ -23,6 +23,7 @@
+ #endif
+ 
+ #include <stdio.h>
++#include <gst/gl/gstglvivdirecttexture.h>
+ 
+ #include "gl.h"
+ #include "gstglupload.h"
+@@ -51,7 +52,7 @@
+ #define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
+ #define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
+ 
+-GST_DEBUG_CATEGORY_STATIC (gst_gl_upload_debug);
++GST_DEBUG_CATEGORY (gst_gl_upload_debug);
+ #define GST_CAT_DEFAULT gst_gl_upload_debug
+ 
+ #define DEBUG_INIT \
+@@ -1169,6 +1170,140 @@ static const UploadMethod _upload_meta_upload = {
+   &_upload_meta_upload_free
+ };
+ 
++struct PhyBufferUpload
++{
++  GstGLUpload *upload;
++  GstGLVideoAllocationParams *params;
++};
++
++static gpointer
++_physical_buffer_upload_new(GstGLUpload *upload)
++{
++  struct PhyBufferUpload *phybuffer = g_new0 (struct PhyBufferUpload, 1);
++
++  phybuffer->upload = upload;
++
++  return phybuffer;
++}
++
++static GstCaps *
++_physical_buffer_upload_transform_caps(GstGLContext *context,
++    GstPadDirection direction, GstCaps *caps)
++{
++  GstCapsFeatures *passthrough =
++      gst_caps_features_from_string
++      (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
++  GstCaps *ret;
++
++  if (direction == GST_PAD_SINK) {
++    GstCaps *tmp;
++
++    ret =
++        _set_caps_features_with_passthrough (caps,
++        GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
++
++    gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL);
++    tmp = _caps_intersect_texture_target (ret, 1 << GST_GL_TEXTURE_TARGET_2D);
++    gst_caps_unref (ret);
++    ret = tmp;
++  } else {
++    ret = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
++            (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY,GST_GL_DIRECTVIV_FORMAT));
++  }
++
++  gst_caps_features_free (passthrough);
++  return ret;
++}
++
++static gboolean
++_physical_buffer_upload_accept(gpointer impl, GstBuffer *buffer, 
++    GstCaps *in_caps, GstCaps *out_caps)
++{
++  struct PhyBufferUpload *upload = impl;  
++  GstCapsFeatures *features;
++
++  features = gst_caps_get_features (out_caps, 0);
++  if (!gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
++    return FALSE;
++  
++  if (upload->params)
++    gst_gl_allocation_params_free ((GstGLAllocationParams *) upload->params);
++  if (!(upload->params =
++          gst_gl_video_allocation_params_new (upload->upload->context, NULL,
++              &upload->upload->priv->in_info, -1, NULL,
++              GST_GL_TEXTURE_TARGET_2D)))
++    return FALSE;
++
++  return gst_is_physical_buffer(buffer);
++}
++
++static void
++_physical_buffer_upload_propose_allocation(gpointer impl, GstQuery *decide_query,
++    GstQuery *query)
++{
++  gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0);
++}
++
++static GstGLUploadReturn
++_physical_buffer_upload_perform(gpointer impl, GstBuffer *buffer, GstBuffer **outbuf)
++{
++  struct PhyBufferUpload *phyBuffer = impl;
++  GstGLMemoryAllocator *allocator;
++  GstVideoInfo *info;
++  gint n_mem;
++
++  info = &phyBuffer->upload->priv->out_info;
++  n_mem = GST_VIDEO_INFO_N_PLANES (info);
++  GST_LOG_OBJECT (phyBuffer->upload, "Attempting viv direct upload");
++  
++  allocator =
++      GST_GL_MEMORY_ALLOCATOR (gst_allocator_find
++      (GST_GL_MEMORY_PBO_ALLOCATOR_NAME));
++
++  /* FIXME: buffer pool */
++  *outbuf = gst_buffer_new ();
++  gst_gl_memory_setup_buffer (allocator, *outbuf, phyBuffer->params);
++  gst_object_unref (allocator);
++
++  GstGLMemory *out_gl_mem =
++      (GstGLMemory *) gst_buffer_peek_memory (*outbuf, 0);
++
++  gst_gl_viv_direct_bind_gstbuffer(phyBuffer->upload->context, out_gl_mem->tex_id,
++      &phyBuffer->upload->priv->in_info, buffer);
++
++  gst_buffer_add_video_meta_full (*outbuf, 0,
++      GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info),
++      GST_VIDEO_INFO_HEIGHT (info), n_mem, info->offset, info->stride);
++
++  return GST_GL_UPLOAD_DONE;
++}
++
++static void
++_physical_buffer_upload_free(gpointer impl)
++{
++  struct PhyBufferUpload *phyBuffer = impl;
++
++  if (phyBuffer->params)
++    gst_gl_allocation_params_free ((GstGLAllocationParams *) phyBuffer->params);
++
++  g_free(phyBuffer);
++}
++
++static GstStaticCaps _physical_buffer_upload_caps =
++GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (GST_GL_DIRECTVIV_FORMAT));
++
++static const UploadMethod _physical_buffer_upload = {
++  "Physical buffer",
++  0,
++  &_physical_buffer_upload_caps,
++  &_physical_buffer_upload_new,
++  &_physical_buffer_upload_transform_caps,
++  &_physical_buffer_upload_accept,
++  &_physical_buffer_upload_propose_allocation,
++  &_physical_buffer_upload_perform,
++  &_physical_buffer_upload_free
++};
++
+ struct RawUploadFrame
+ {
+   gint ref_count;
+@@ -1391,7 +1526,7 @@ static const UploadMethod *upload_methods[] = { &_gl_memory_upload,
+ #if GST_GL_HAVE_DMABUF
+   &_dma_buf_upload,
+ #endif
+-  &_upload_meta_upload, &_raw_data_upload
++  &_upload_meta_upload, &_physical_buffer_upload, &_raw_data_upload
+ };
+ 
+ static GMutex upload_global_lock;
+@@ -1514,7 +1649,7 @@ gst_gl_upload_transform_caps (GstGLContext * context, GstPadDirection direction,
+   }
+ 
+   if (filter) {
+-    result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
++    result = gst_caps_intersect_full (tmp, filter, GST_CAPS_INTERSECT_FIRST);
+     gst_caps_unref (tmp);
+   } else {
+     result = tmp;
+diff --git a/gst-libs/gst/gl/gstglvivdirecttexture.c b/gst-libs/gst/gl/gstglvivdirecttexture.c
+new file mode 100644
+index 0000000..c19b617
+--- /dev/null
++++ b/gst-libs/gst/gl/gstglvivdirecttexture.c
+@@ -0,0 +1,143 @@
++/*
++ * GStreamer
++ * Copyright (c) 2015, Freescale Semiconductor, Inc. 
++ *
++ * This library is free software; you can redistribute it and/or
++ * modify it under the terms of the GNU Library General Public
++ * License as published by the Free Software Foundation; either
++ * version 2 of the License, or (at your option) any later version.
++ *
++ * This library is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
++ * Library General Public License for more details.
++ *
++ * You should have received a copy of the GNU Library General Public
++ * License along with this library; if not, write to the
++ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
++ * Boston, MA 02110-1301, USA.
++ */
++
++#ifdef HAVE_CONFIG_H
++#include "config.h"
++#endif
++
++#include "gl.h"
++
++GST_DEBUG_CATEGORY_EXTERN (gst_gl_upload_debug);
++#define GST_CAT_DEFAULT gst_gl_upload_debug
++
++typedef struct {
++  guint tex_id;
++  guint w;
++  guint h;
++  guint fmt;
++  void *vaddr;
++  guint paddr;
++  gboolean ret;
++} GstVivDirectTexture; 
++
++gboolean
++gst_is_physical_buffer (GstBuffer *buffer)
++{
++
++  GstMemory *mem;
++
++  mem = gst_buffer_peek_memory (buffer, 0);
++  if (!mem->allocator)
++    return FALSE;
++
++  return g_type_check_instance_is_a (mem->allocator, g_type_from_name("GstAllocatorPhyMem"));
++}
++
++static void
++_do_viv_direct_tex_bind_mem (GstGLContext * context, GstVivDirectTexture * viv_tex)
++{
++  GST_DEBUG ("viv direct upload, tex_id %d, fmt: %d, res: (%dx%d)", viv_tex->tex_id, viv_tex->fmt, viv_tex->w, viv_tex->h);
++  GST_DEBUG ("Physical memory buffer, vaddr: %p, paddr: %p", viv_tex->vaddr, viv_tex->paddr);
++
++  glBindTexture (GL_TEXTURE_2D, viv_tex->tex_id);
++  glTexDirectVIVMap (GL_TEXTURE_2D, viv_tex->w, viv_tex->h, viv_tex->fmt, &viv_tex->vaddr, &viv_tex->paddr);
++  glTexDirectInvalidateVIV (GL_TEXTURE_2D);
++  viv_tex->ret = TRUE;
++
++  return;
++}
++
++gboolean
++gst_gl_viv_direct_bind_gstbuffer (GstGLContext * context, guint tex_id, GstVideoInfo * info, GstBuffer * buffer)
++{
++  typedef struct {
++    guint8 *vaddr;
++    guint8 *paddr;
++    guint8 *caddr;
++    gsize size;
++    gpointer *user_data;
++  } PhyMemBlock;
++  //Note: structure PhyMemBlock is copied from gst1.0-fsl-plugin/libs/allocator/gstallocatorphymem.h
++
++  typedef struct {
++    GstMemory mem;
++    guint8 *vaddr;
++    guint8 *paddr;
++    PhyMemBlock block;
++  } GstMemoryPhy;
++  //Note: structure GstMemoryPhy is copied from gst1.0-fsl-plugin/libs/allocator/gstallocatorphymem.c
++
++  GstMemory *mem = gst_buffer_peek_memory (buffer, 0);
++  GstMemoryPhy *memphy = (GstMemoryPhy*) mem;
++  PhyMemBlock *memblk = &memphy->block;
++
++  GstVideoFormat fmt = GST_VIDEO_INFO_FORMAT (info);
++  gint width, height;
++  GstVideoMeta *vmeta = gst_buffer_get_video_meta (buffer);
++  if (vmeta && (fmt == GST_VIDEO_FORMAT_I420 || fmt == GST_VIDEO_FORMAT_NV12)) {
++    width = vmeta->stride[0];
++    height = vmeta->offset[1] / width;
++  }
++  else {
++    width = GST_VIDEO_INFO_WIDTH (info);
++    height = GST_VIDEO_INFO_HEIGHT (info);
++  }
++
++  guint viv_fmt;
++  switch (fmt) {
++    case GST_VIDEO_FORMAT_I420:
++      viv_fmt = GL_VIV_I420;
++      break;
++    case GST_VIDEO_FORMAT_YV12:
++      viv_fmt = GL_VIV_YV12;
++      break;
++    case GST_VIDEO_FORMAT_NV12:
++      viv_fmt = GL_VIV_NV12;
++      break;
++    case GST_VIDEO_FORMAT_NV21:
++      viv_fmt = GL_VIV_NV21;
++      break;
++    case GST_VIDEO_FORMAT_YUY2:
++      viv_fmt = GL_VIV_YUY2;
++      break;
++    case GST_VIDEO_FORMAT_UYVY:
++      viv_fmt = GL_VIV_UYVY;
++      break;
++    case GST_VIDEO_FORMAT_RGBA:
++      viv_fmt = GL_RGBA;
++      break;
++    case GST_VIDEO_FORMAT_BGRA:
++      viv_fmt = GL_BGRA_EXT;
++      break;
++    case GST_VIDEO_FORMAT_RGB16:
++      viv_fmt = GL_RGB565_OES;
++      break;
++    default:
++      GST_ERROR ("Not supported format %d for viv direct texture upload.", fmt);
++      viv_fmt = GL_NONE;
++      return FALSE;
++  }
++
++  GstVivDirectTexture viv_tex = {tex_id, width, height, viv_fmt, memblk->vaddr, memblk->paddr, FALSE};
++  gst_gl_context_thread_add (context, (GstGLContextThreadFunc) _do_viv_direct_tex_bind_mem, &viv_tex);
++
++  return viv_tex.ret;
++}
++
+diff --git a/gst-libs/gst/gl/gstglvivdirecttexture.h b/gst-libs/gst/gl/gstglvivdirecttexture.h
+new file mode 100644
+index 0000000..fa88e1a
+--- /dev/null
++++ b/gst-libs/gst/gl/gstglvivdirecttexture.h
+@@ -0,0 +1,35 @@
++/*
++ * GStreamer
++ * Copyright (c) 2015, Freescale Semiconductor, Inc. 
++ *
++ * This library is free software; you can redistribute it and/or
++ * modify it under the terms of the GNU Library General Public
++ * License as published by the Free Software Foundation; either
++ * version 2 of the License, or (at your option) any later version.
++ *
++ * This library is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
++ * Library General Public License for more details.
++ *
++ * You should have received a copy of the GNU Library General Public
++ * License along with this library; if not, write to the
++ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
++ * Boston, MA 02110-1301, USA.
++ */
++
++#ifndef __GST_GL_VIVDIRECT_H__
++#define __GST_GL_VIVDIRECT_H__
++
++#include <gst/video/video.h>
++#include <gst/gl/gstgl_fwd.h>
++
++#define GST_GL_DIRECTVIV_FORMAT "{RGBA, I420, YV12, NV12, NV21, YUY2, UYVY, BGRA, RGB16}"
++G_BEGIN_DECLS
++
++gboolean gst_is_physical_buffer (GstBuffer *buffer);
++gboolean gst_gl_viv_direct_bind_gstbuffer (GstGLContext * context, guint tex_id, GstVideoInfo * info, GstBuffer * buffer);
++
++G_END_DECLS
++
++#endif /* __GST_GL_VIVDIRECT_H__ */
+-- 
+1.9.1
+
diff --git a/recipes-multimedia/gstreamer/gstreamer1.0-plugins-bad_%.bbappend b/recipes-multimedia/gstreamer/gstreamer1.0-plugins-bad_%.bbappend
index 98e8d144..16d0ce84 100644
--- a/recipes-multimedia/gstreamer/gstreamer1.0-plugins-bad_%.bbappend
+++ b/recipes-multimedia/gstreamer/gstreamer1.0-plugins-bad_%.bbappend
@@ -11,6 +11,7 @@ PACKAGECONFIG_GL_mx6sl = "${@bb.utils.contains('DISTRO_FEATURES', 'opengl', \
                                     'opengl', '', d), '', d)}"
 
 SRC_URI_append = " file://0001-glplugin-Change-wayland-default-res-to-1024x768.patch \
+                   file://0002-Add-directviv-to-glimagesink-to-improve-playback-per.patch \
 "