[video_player] Add support GstEGLImage to improve playback performance

packages/video_player/elinux/CMakeLists.txt

@@ -9,6 +9,7 @@ set(PLUGIN_NAME "video_player_elinux_plugin")
 find_package(PkgConfig)
 pkg_check_modules(GLIB REQUIRED glib-2.0)
 pkg_check_modules(GSTREAMER REQUIRED gstreamer-1.0)
+pkg_check_modules(GSTREAMER_GL REQUIRED gstreamer-gl-1.0)
 
 add_library(${PLUGIN_NAME} SHARED
   "video_player_elinux_plugin.cc"
@@ -26,12 +27,14 @@ target_include_directories(${PLUGIN_NAME}
   PRIVATE
     ${GLIB_INCLUDE_DIRS}
     ${GSTREAMER_INCLUDE_DIRS}
+    ${GSTREAMER_GL_INCLUDE_DIRS}
 )
 
 target_link_libraries(${PLUGIN_NAME}
   PRIVATE
     ${GLIB_LIBRARIES}
     ${GSTREAMER_LIBRARIES}
+    ${GSTREAMER_GL_LIBRARIES}
 )
 
 # List of absolute paths to libraries that should be bundled with the plugin

packages/video_player/elinux/gst_video_player.cc

@@ -35,6 +35,9 @@ GstVideoPlayer::GstVideoPlayer(
 }
 
 GstVideoPlayer::~GstVideoPlayer() {
+#ifdef USE_EGL_IMAGE_DMABUF
+  UnrefEGLImage();
+#endif // USE_EGL_IMAGE_DMABUF
   Stop();
   DestroyPipeline();
 }
@@ -167,6 +170,50 @@ int64_t GstVideoPlayer::GetCurrentPosition() {
   return position / GST_MSECOND;
 }
 
+#ifdef USE_EGL_IMAGE_DMABUF
+void* GstVideoPlayer::GetEGLImage(void* egl_display, void* egl_context) {
+  std::shared_lock<std::shared_mutex> lock(mutex_buffer_);
+  if (!gst_.buffer) {
+    return nullptr;
+  }
+
+  GstMemory* memory = gst_buffer_peek_memory(gst_.buffer, 0);
+  if (gst_is_dmabuf_memory(memory)) {
+    UnrefEGLImage();
+
+    gint fd = gst_dmabuf_memory_get_fd(memory);
+    gldisplayegl_ = gst_gl_display_egl_new_with_egl_display((gpointer) egl_display);
+    gstglctx_ = gst_gl_context_new_wrapped(
+      GST_GL_DISPLAY_CAST(gldisplayegl_),
+      (guintptr) egl_context,
+      GST_GL_PLATFORM_EGL,
+      GST_GL_API_GLES2);
+
+    gst_gl_context_activate(gstglctx_, TRUE);
+
+    gsteglimage_ = gst_egl_image_from_dmabuf(
+      gstglctx_,
+      fd,
+      &gstVideoInfo_,
+      0,
+      0);
+    return reinterpret_cast<void*>(gst_egl_image_get_image(gsteglimage_));
+  }
+  return nullptr;
+}
+
+void GstVideoPlayer::UnrefEGLImage() {
+  if (gsteglimage_) {
+    gst_egl_image_unref(gsteglimage_);
+    gst_object_unref(gstglctx_);
+    gst_object_unref(gldisplayegl_);
+    gsteglimage_ = NULL;
+    gstglctx_ = NULL;
+    gldisplayegl_ = NULL;
+  }
+}
+#endif // USE_EGL_IMAGE_DMABUF
+
 const uint8_t* GstVideoPlayer::GetFrameBuffer() {
   std::shared_lock<std::shared_mutex> lock(mutex_buffer_);
   if (!gst_.buffer) {
@@ -346,6 +393,14 @@ void GstVideoPlayer::GetVideoSize(int32_t& width, int32_t& height) {
 
   gst_structure_get_int(structure, "width", &width);
   gst_structure_get_int(structure, "height", &height);
+
+#ifdef USE_EGL_IMAGE_DMABUF
+  gboolean res = gst_video_info_from_caps(&gstVideoInfo_, caps);
+  if (!res) {
+    std::cerr << "Failed to get a gst_video_info";
+    return;
+  }
+#endif // USE_EGL_IMAGE_DMABUF
 }
 
 // static

packages/video_player/elinux/gst_video_player.h

@@ -7,6 +7,13 @@
 
 #include <gst/gst.h>
 
+#ifdef USE_EGL_IMAGE_DMABUF
+#include <gst/gl/gl.h>
+#include <gst/gl/egl/egl.h>
+#include <gst/video/video.h>
+#include <gst/allocators/gstdmabuf.h>
+#endif // USE_EGL_IMAGE_DMABUF
+
 #include <memory>
 #include <mutex>
 #include <shared_mutex>
@@ -33,6 +40,9 @@ class GstVideoPlayer {
   int64_t GetDuration();
   int64_t GetCurrentPosition();
   const uint8_t* GetFrameBuffer();
+#ifdef USE_EGL_IMAGE_DMABUF
+  void* GetEGLImage(void* egl_display, void* egl_context);
+#endif // USE_EGL_IMAGE_DMABUF
   int32_t GetWidth() const { return width_; };
   int32_t GetHeight() const { return height_; };
 
@@ -56,6 +66,9 @@ class GstVideoPlayer {
   void DestroyPipeline();
   void Preroll();
   void GetVideoSize(int32_t& width, int32_t& height);
+#ifdef USE_EGL_IMAGE_DMABUF
+  void UnrefEGLImage();
+#endif // USE_EGL_IMAGE_DMABUF
 
   GstVideoElements gst_;
   std::string uri_;
@@ -70,6 +83,13 @@ class GstVideoPlayer {
   std::mutex mutex_event_completed_;
   std::shared_mutex mutex_buffer_;
   std::unique_ptr<VideoPlayerStreamHandler> stream_handler_;
+
+#ifdef USE_EGL_IMAGE_DMABUF
+  GstVideoInfo gstVideoInfo_;
+  GstEGLImage* gsteglimage_ = NULL;
+  GstGLContext* gstglctx_ = NULL;
+  GstGLDisplayEGL* gldisplayegl_ = NULL;
+#endif // USE_EGL_IMAGE_DMABUF
 };
 
 #endif  // PACKAGES_VIDEO_PLAYER_VIDEO_PLAYER_ELINUX_GST_VIDEO_PLAYER_H_

packages/video_player/elinux/video_player_elinux_plugin.cc

@@ -86,6 +86,7 @@ class VideoPlayerPlugin : public flutter::Plugin {
     std::unique_ptr<GstVideoPlayer> player;
     std::unique_ptr<flutter::TextureVariant> texture;
     std::unique_ptr<FlutterDesktopPixelBuffer> buffer;
+    std::unique_ptr<FlutterDesktopEGLImage> egl_image;
     std::unique_ptr<flutter::EventChannel<flutter::EncodableValue>>
         event_channel;
     std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> event_sink;
@@ -293,6 +294,18 @@ void VideoPlayerPlugin::HandleCreateMethodCall(
   }
 
   auto instance = std::make_unique<FlutterVideoPlayer>();
+#ifdef USE_EGL_IMAGE_DMABUF
+  instance->egl_image = std::make_unique<FlutterDesktopEGLImage>();
+  instance->texture =
+      std::make_unique<flutter::TextureVariant>(flutter::EGLImageTexture(
+          [instance = instance.get()](
+              size_t width, size_t height, void* egl_display, void* egl_context) -> const FlutterDesktopEGLImage* {
+            instance->egl_image->width = instance->player->GetWidth();
+            instance->egl_image->height = instance->player->GetHeight();
+            instance->egl_image->egl_image = instance->player->GetEGLImage(egl_display, egl_context);
+            return instance->egl_image.get();
+          }));
+#else
   instance->buffer = std::make_unique<FlutterDesktopPixelBuffer>();
   instance->texture =
       std::make_unique<flutter::TextureVariant>(flutter::PixelBufferTexture(
@@ -303,6 +316,7 @@ void VideoPlayerPlugin::HandleCreateMethodCall(
             instance->buffer->buffer = instance->player->GetFrameBuffer();
             return instance->buffer.get();
           }));
+#endif // USE_EGL_IMAGE_DMABUF
   const auto texture_id =
       texture_registrar_->RegisterTexture(instance->texture.get());
   instance->texture_id = texture_id;