feat(linux): add zero-copy v4l2 encoder

cmake/compile_definitions/linux.cmake

@@ -124,6 +124,22 @@ if(LIBVA_FOUND)
             "${CMAKE_SOURCE_DIR}/src/platform/linux/vaapi.cpp")
 endif()
 
+# v4l2
+if(${SUNSHINE_ENABLE_V4L2})
+    include(CheckIncludeFiles)
+    check_include_files("linux/videodev2.h" V4L2_FOUND)
+else()
+    set(V4L2_FOUND OFF)
+endif()
+if(V4L2_FOUND)
+    add_compile_definitions(SUNSHINE_BUILD_V4L2)
+    list(APPEND PLATFORM_TARGET_FILES
+            "${CMAKE_SOURCE_DIR}/src/platform/linux/v4l2_wrapper.h"
+            "${CMAKE_SOURCE_DIR}/src/platform/linux/v4l2_wrapper.c"
+            "${CMAKE_SOURCE_DIR}/src/platform/linux/v4l2.h"
+            "${CMAKE_SOURCE_DIR}/src/platform/linux/v4l2.cpp")
+endif()
+
 # vulkan video encoding (via FFmpeg)
 if(${SUNSHINE_ENABLE_VULKAN})
     if(NOT SUNSHINE_SYSTEM_VULKAN_HEADERS)

cmake/prep/options.cmake

@@ -62,6 +62,8 @@ elseif(UNIX)  # Linux
             "Enable KMS grab if available." ON)
     option(SUNSHINE_ENABLE_VAAPI
             "Enable building vaapi specific code." ON)
+    option(SUNSHINE_ENABLE_V4L2
+            "Enable building v4l2 specific code." ON)
     option(SUNSHINE_ENABLE_VULKAN
             "Enable Vulkan video encoding." ON)
     option(SUNSHINE_ENABLE_WAYLAND

docs/configuration.md

@@ -2255,7 +2255,7 @@ supported on the current platform.
             @endcode</td>
     </tr>
     <tr>
-        <td rowspan="5">Choices</td>
+        <td rowspan="7">Choices</td>
         <td>nvenc</td>
         <td>For NVIDIA graphics cards</td>
     </tr>
@@ -2276,6 +2276,11 @@ supported on the current platform.
         <td>Use Vulkan encoder (AMD, Intel, NVIDIA).
             @note{Applies to Linux only.}</td>
     </tr>
+    <tr>
+        <td>v4l2</td>
+        <td>Use V4L2 encoder.
+            @note{Applies to Linux only.}</td>
+    </tr>
     <tr>
         <td>software</td>
         <td>Encoding occurs on the CPU</td>

src/cbs.cpp

@@ -260,38 +260,39 @@
   }
 
   /**
-   * @brief Check whether an encoded H.264 or HEVC packet contains active SPS VUI metadata.
+   * @brief Check whether the given encoded H.264 or HEVC packets contain active SPS VUI metadata.
    *
-   * @param packet Encoded packet to parse with FFmpeg's coded bitstream reader.
+   * @param packets Encoded packets to parse with FFmpeg's coded bitstream reader.
    * @param codec_id FFmpeg codec identifier; expected to be AV_CODEC_ID_H264 or AV_CODEC_ID_H265.
    * @return `true` when the packet's active SPS advertises VUI parameters.
    */
-  bool validate_sps(const AVPacket *packet, int codec_id) {
+  bool validate_sps(const std::vector<AVPacket *> &packets, int codec_id) {
     cbs::ctx_t ctx;
     if (ff_cbs_init(&ctx, (AVCodecID) codec_id, nullptr)) {
       return false;
     }
 
-    cbs::frag_t frag;
+    for (const auto &packet : packets) {
+      cbs::frag_t frag;
 
-    int err = ff_cbs_read_packet(ctx.get(), &frag, packet);
-    if (err < 0) {
-      char err_str[AV_ERROR_MAX_STRING_SIZE] {0};
-      BOOST_LOG(error) << "Couldn't read packet: "sv << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, err);
+      int err = ff_cbs_read_packet(ctx.get(), &frag, packet);
+      if (err < 0) {
+        char err_str[AV_ERROR_MAX_STRING_SIZE] {0};
+        BOOST_LOG(error) << "Couldn't read packet: "sv << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, err);
 
-      return false;
+        return false;
+      }
     }
 
     if (codec_id == AV_CODEC_ID_H264) {
       auto h264 = (CodedBitstreamH264Context *) ctx->priv_data;
-
-      if (!h264->active_sps->vui_parameters_present_flag) {
-        return false;
-      }
-
-      return true;
+      return h264 && h264->active_sps && h264->active_sps->vui_parameters_present_flag;
+    } else if (codec_id == AV_CODEC_ID_H265) {
+      auto h265 = (CodedBitstreamH265Context *) ctx->priv_data;
+      return h265 && h265->active_sps && h265->active_sps->vui_parameters_present_flag;
+    } else {
+      BOOST_LOG(error) << "Unsupported codec ID for validate_sps: "sv << codec_id;
+      return false;
     }
-
-    return ((CodedBitstreamH265Context *) ctx->priv_data)->active_sps->vui_parameters_present_flag;
   }
 }  // namespace cbs

src/cbs.h

@@ -53,11 +53,11 @@ namespace cbs {
   h264_t make_sps_h264(const AVCodecContext *ctx, const AVPacket *packet);
 
   /**
-   * @brief Check whether an encoded H.264 or HEVC packet contains active SPS VUI metadata.
+   * @brief Check whether the given encoded H.264 or HEVC packets contain active SPS VUI metadata.
    *
-   * @param packet Encoded packet to parse with FFmpeg's coded bitstream reader.
+   * @param packets Encoded packets to parse with FFmpeg's coded bitstream reader.
    * @param codec_id FFmpeg codec identifier; expected to be AV_CODEC_ID_H264 or AV_CODEC_ID_H265.
    * @return `true` when the packet's active SPS advertises VUI parameters.
    */
-  bool validate_sps(const AVPacket *packet, int codec_id);
+  bool validate_sps(const std::vector<AVPacket *> &packets, int codec_id);
 }  // namespace cbs

src/platform/common.h

@@ -41,6 +41,17 @@ struct AVHWFramesContext;
 struct AVCodecContext;
 struct AVDictionary;
 
+extern "C" {
+/**
+ * @brief Submit a frame to an opened FFmpeg encoder.
+ *
+ * @param avctx FFmpeg codec context that receives the frame.
+ * @param frame Frame to submit, or nullptr to flush the encoder.
+ * @return 0 on success, AVERROR(EAGAIN) when output must be drained, or another negative AVERROR code on failure.
+ */
+int avcodec_send_frame(AVCodecContext *avctx, const AVFrame *frame);
+}
+
 #ifdef _WIN32
 // Forward declarations of boost classes to avoid having to include boost headers
 // here, which results in issues with Windows.h and WinSock2.h include order.
@@ -331,6 +342,7 @@ namespace platf {
     cuda,  ///< CUDA
     videotoolbox,  ///< VideoToolbox
     vulkan,  ///< Vulkan
+    v4l2,  ///< V4L2
     unknown  ///< Unknown
   };
 
@@ -647,6 +659,28 @@ namespace platf {
     virtual int prepare_to_derive_context(int hw_device_type) {
       return 0;
     };
+
+    /**
+     * @brief Load the opened context.
+     * @note Implementations may load the opened context after initialization.
+     *
+     * @param ctx FFmpeg codec context to load.
+     * @return Status from loading the opened context; 0 on success, nonzero to abort.
+     */
+    virtual int load_opened_context(AVCodecContext *ctx) {
+      return 0;
+    }
+
+    /**
+     * @brief Send a frame to the codec.
+     * @note Implementations may perform additional operations besides avcodec_send_frame().
+     *
+     * @param ctx FFmpeg codec context to use.
+     * @return Status from sending the frame; 0 on success, nonzero to abort.
+     */
+    virtual int send_frame(AVCodecContext *ctx) {
+      return avcodec_send_frame(ctx, frame);
+    }
   };
 
   /**

src/platform/linux/graphics.cpp

@@ -64,6 +64,9 @@
     return egl_image_target_texture_2d_fn;
   }
 
+  PFNEGLCREATEIMAGEPROC egl_create_image = nullptr;  ///< Selected core or KHR EGL image-creation entry point.
+  PFNEGLDESTROYIMAGEPROC egl_destroy_image = nullptr;  ///< Selected core or KHR EGL image-destruction entry point.
+
   /**
    * @brief Drain and log pending OpenGL errors.
    */
@@ -335,6 +338,32 @@
 }  // namespace gbm
 
 namespace egl {
+  /**
+   * @brief Owning type-erased EGL attribute array with a type-aware deleter.
+   *
+   * The deleter restores the EGL attribute element type used for allocation.
+   */
+  using egl_attribs_t = std::unique_ptr<void, void (*)(void *)>;
+
+  /**
+   * @brief Function pointer type for constructing attributes for the selected EGL image API.
+   *
+   * @param surface DMA-BUF surface descriptor to translate.
+   * @return Type-erased EGL attribute array terminated by EGL_NONE.
+   */
+  using surface_descriptor_to_egl_attribs_fn = egl_attribs_t (*)(const surface_descriptor_t &surface);
+
+  surface_descriptor_to_egl_attribs_fn surface_descriptor_to_egl_attribs;  ///< Attribute builder selected for the active EGL image API.
+
+  /**
+   * @brief Build EGL DMA-BUF import attributes using a specific EGL attribute type.
+   *
+   * @tparam EglType EGLAttrib for the core API or EGLint for the KHR API.
+   * @param surface DMA-BUF surface descriptor to translate.
+   * @return Type-erased EGL attribute array terminated by EGL_NONE.
+   */
+  template<typename EglType>
+  egl_attribs_t surface_descriptor_to_egl_attribs_impl(const surface_descriptor_t &surface);
 
   namespace {  // Keep privileged implementation details anonymous/local to this translation unit
 
@@ -639,6 +668,19 @@
       BOOST_LOG(warning) << "GL: glEGLImageTargetTexture2DOES not available; DMA-BUF import will fail"sv;
     }
 
+    if (eglCreateImage != nullptr && eglDestroyImage != nullptr) {
+      gl::egl_create_image = (gl::PFNEGLCREATEIMAGEPROC) eglCreateImage;
+      gl::egl_destroy_image = (gl::PFNEGLDESTROYIMAGEPROC) eglDestroyImage;
+      egl::surface_descriptor_to_egl_attribs = surface_descriptor_to_egl_attribs_impl<EGLAttrib>;
+    } else if (eglCreateImageKHR != nullptr && eglDestroyImageKHR != nullptr) {
+      gl::egl_create_image = (gl::PFNEGLCREATEIMAGEPROC) eglCreateImageKHR;
+      gl::egl_destroy_image = (gl::PFNEGLDESTROYIMAGEPROC) eglDestroyImageKHR;
+      egl::surface_descriptor_to_egl_attribs = surface_descriptor_to_egl_attribs_impl<EGLint>;
+    } else {
+      BOOST_LOG(error) << "Neither eglCreateImage/eglDestroyImage nor eglCreateImageKHR/eglDestroyImageKHR are available"sv;
+      return std::nullopt;
+    }
+
     // GetString returns const GLubyte* (unsigned char*); convert to std::string safely (avoids sonar cpp:S6996).
     auto gl_string = [](const GLubyte *s) {
       std::string result;
@@ -719,12 +761,42 @@
   }
 
   /**
-   * @brief Get EGL attributes for eglCreateImage() to import the provided surface.
-   * @param surface The surface descriptor.
-   * @return Vector of EGL attributes.
+   * @brief Build EGL DMA-BUF import attributes using a specific EGL attribute type.
+   *
+   * eglCreateImage uses EGLAttrib while eglCreateImageKHR uses EGLint, so the
+   * selected specialization controls both allocation and type-aware deletion.
+   *
+   * @tparam EglType EGLAttrib for the core API or EGLint for the KHR API.
+   * @param surface DMA-BUF surface descriptor to translate.
+   * @return Type-erased EGL attribute array terminated by EGL_NONE.
    */
-  std::vector<EGLAttrib> surface_descriptor_to_egl_attribs(const surface_descriptor_t &surface) {
-    std::vector<EGLAttrib> attribs;
+  template<typename EglType>
+  egl_attribs_t surface_descriptor_to_egl_attribs_impl(const surface_descriptor_t &surface) {
+    // Use a custom vector to abstract away the underlying element type.
+    struct {
+      static_assert(std::is_trivially_copyable_v<EglType>);
+      static_assert(std::is_trivially_destructible_v<EglType>);
+
+      static void deleter(void *ptr) {
+        delete[] reinterpret_cast<EglType *>(ptr);
+      };
+
+      size_t len = 0;
+      size_t cap = 32;
+      egl_attribs_t buf {new EglType[cap], &deleter};
+
+      void emplace_back(EglType value) {
+        if (len == cap) {
+          size_t new_cap = cap + 32;
+          egl_attribs_t new_buf {new EglType[new_cap], &deleter};
+          memcpy(new_buf.get(), buf.get(), len * sizeof(EglType));
+          buf = std::move(new_buf);
+          cap = new_cap;
+        }
+        reinterpret_cast<EglType *>(buf.get())[len] = value;
+        ++len;
+      }
+    } attribs;
 
     attribs.emplace_back(EGL_WIDTH);
     attribs.emplace_back(surface.width);
@@ -757,9 +829,14 @@
     }
 
     attribs.emplace_back(EGL_NONE);
-    return attribs;
+    return std::move(attribs.buf);
   }
 
+  /** @copydoc surface_descriptor_to_egl_attribs_impl */
+  template egl_attribs_t surface_descriptor_to_egl_attribs_impl<EGLAttrib>(const surface_descriptor_t &surface);
+  /** @copydoc surface_descriptor_to_egl_attribs_impl */
+  template egl_attribs_t surface_descriptor_to_egl_attribs_impl<EGLint>(const surface_descriptor_t &surface);
+
   /**
    * @brief Import the source frame texture for EGL/OpenGL conversion.
    *
@@ -768,11 +845,11 @@
    * @return Imported RGB image, or empty when import fails.
    */
   std::optional<rgb_t> import_source(display_t::pointer egl_display, const surface_descriptor_t &xrgb) {
-    auto attribs = surface_descriptor_to_egl_attribs(xrgb);
+    egl_attribs_t attribs = surface_descriptor_to_egl_attribs(xrgb);
 
     rgb_t rgb {
       egl_display,
-      eglCreateImage(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, attribs.data()),
+      gl::egl_create_image(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, attribs.get()),
       gl::tex_t::make(1)
     };
 
@@ -877,13 +954,13 @@
    * @return Imported NV12 image, or empty when import fails.
    */
   std::optional<nv12_t> import_target(display_t::pointer egl_display, std::array<file_t, nv12_img_t::num_fds> &&fds, const surface_descriptor_t &y, const surface_descriptor_t &uv) {
-    auto y_attribs = surface_descriptor_to_egl_attribs(y);
-    auto uv_attribs = surface_descriptor_to_egl_attribs(uv);
+    egl_attribs_t y_attribs = surface_descriptor_to_egl_attribs(y);
+    egl_attribs_t uv_attribs = surface_descriptor_to_egl_attribs(uv);
 
     nv12_t nv12 {
       egl_display,
-      eglCreateImage(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, y_attribs.data()),
-      eglCreateImage(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, uv_attribs.data()),
+      gl::egl_create_image(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, y_attribs.get()),
+      gl::egl_create_image(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, uv_attribs.get()),
       gl::tex_t::make(2),
       gl::frame_buf_t::make(2),
       std::move(fds)
@@ -935,9 +1012,9 @@
 
     yuv444_t yuv444 {
       egl_display,
-      eglCreateImage(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, y_attribs.data()),
-      eglCreateImage(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, u_attribs.data()),
-      eglCreateImage(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, v_attribs.data()),
+      gl::egl_create_image(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, y_attribs.get()),
+      gl::egl_create_image(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, u_attribs.get()),
+      gl::egl_create_image(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, v_attribs.get()),
       gl::tex_t::make(3),
       gl::frame_buf_t::make(3),
       std::move(fds)
@@ -1527,6 +1604,35 @@
 
     return 0;
   }
+
+  int sws_t::copy(gl::frame_buf_t &dst, gl::frame_buf_t &src) {
+    for (int plane = 0; plane < 2; ++plane) {
+      const int plane_width = out_width / (plane + 1);  // UV plane is half the width of the Y plane
+      const int plane_height = out_height / (plane + 1);
+      const GLenum attachment = GL_COLOR_ATTACHMENT0 + plane;
+
+      gl::ctx.BindFramebuffer(GL_READ_FRAMEBUFFER, src[plane]);
+      gl::ctx.ReadBuffer(attachment);
+
+      gl::ctx.BindFramebuffer(GL_DRAW_FRAMEBUFFER, dst[plane]);
+      gl::ctx.DrawBuffers(1, &attachment);
+
+#ifndef NDEBUG
+      if (gl::ctx.CheckFramebufferStatus(GL_READ_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE || gl::ctx.CheckFramebufferStatus(GL_DRAW_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) {
+        BOOST_LOG(error) << "Copy: incomplete framebuffer for NV12 plane "sv << plane;
+        return -1;
+      }
+#endif
+
+      gl::ctx.BlitFramebuffer(0, 0, plane_width, plane_height, 0, 0, plane_width, plane_height, GL_COLOR_BUFFER_BIT, GL_NEAREST);
+    }
+
+    gl::ctx.BindFramebuffer(GL_FRAMEBUFFER, 0);
+
+    gl::ctx.Flush();
+
+    return 0;
+  }
 }  // namespace egl
 
 /**