fix: resolve SHM hang, DRM device mismatch, and duplicate VAAPI context

BUG-2 (HIGH): SHM Buffer event caused permanent hang
  In the ZwlrScreencopyFrameV1 dispatcher, receiving a SHM Buffer event
  left in_flight_surface stuck at AllocQueued forever, preventing
  queue_alloc_frame() from requesting new frames.
  Fix: treat Buffer as a metadata offer (v3 protocol), wait for
  BufferDone to decide failure, and add AllocQueued state guard to
  LinuxDmabuf handler.

BUG-3 (MEDIUM): Portal backend picked wrong GPU on multi-GPU systems
  state_portal.rs hardcoded /dev/dri/renderD128 then renderD129, which
  selects the wrong GPU when PipeWire uses a different device.
  Fix: extract find_drm_render_nodes() as shared utility; defer DRM
  device selection to first PipeWire frame; test each candidate with
  av_hwframe_transfer_data to find the GPU that can actually import
  the DMA-BUF frame.

BUG-4 (LOW): VAAPI device context created twice unnecessarily
  try_finalize_output() created an AvHwDevCtx stored in EverythingButFmt,
  but negotiate_format() discarded it (_hw_device_ctx) and EncState::new
  created a new one.
  Fix: thread the existing hw_device_ctx through negotiate_format() and
  create_encoder() to EncState::new() which reuses it when provided.
This commit is contained in:
dailz
2026-05-25 14:32:58 +08:00
parent 460a3ee711
commit 5100d78aa8
3 changed files with 183 additions and 43 deletions
+80 -3
View File
@@ -1,4 +1,6 @@
use std::ffi::CString;
use std::mem;
use std::os::fd::AsRawFd;
use std::path::Path;
use std::ptr;
@@ -7,6 +9,7 @@ use ffmpeg_next as ff;
use ffmpeg_next::ffi;
use ffmpeg_next::packet::Mut as _;
use crate::cap_portal::PwDmaBufFrame;
use crate::transform::{transpose_if_transform_transposed, Transform};
// ---------------------------------------------------------------------------
@@ -123,6 +126,74 @@ impl Drop for AvHwFrameCtx {
}
}
/// Test whether `drm_device` can import the PipeWire DMA-BUF frame via VAAPI.
pub fn test_dma_buf_import(drm_device: &Path, frame: &PwDmaBufFrame) -> Result<()> {
let hw_dev = AvHwDevCtx::new_vaapi(drm_device)?;
let frames = AvHwFrameCtx::for_capture(
&hw_dev,
frame.width,
frame.height,
ff::format::Pixel::RGBZ,
)?;
// SAFETY: AVDRMFrameDescriptor is a C POD struct. Zero-initialization is the
// expected FFmpeg setup before filling the fields used below.
let mut desc: ffi::AVDRMFrameDescriptor = unsafe { mem::zeroed() };
desc.nb_objects = 1;
desc.objects[0].fd = frame.fd.as_raw_fd();
desc.objects[0].size = 0;
desc.objects[0].format_modifier = frame.modifier;
desc.nb_layers = 1;
desc.layers[0].format = frame.format;
desc.layers[0].nb_planes = 1;
desc.layers[0].planes[0].object_index = 0;
desc.layers[0].planes[0].offset = frame.offset as isize;
desc.layers[0].planes[0].pitch = frame.stride as isize;
let desc_box = Box::new(desc);
let mut raw_frame = ff::frame::Video::empty();
// SAFETY: raw_frame owns a valid AVFrame. data[0] is used by FFmpeg's
// DRM_PRIME frame convention to point at an AVDRMFrameDescriptor. The Box is
// recovered before every return path below.
unsafe {
let raw_ptr = raw_frame.as_mut_ptr();
(*raw_ptr).data[0] = Box::into_raw(desc_box) as *mut u8;
(*raw_ptr).format = ffi::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32;
(*raw_ptr).width = frame.width as i32;
(*raw_ptr).height = frame.height as i32;
}
let mut hw_frame = ff::frame::Video::empty();
// SAFETY: frames is an initialized AVHWFramesContext and hw_frame is a valid
// writable AVFrame wrapper.
let ret = unsafe { ffi::av_hwframe_get_buffer(frames.as_ptr(), hw_frame.as_mut_ptr(), 0) };
if ret < 0 {
// SAFETY: data[0] still contains the Box pointer installed above.
unsafe {
let _ = Box::from_raw((*raw_frame.as_ptr()).data[0] as *mut ffi::AVDRMFrameDescriptor);
(*raw_frame.as_mut_ptr()).data[0] = ptr::null_mut();
}
bail!("av_hwframe_get_buffer failed: error {ret}");
}
// SAFETY: hw_frame is a valid VAAPI frame allocated from `frames`; raw_frame
// is a DRM_PRIME source frame whose descriptor describes `frame`'s DMA-BUF.
let ret = unsafe { ffi::av_hwframe_transfer_data(hw_frame.as_mut_ptr(), raw_frame.as_ptr(), 0) };
// SAFETY: data[0] still contains the Box pointer installed above. Recover it
// before checking the transfer result so all paths clean up the descriptor.
unsafe {
let _ = Box::from_raw((*raw_frame.as_ptr()).data[0] as *mut ffi::AVDRMFrameDescriptor);
(*raw_frame.as_mut_ptr()).data[0] = ptr::null_mut();
}
if ret < 0 {
bail!("av_hwframe_transfer_data failed: error {ret}");
}
Ok(())
}
// ---------------------------------------------------------------------------
// EncState
// ---------------------------------------------------------------------------
@@ -139,8 +210,8 @@ pub struct EncState {
unsafe impl Send for EncState {}
#[allow(clippy::too_many_arguments)]
impl EncState {
#[allow(clippy::too_many_arguments)]
pub fn new(
drm_device: &Path,
output_path: &Path,
@@ -152,12 +223,16 @@ impl EncState {
gop_size: u32,
fps: u32,
transform: Transform,
existing_hw_ctx: Option<AvHwDevCtx>,
) -> Result<Self> {
tracing::info!(
"EncState::new: {width}x{height} enc={enc_width}x{enc_height} transform={transform:?}"
);
// 1. VAAPI device
let hw_device_ctx = AvHwDevCtx::new_vaapi(drm_device)?;
// 1. VAAPI device — reuse existing context if provided
let hw_device_ctx = match existing_hw_ctx {
Some(ctx) => ctx,
None => AvHwDevCtx::new_vaapi(drm_device)?,
};
// 2. Frame context for capture (XRGB/RGBZ)
let frames_rgb =
@@ -482,6 +557,7 @@ pub fn create_encoder(
transform: Transform,
bitrate: Option<u64>,
gop_size: Option<u32>,
existing_hw_ctx: Option<AvHwDevCtx>,
) -> Result<EncState> {
let (enc_w, enc_h) =
transpose_if_transform_transposed(transform, width as i32, height as i32);
@@ -500,6 +576,7 @@ pub fn create_encoder(
actual_gop_size,
fps,
transform,
existing_hw_ctx,
)
}