fix: resolve SHM hang, DRM device mismatch, and duplicate VAAPI context
BUG-2 (HIGH): SHM Buffer event caused permanent hang In the ZwlrScreencopyFrameV1 dispatcher, receiving a SHM Buffer event left in_flight_surface stuck at AllocQueued forever, preventing queue_alloc_frame() from requesting new frames. Fix: treat Buffer as a metadata offer (v3 protocol), wait for BufferDone to decide failure, and add AllocQueued state guard to LinuxDmabuf handler. BUG-3 (MEDIUM): Portal backend picked wrong GPU on multi-GPU systems state_portal.rs hardcoded /dev/dri/renderD128 then renderD129, which selects the wrong GPU when PipeWire uses a different device. Fix: extract find_drm_render_nodes() as shared utility; defer DRM device selection to first PipeWire frame; test each candidate with av_hwframe_transfer_data to find the GPU that can actually import the DMA-BUF frame. BUG-4 (LOW): VAAPI device context created twice unnecessarily try_finalize_output() created an AvHwDevCtx stored in EverythingButFmt, but negotiate_format() discarded it (_hw_device_ctx) and EncState::new created a new one. Fix: thread the existing hw_device_ctx through negotiate_format() and create_encoder() to EncState::new() which reuses it when provided.
This commit is contained in:
+80
-3
@@ -1,4 +1,6 @@
|
||||
use std::ffi::CString;
|
||||
use std::mem;
|
||||
use std::os::fd::AsRawFd;
|
||||
use std::path::Path;
|
||||
use std::ptr;
|
||||
|
||||
@@ -7,6 +9,7 @@ use ffmpeg_next as ff;
|
||||
use ffmpeg_next::ffi;
|
||||
use ffmpeg_next::packet::Mut as _;
|
||||
|
||||
use crate::cap_portal::PwDmaBufFrame;
|
||||
use crate::transform::{transpose_if_transform_transposed, Transform};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -123,6 +126,74 @@ impl Drop for AvHwFrameCtx {
|
||||
}
|
||||
}
|
||||
|
||||
/// Test whether `drm_device` can import the PipeWire DMA-BUF frame via VAAPI.
|
||||
pub fn test_dma_buf_import(drm_device: &Path, frame: &PwDmaBufFrame) -> Result<()> {
|
||||
let hw_dev = AvHwDevCtx::new_vaapi(drm_device)?;
|
||||
let frames = AvHwFrameCtx::for_capture(
|
||||
&hw_dev,
|
||||
frame.width,
|
||||
frame.height,
|
||||
ff::format::Pixel::RGBZ,
|
||||
)?;
|
||||
|
||||
// SAFETY: AVDRMFrameDescriptor is a C POD struct. Zero-initialization is the
|
||||
// expected FFmpeg setup before filling the fields used below.
|
||||
let mut desc: ffi::AVDRMFrameDescriptor = unsafe { mem::zeroed() };
|
||||
desc.nb_objects = 1;
|
||||
desc.objects[0].fd = frame.fd.as_raw_fd();
|
||||
desc.objects[0].size = 0;
|
||||
desc.objects[0].format_modifier = frame.modifier;
|
||||
desc.nb_layers = 1;
|
||||
desc.layers[0].format = frame.format;
|
||||
desc.layers[0].nb_planes = 1;
|
||||
desc.layers[0].planes[0].object_index = 0;
|
||||
desc.layers[0].planes[0].offset = frame.offset as isize;
|
||||
desc.layers[0].planes[0].pitch = frame.stride as isize;
|
||||
|
||||
let desc_box = Box::new(desc);
|
||||
let mut raw_frame = ff::frame::Video::empty();
|
||||
// SAFETY: raw_frame owns a valid AVFrame. data[0] is used by FFmpeg's
|
||||
// DRM_PRIME frame convention to point at an AVDRMFrameDescriptor. The Box is
|
||||
// recovered before every return path below.
|
||||
unsafe {
|
||||
let raw_ptr = raw_frame.as_mut_ptr();
|
||||
(*raw_ptr).data[0] = Box::into_raw(desc_box) as *mut u8;
|
||||
(*raw_ptr).format = ffi::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32;
|
||||
(*raw_ptr).width = frame.width as i32;
|
||||
(*raw_ptr).height = frame.height as i32;
|
||||
}
|
||||
|
||||
let mut hw_frame = ff::frame::Video::empty();
|
||||
// SAFETY: frames is an initialized AVHWFramesContext and hw_frame is a valid
|
||||
// writable AVFrame wrapper.
|
||||
let ret = unsafe { ffi::av_hwframe_get_buffer(frames.as_ptr(), hw_frame.as_mut_ptr(), 0) };
|
||||
if ret < 0 {
|
||||
// SAFETY: data[0] still contains the Box pointer installed above.
|
||||
unsafe {
|
||||
let _ = Box::from_raw((*raw_frame.as_ptr()).data[0] as *mut ffi::AVDRMFrameDescriptor);
|
||||
(*raw_frame.as_mut_ptr()).data[0] = ptr::null_mut();
|
||||
}
|
||||
bail!("av_hwframe_get_buffer failed: error {ret}");
|
||||
}
|
||||
|
||||
// SAFETY: hw_frame is a valid VAAPI frame allocated from `frames`; raw_frame
|
||||
// is a DRM_PRIME source frame whose descriptor describes `frame`'s DMA-BUF.
|
||||
let ret = unsafe { ffi::av_hwframe_transfer_data(hw_frame.as_mut_ptr(), raw_frame.as_ptr(), 0) };
|
||||
|
||||
// SAFETY: data[0] still contains the Box pointer installed above. Recover it
|
||||
// before checking the transfer result so all paths clean up the descriptor.
|
||||
unsafe {
|
||||
let _ = Box::from_raw((*raw_frame.as_ptr()).data[0] as *mut ffi::AVDRMFrameDescriptor);
|
||||
(*raw_frame.as_mut_ptr()).data[0] = ptr::null_mut();
|
||||
}
|
||||
|
||||
if ret < 0 {
|
||||
bail!("av_hwframe_transfer_data failed: error {ret}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// EncState
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -139,8 +210,8 @@ pub struct EncState {
|
||||
|
||||
unsafe impl Send for EncState {}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
impl EncState {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
drm_device: &Path,
|
||||
output_path: &Path,
|
||||
@@ -152,12 +223,16 @@ impl EncState {
|
||||
gop_size: u32,
|
||||
fps: u32,
|
||||
transform: Transform,
|
||||
existing_hw_ctx: Option<AvHwDevCtx>,
|
||||
) -> Result<Self> {
|
||||
tracing::info!(
|
||||
"EncState::new: {width}x{height} enc={enc_width}x{enc_height} transform={transform:?}"
|
||||
);
|
||||
// 1. VAAPI device
|
||||
let hw_device_ctx = AvHwDevCtx::new_vaapi(drm_device)?;
|
||||
// 1. VAAPI device — reuse existing context if provided
|
||||
let hw_device_ctx = match existing_hw_ctx {
|
||||
Some(ctx) => ctx,
|
||||
None => AvHwDevCtx::new_vaapi(drm_device)?,
|
||||
};
|
||||
|
||||
// 2. Frame context for capture (XRGB/RGBZ)
|
||||
let frames_rgb =
|
||||
@@ -482,6 +557,7 @@ pub fn create_encoder(
|
||||
transform: Transform,
|
||||
bitrate: Option<u64>,
|
||||
gop_size: Option<u32>,
|
||||
existing_hw_ctx: Option<AvHwDevCtx>,
|
||||
) -> Result<EncState> {
|
||||
let (enc_w, enc_h) =
|
||||
transpose_if_transform_transposed(transform, width as i32, height as i32);
|
||||
@@ -500,6 +576,7 @@ pub fn create_encoder(
|
||||
actual_gop_size,
|
||||
fps,
|
||||
transform,
|
||||
existing_hw_ctx,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user