From 5100d78aa8af9df3adaf1519396ce95ca48703bc Mon Sep 17 00:00:00 2001 From: dailz Date: Mon, 25 May 2026 14:32:58 +0800 Subject: [PATCH] fix: resolve SHM hang, DRM device mismatch, and duplicate VAAPI context BUG-2 (HIGH): SHM Buffer event caused permanent hang In the ZwlrScreencopyFrameV1 dispatcher, receiving a SHM Buffer event left in_flight_surface stuck at AllocQueued forever, preventing queue_alloc_frame() from requesting new frames. Fix: treat Buffer as a metadata offer (v3 protocol), wait for BufferDone to decide failure, and add AllocQueued state guard to LinuxDmabuf handler. BUG-3 (MEDIUM): Portal backend picked wrong GPU on multi-GPU systems state_portal.rs hardcoded /dev/dri/renderD128 then renderD129, which selects the wrong GPU when PipeWire uses a different device. Fix: extract find_drm_render_nodes() as shared utility; defer DRM device selection to first PipeWire frame; test each candidate with av_hwframe_transfer_data to find the GPU that can actually import the DMA-BUF frame. BUG-4 (LOW): VAAPI device context created twice unnecessarily try_finalize_output() created an AvHwDevCtx stored in EverythingButFmt, but negotiate_format() discarded it (_hw_device_ctx) and EncState::new created a new one. Fix: thread the existing hw_device_ctx through negotiate_format() and create_encoder() to EncState::new() which reuses it when provided. --- src/avhw.rs | 83 +++++++++++++++++++++++++++++++++++++++++++-- src/state.rs | 71 ++++++++++++++++++++++++++------------ src/state_portal.rs | 72 ++++++++++++++++++++++++++++----------- 3 files changed, 183 insertions(+), 43 deletions(-) diff --git a/src/avhw.rs b/src/avhw.rs index 6895325..d123e31 100644 --- a/src/avhw.rs +++ b/src/avhw.rs @@ -1,4 +1,6 @@ use std::ffi::CString; +use std::mem; +use std::os::fd::AsRawFd; use std::path::Path; use std::ptr; @@ -7,6 +9,7 @@ use ffmpeg_next as ff; use ffmpeg_next::ffi; use ffmpeg_next::packet::Mut as _; +use crate::cap_portal::PwDmaBufFrame; use crate::transform::{transpose_if_transform_transposed, Transform}; // --------------------------------------------------------------------------- @@ -123,6 +126,74 @@ impl Drop for AvHwFrameCtx { } } +/// Test whether `drm_device` can import the PipeWire DMA-BUF frame via VAAPI. +pub fn test_dma_buf_import(drm_device: &Path, frame: &PwDmaBufFrame) -> Result<()> { + let hw_dev = AvHwDevCtx::new_vaapi(drm_device)?; + let frames = AvHwFrameCtx::for_capture( + &hw_dev, + frame.width, + frame.height, + ff::format::Pixel::RGBZ, + )?; + + // SAFETY: AVDRMFrameDescriptor is a C POD struct. Zero-initialization is the + // expected FFmpeg setup before filling the fields used below. + let mut desc: ffi::AVDRMFrameDescriptor = unsafe { mem::zeroed() }; + desc.nb_objects = 1; + desc.objects[0].fd = frame.fd.as_raw_fd(); + desc.objects[0].size = 0; + desc.objects[0].format_modifier = frame.modifier; + desc.nb_layers = 1; + desc.layers[0].format = frame.format; + desc.layers[0].nb_planes = 1; + desc.layers[0].planes[0].object_index = 0; + desc.layers[0].planes[0].offset = frame.offset as isize; + desc.layers[0].planes[0].pitch = frame.stride as isize; + + let desc_box = Box::new(desc); + let mut raw_frame = ff::frame::Video::empty(); + // SAFETY: raw_frame owns a valid AVFrame. data[0] is used by FFmpeg's + // DRM_PRIME frame convention to point at an AVDRMFrameDescriptor. The Box is + // recovered before every return path below. + unsafe { + let raw_ptr = raw_frame.as_mut_ptr(); + (*raw_ptr).data[0] = Box::into_raw(desc_box) as *mut u8; + (*raw_ptr).format = ffi::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32; + (*raw_ptr).width = frame.width as i32; + (*raw_ptr).height = frame.height as i32; + } + + let mut hw_frame = ff::frame::Video::empty(); + // SAFETY: frames is an initialized AVHWFramesContext and hw_frame is a valid + // writable AVFrame wrapper. + let ret = unsafe { ffi::av_hwframe_get_buffer(frames.as_ptr(), hw_frame.as_mut_ptr(), 0) }; + if ret < 0 { + // SAFETY: data[0] still contains the Box pointer installed above. + unsafe { + let _ = Box::from_raw((*raw_frame.as_ptr()).data[0] as *mut ffi::AVDRMFrameDescriptor); + (*raw_frame.as_mut_ptr()).data[0] = ptr::null_mut(); + } + bail!("av_hwframe_get_buffer failed: error {ret}"); + } + + // SAFETY: hw_frame is a valid VAAPI frame allocated from `frames`; raw_frame + // is a DRM_PRIME source frame whose descriptor describes `frame`'s DMA-BUF. + let ret = unsafe { ffi::av_hwframe_transfer_data(hw_frame.as_mut_ptr(), raw_frame.as_ptr(), 0) }; + + // SAFETY: data[0] still contains the Box pointer installed above. Recover it + // before checking the transfer result so all paths clean up the descriptor. + unsafe { + let _ = Box::from_raw((*raw_frame.as_ptr()).data[0] as *mut ffi::AVDRMFrameDescriptor); + (*raw_frame.as_mut_ptr()).data[0] = ptr::null_mut(); + } + + if ret < 0 { + bail!("av_hwframe_transfer_data failed: error {ret}"); + } + + Ok(()) +} + // --------------------------------------------------------------------------- // EncState // --------------------------------------------------------------------------- @@ -139,8 +210,8 @@ pub struct EncState { unsafe impl Send for EncState {} -#[allow(clippy::too_many_arguments)] impl EncState { + #[allow(clippy::too_many_arguments)] pub fn new( drm_device: &Path, output_path: &Path, @@ -152,12 +223,16 @@ impl EncState { gop_size: u32, fps: u32, transform: Transform, + existing_hw_ctx: Option, ) -> Result { tracing::info!( "EncState::new: {width}x{height} enc={enc_width}x{enc_height} transform={transform:?}" ); - // 1. VAAPI device - let hw_device_ctx = AvHwDevCtx::new_vaapi(drm_device)?; + // 1. VAAPI device — reuse existing context if provided + let hw_device_ctx = match existing_hw_ctx { + Some(ctx) => ctx, + None => AvHwDevCtx::new_vaapi(drm_device)?, + }; // 2. Frame context for capture (XRGB/RGBZ) let frames_rgb = @@ -482,6 +557,7 @@ pub fn create_encoder( transform: Transform, bitrate: Option, gop_size: Option, + existing_hw_ctx: Option, ) -> Result { let (enc_w, enc_h) = transpose_if_transform_transposed(transform, width as i32, height as i32); @@ -500,6 +576,7 @@ pub fn create_encoder( actual_gop_size, fps, transform, + existing_hw_ctx, ) } diff --git a/src/state.rs b/src/state.rs index 9849cbc..5e428c0 100644 --- a/src/state.rs +++ b/src/state.rs @@ -188,23 +188,29 @@ pub struct State { // Helpers // --------------------------------------------------------------------------- +/// Scan /dev/dri for all available DRM render nodes (renderD*), sorted by node number. +pub(crate) fn find_drm_render_nodes() -> Vec { + let Ok(entries) = std::fs::read_dir("/dev/dri") else { + return Vec::new(); + }; + + let mut nodes: Vec<(u32, PathBuf)> = entries + .filter_map(Result::ok) + .filter_map(|entry| { + let path = entry.path(); + let name = path.file_name()?.to_str()?; + let number = name.strip_prefix("renderD")?.parse::().ok()?; + std::fs::metadata(&path).ok()?; + Some((number, path)) + }) + .collect(); + nodes.sort_by_key(|(number, _)| *number); + nodes.into_iter().map(|(_, path)| path).collect() +} + /// Scan /dev/dri for the first available DRM render node (renderD*). fn find_drm_render_node() -> Option { - std::fs::read_dir("/dev/dri") - .ok()? - .filter_map(|e| e.ok()) - .filter(|e| { - e.file_name() - .to_str() - .map(|s| s.starts_with("renderD")) - .unwrap_or(false) - }) - .filter_map(|e| { - let path = e.path(); - std::fs::metadata(&path).ok()?; - Some(path) - }) - .min_by_key(|e| e.to_path_buf()) + find_drm_render_nodes().into_iter().next() } impl State { @@ -584,18 +590,18 @@ impl State { EncConstructionStage::EverythingButFmt { output_info, output, - hw_device_ctx: _hw_device_ctx, + hw_device_ctx, cap, screencopy_manager, dmabuf, - } => (output_info, output, cap, screencopy_manager, dmabuf), + } => (output_info, output, hw_device_ctx, cap, screencopy_manager, dmabuf), other => { tracing::warn!("negotiate_format: not in EverythingButFmt stage"); self.stage = other; return; } }; - let (output_info, output, cap, screencopy_manager, dmabuf) = stage_data; + let (output_info, output, hw_device_ctx, cap, screencopy_manager, dmabuf) = stage_data; let drm_path = self.resolve_drm_path(); let fps = self.args.fps; let bitrate = self.args.bitrate.unwrap_or_else(|| { @@ -610,6 +616,7 @@ impl State { output_info.transform, self.args.bitrate, self.args.gop_size, + Some(hw_device_ctx), ) { Ok(enc) => enc, Err(e) => { @@ -1228,11 +1235,13 @@ impl Dispatch for State { _qhandle: &QueueHandle>, ) { match event { + // SHM buffer offer — in v3 the compositor enumerates supported buffer + // types (buffer and/or linux_dmabuf) before buffer_done. We only + // support DMA-BUF, so just log and wait for linux_dmabuf / buffer_done. ScreencopyFrameEvent::Buffer { .. } => { - tracing::warn!( - "Received SHM Buffer event — only DMA-BUF capture is supported. Ignoring." + tracing::debug!( + "Received SHM Buffer offer — only DMA-BUF capture is supported" ); - return; } ScreencopyFrameEvent::LinuxDmabuf { format, @@ -1240,6 +1249,12 @@ impl Dispatch for State { height, } => { tracing::debug!("Screencopy LinuxDmabuf: format={format}, {width}x{height}"); + + if !matches!(state.in_flight_surface, InFlightSurface::AllocQueued) { + tracing::warn!("Received LinuxDmabuf while no frame allocation was queued"); + return; + } + if matches!(state.stage, EncConstructionStage::EverythingButFmt { .. }) { state.negotiate_format(format, width, height); if state.errored { @@ -1251,6 +1266,20 @@ impl Dispatch for State { } state.on_frame_allocd((), format, width, height); } + // v3 terminal event: all buffer offers have been enumerated. + // If still AllocQueued, the compositor never sent linux_dmabuf — + // DMA-BUF screencopy is unsupported, so we must error out. + ScreencopyFrameEvent::BufferDone => { + if matches!(state.in_flight_surface, InFlightSurface::AllocQueued) { + tracing::error!( + "Compositor did not offer DMA-BUF screencopy (only SHM); \ + DMA-BUF capture is required" + ); + state.in_flight_surface = InFlightSurface::None; + proxy.destroy(); + state.errored = true; + } + } ScreencopyFrameEvent::Ready { tv_sec_hi, tv_sec_lo, diff --git a/src/state_portal.rs b/src/state_portal.rs index 7f718ba..0ab9655 100644 --- a/src/state_portal.rs +++ b/src/state_portal.rs @@ -1,5 +1,6 @@ // 采集门户状态模块 —— 通过 PipeWire/DMA-BUF 进行屏幕采集并编码 use std::mem; +use std::os::fd::AsRawFd; use std::path::PathBuf; use anyhow::{bail, Result}; @@ -39,8 +40,8 @@ pub struct StatePortal { errored: bool, /// 是否为第一帧（首帧跳过帧率限制） first_frame: bool, - /// DRM 渲染设备路径（如 /dev/dri/renderD128） - drm_device: PathBuf, + /// DRM 渲染设备路径（如 /dev/dri/renderD128）；None 表示首帧自动检测 + drm_device: Option, /// 第一帧的时间戳（纳秒），用于计算相对 PTS first_pts_ns: Option, } @@ -51,7 +52,11 @@ impl StatePortal { /// 初始化 DRM 设备路径和 PipeWire 采集端点，编码器延迟到第一帧到达时创建。 pub fn new(args: Args) -> Result { let drm_device = resolve_drm_device(&args)?; - tracing::info!("Using DRM device: {}", drm_device.display()); + if let Some(ref drm_device) = drm_device { + tracing::info!("Using DRM device: {}", drm_device.display()); + } else { + tracing::info!("DRM device auto-detection enabled"); + } let cap = CapPortal::new(&args)?; @@ -92,8 +97,9 @@ impl StatePortal { frame.modifier ); + let drm_path = self.resolve_drm_device_for_frame(&frame)?; let enc = avhw::create_encoder( - &self.drm_device, + &drm_path, self.args.output.as_ref(), frame.width, frame.height, @@ -101,6 +107,7 @@ impl StatePortal { Transform::Normal, self.args.bitrate, self.args.gop_size, + None, )?; self.enc = Some(enc); @@ -128,6 +135,44 @@ impl StatePortal { Ok(true) } + fn resolve_drm_device_for_frame(&mut self, frame: &PwDmaBufFrame) -> Result { + if let Some(ref drm) = self.drm_device { + return Ok(drm.clone()); + } + + let candidates = crate::state::find_drm_render_nodes(); + if candidates.is_empty() { + bail!("No DRM render device found. Specify --drm-device."); + } + + let mut failures = Vec::new(); + for candidate in &candidates { + match crate::avhw::test_dma_buf_import(candidate, frame) { + Ok(()) => { + tracing::info!( + "Auto-selected DRM device: {} (can import PipeWire DMA-BUF)", + candidate.display() + ); + self.drm_device = Some(candidate.clone()); + return Ok(candidate.clone()); + } + Err(err) => { + tracing::debug!( + "DRM device {} cannot import frame: {err:#}", + candidate.display() + ); + failures.push(format!("{}: {err:#}", candidate.display())); + } + } + } + + bail!( + "No DRM render device can import the PipeWire DMA-BUF frame. \ + Specify --drm-device. Tried: {}", + failures.join("; ") + ) + } + /// 处理单帧 DMA-BUF 数据 /// /// 完整的帧处理流水线： @@ -311,25 +356,14 @@ fn build_drm_descriptor(frame: &PwDmaBufFrame) -> ffi::AVDRMFrameDescriptor { desc } -use std::os::fd::AsRawFd; - /// 解析 DRM 渲染设备路径 /// -/// 优先使用命令行指定的设备路径，否则依次尝试 -/// `/dev/dri/renderD128` 和 `/dev/dri/renderD129`。 -fn resolve_drm_device(args: &Args) -> Result { +/// 仅使用命令行指定的设备路径；未指定则在首帧到达时自动检测。 +fn resolve_drm_device(args: &Args) -> Result> { if let Some(ref drm) = args.drm_device { - return Ok(PathBuf::from(drm)); + return Ok(Some(PathBuf::from(drm))); } - - for render in &["/dev/dri/renderD128", "/dev/dri/renderD129"] { - let path = PathBuf::from(render); - if path.exists() { - return Ok(path); - } - } - - bail!("No DRM render device found. Specify --drm-device.") + Ok(None) } #[cfg(test)]