fix: resolve SHM hang, DRM device mismatch, and duplicate VAAPI context

BUG-2 (HIGH): SHM Buffer event caused permanent hang
  In the ZwlrScreencopyFrameV1 dispatcher, receiving a SHM Buffer event
  left in_flight_surface stuck at AllocQueued forever, preventing
  queue_alloc_frame() from requesting new frames.
  Fix: treat Buffer as a metadata offer (v3 protocol), wait for
  BufferDone to decide failure, and add AllocQueued state guard to
  LinuxDmabuf handler.

BUG-3 (MEDIUM): Portal backend picked wrong GPU on multi-GPU systems
  state_portal.rs hardcoded /dev/dri/renderD128 then renderD129, which
  selects the wrong GPU when PipeWire uses a different device.
  Fix: extract find_drm_render_nodes() as shared utility; defer DRM
  device selection to first PipeWire frame; test each candidate with
  av_hwframe_transfer_data to find the GPU that can actually import
  the DMA-BUF frame.

BUG-4 (LOW): VAAPI device context created twice unnecessarily
  try_finalize_output() created an AvHwDevCtx stored in EverythingButFmt,
  but negotiate_format() discarded it (_hw_device_ctx) and EncState::new
  created a new one.
  Fix: thread the existing hw_device_ctx through negotiate_format() and
  create_encoder() to EncState::new() which reuses it when provided.
This commit is contained in:
dailz
2026-05-25 14:32:58 +08:00
parent 460a3ee711
commit 5100d78aa8
3 changed files with 183 additions and 43 deletions

View File

@@ -188,23 +188,29 @@ pub struct State<S: CaptureSource> {
// Helpers
// ---------------------------------------------------------------------------
/// Scan /dev/dri for all available DRM render nodes (renderD*), sorted by node number.
pub(crate) fn find_drm_render_nodes() -> Vec<PathBuf> {
let Ok(entries) = std::fs::read_dir("/dev/dri") else {
return Vec::new();
};
let mut nodes: Vec<(u32, PathBuf)> = entries
.filter_map(Result::ok)
.filter_map(|entry| {
let path = entry.path();
let name = path.file_name()?.to_str()?;
let number = name.strip_prefix("renderD")?.parse::<u32>().ok()?;
std::fs::metadata(&path).ok()?;
Some((number, path))
})
.collect();
nodes.sort_by_key(|(number, _)| *number);
nodes.into_iter().map(|(_, path)| path).collect()
}
/// Scan /dev/dri for the first available DRM render node (renderD*).
fn find_drm_render_node() -> Option<PathBuf> {
std::fs::read_dir("/dev/dri")
.ok()?
.filter_map(|e| e.ok())
.filter(|e| {
e.file_name()
.to_str()
.map(|s| s.starts_with("renderD"))
.unwrap_or(false)
})
.filter_map(|e| {
let path = e.path();
std::fs::metadata(&path).ok()?;
Some(path)
})
.min_by_key(|e| e.to_path_buf())
find_drm_render_nodes().into_iter().next()
}
impl<S: CaptureSource> State<S> {
@@ -584,18 +590,18 @@ impl<S: CaptureSource> State<S> {
EncConstructionStage::EverythingButFmt {
output_info,
output,
hw_device_ctx: _hw_device_ctx,
hw_device_ctx,
cap,
screencopy_manager,
dmabuf,
} => (output_info, output, cap, screencopy_manager, dmabuf),
} => (output_info, output, hw_device_ctx, cap, screencopy_manager, dmabuf),
other => {
tracing::warn!("negotiate_format: not in EverythingButFmt stage");
self.stage = other;
return;
}
};
let (output_info, output, cap, screencopy_manager, dmabuf) = stage_data;
let (output_info, output, hw_device_ctx, cap, screencopy_manager, dmabuf) = stage_data;
let drm_path = self.resolve_drm_path();
let fps = self.args.fps;
let bitrate = self.args.bitrate.unwrap_or_else(|| {
@@ -610,6 +616,7 @@ impl<S: CaptureSource> State<S> {
output_info.transform,
self.args.bitrate,
self.args.gop_size,
Some(hw_device_ctx),
) {
Ok(enc) => enc,
Err(e) => {
@@ -1228,11 +1235,13 @@ impl Dispatch<ZwlrScreencopyFrameV1, ()> for State<CapWlrScreencopy> {
_qhandle: &QueueHandle<State<CapWlrScreencopy>>,
) {
match event {
// SHM buffer offer — in v3 the compositor enumerates supported buffer
// types (buffer and/or linux_dmabuf) before buffer_done. We only
// support DMA-BUF, so just log and wait for linux_dmabuf / buffer_done.
ScreencopyFrameEvent::Buffer { .. } => {
tracing::warn!(
"Received SHM Buffer event — only DMA-BUF capture is supported. Ignoring."
tracing::debug!(
"Received SHM Buffer offer — only DMA-BUF capture is supported"
);
return;
}
ScreencopyFrameEvent::LinuxDmabuf {
format,
@@ -1240,6 +1249,12 @@ impl Dispatch<ZwlrScreencopyFrameV1, ()> for State<CapWlrScreencopy> {
height,
} => {
tracing::debug!("Screencopy LinuxDmabuf: format={format}, {width}x{height}");
if !matches!(state.in_flight_surface, InFlightSurface::AllocQueued) {
tracing::warn!("Received LinuxDmabuf while no frame allocation was queued");
return;
}
if matches!(state.stage, EncConstructionStage::EverythingButFmt { .. }) {
state.negotiate_format(format, width, height);
if state.errored {
@@ -1251,6 +1266,20 @@ impl Dispatch<ZwlrScreencopyFrameV1, ()> for State<CapWlrScreencopy> {
}
state.on_frame_allocd((), format, width, height);
}
// v3 terminal event: all buffer offers have been enumerated.
// If still AllocQueued, the compositor never sent linux_dmabuf —
// DMA-BUF screencopy is unsupported, so we must error out.
ScreencopyFrameEvent::BufferDone => {
if matches!(state.in_flight_surface, InFlightSurface::AllocQueued) {
tracing::error!(
"Compositor did not offer DMA-BUF screencopy (only SHM); \
DMA-BUF capture is required"
);
state.in_flight_surface = InFlightSurface::None;
proxy.destroy();
state.errored = true;
}
}
ScreencopyFrameEvent::Ready {
tv_sec_hi,
tv_sec_lo,