dailz/wl-webrtc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: GPU-downscale + software H.264 encode pipeline (WIP)

Browse Source 
Add SwEncState in avhw.rs: GPU pipeline using scale_vaapi to downscale
4K BGRA -> 2K NV12 on AMD iGPU, then software encode with libopenh264.

- import_dma_buf_to_vaapi: av_hwframe_map based DMA-BUF import
- SwEncState: GPU filter graph (scale_vaapi) + NV12->YUV420P + libopenh264
- state_portal.rs: integrated SwEncState, auto DRM device detection
- vaapi_import_bench.rs: CPU vs GPU pipeline benchmark
- sw_encode_bench.rs: software encode benchmark

Benchmark results: GPU pipeline ~91 FPS theoretical (10.95ms/frame)
vs CPU pipeline ~33 FPS (30.21ms/frame).

Known issue: only 1 frame encoded in production recording,
diagnostic STATS logging added to debug frame flow.
This commit is contained in:
dailz
2026-05-29 22:04:12 +08:00
parent 55abb5e56d
commit d80b34f44f
9 changed files with 2416 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/state_portal.rs
+133 -149
View File
@@ -1,17 +1,13 @@
// 采集门户状态模块 —— 通过 PipeWire/DMA-BUF 进行屏幕采集并编码
use std::mem;
use std::os::fd::AsRawFd;
use std::path::PathBuf;
use anyhow::{bail, Result};
use ffmpeg_next as ff;
use ffmpeg_next::ffi;
use crate::args::Args;
use crate::avhw::{self, EncState};
use crate::avhw::{self, SwEncState};
use crate::cap_portal::{CapPortal, PwCtrlEvent, PwDmaBufFrame};
use crate::fps_limit::FpsLimit;
use crate::transform::Transform;
/// 门户采集的阶段状态
/// - WaitingForFormat: 等待接收到第一帧 DMA-BUF 以确定视频格式参数
@@ -28,8 +24,8 @@ enum PortalStage {
pub struct StatePortal {
/// 当前采集阶段
stage: PortalStage,
/// 硬件编码器状态（第一帧到达后才初始化）
enc: Option<EncState>,
/// GPU 缩放 + 软件编码器状态（第一帧到达后才初始化）
enc: Option<SwEncState>,
/// 帧率限制器
fps_limit: FpsLimit<()>,
/// PipeWire 屏幕采集端点
@@ -44,6 +40,14 @@ pub struct StatePortal {
drm_device: Option<PathBuf>,
/// 第一帧的时间戳（纳秒），用于计算相对 PTS
first_pts_ns: Option<i64>,
/// Diagnostic: frames received from PipeWire channel
frames_received: u64,
/// Diagnostic: frames dropped by FPS limiter
frames_fps_dropped: u64,
/// Diagnostic: frames successfully encoded
frames_encoded: u64,
/// Diagnostic: last time we printed stats
last_stats_time: Option<std::time::Instant>,
}
impl StatePortal {
@@ -70,6 +74,10 @@ impl StatePortal {
first_frame: true,
drm_device,
first_pts_ns: None,
frames_received: 0,
frames_fps_dropped: 0,
frames_encoded: 0,
last_stats_time: None,
})
}
@@ -94,7 +102,11 @@ impl StatePortal {
}
let frame = match self.cap.frame_receiver().try_recv() {
Ok(frame) => frame,
Ok(frame) => {
self.frames_received += 1;
tracing::debug!("poll_and_encode: got frame #{} from channel", self.frames_received);
frame
}
Err(_) => return Ok(false),
};
@@ -110,20 +122,35 @@ impl StatePortal {
);
let drm_path = self.resolve_drm_device_for_frame(&frame)?;
let enc = avhw::create_encoder(
let (enc_width, enc_height) = portal_encode_dimensions(frame.width, frame.height);
tracing::info!(
"Portal software encode target: {}x{} -> {}x{} @ {} fps",
frame.width,
frame.height,
enc_width,
enc_height,
self.args.fps,
);
let actual_bitrate = self.args.bitrate.unwrap_or_else(|| {
2 * (enc_width as u64) * (enc_height as u64) * (self.args.fps as u64) / 100
});
let actual_gop_size = self.args.gop_size.unwrap_or(self.args.fps);
let enc = avhw::SwEncState::new(
&drm_path,
self.args.output.as_ref(),
frame.width,
frame.height,
enc_width,
enc_height,
self.args.fps,
Transform::Normal,
self.args.bitrate,
self.args.gop_size,
None,
actual_bitrate,
actual_gop_size,
)?;
self.enc = Some(enc);
self.stage = PortalStage::Streaming;
tracing::info!("First frame processed, encoder initialized, transitioning to Streaming");
drop(frame);
}
PortalStage::Streaming => {
@@ -149,160 +176,105 @@ impl StatePortal {
match crate::avhw::test_dma_buf_import(candidate, frame) {
Ok(()) => {
tracing::info!(
"Auto-selected DRM device: {} (can import PipeWire DMA-BUF)",
candidate.display()
"Auto-detected DRM device: {} (tested {} candidates)",
candidate.display(),
candidates.len(),
);
self.drm_device = Some(candidate.clone());
return Ok(candidate.clone());
}
Err(err) => {
Err(e) => {
tracing::debug!(
"DRM device {} cannot import frame: {err:#}",
candidate.display()
"DRM device {} cannot import DMA-BUF: {e}",
candidate.display(),
);
failures.push(format!("{}: {err:#}", candidate.display()));
failures.push((candidate, e));
}
}
}
bail!(
"No DRM render device can import the PipeWire DMA-BUF frame. \
Specify --drm-device. Tried: {}",
failures.join("; ")
)
"No DRM render device can import the DMA-BUF frame. Tried: {}",
failures
.into_iter()
.map(|(p, e)| format!("{} ({e})", p.display()))
.collect::<Vec<_>>()
.join(", ")
);
}
/// 处理单帧 DMA-BUF 数据
///
/// 完整的帧处理流水线：
/// 1. 帧率限制（首帧跳过）
/// 2. 构建 DRM 描述符
/// 3. 分配 DRM_PRIME 源帧
/// 4. 分配 VAAPI 硬件目标帧
/// 5. 通过 DMA-BUF 导入将帧数据导入 VAAPI
/// 6. 计算 PTS 时间戳
/// 7. 回收 DRM 描述符内存
/// 8. 编码输出
/// 通过 `av_hwframe_map` 零拷贝导入 VAAPI，然后交给 SwEncState 完成：
/// scale_vaapi GPU 缩放、2K NV12 回读、YUV420P 格式转换、软件 H.264 编码。
fn handle_pw_frame(&mut self, frame: PwDmaBufFrame) -> Result<()> {
// 1. FPS limiting (first frame bypasses)
// 帧率限制（首帧跳过限制，确保立即编码）
if self.first_frame {
self.first_frame = false;
} else {
let now = std::time::Instant::now();
if self.fps_limit.on_new_frame((), now).is_none() {
self.frames_fps_dropped += 1;
tracing::debug!("handle_pw_frame: FPS limit, dropping frame (#{})", self.frames_fps_dropped);
self.maybe_print_stats(now);
return Ok(());
}
}
// 2. Build DRM descriptor for DMA-BUF import
// 根据 DMA-BUF 帧信息构建 FFmpeg DRM 描述符
let desc = build_drm_descriptor(&frame);
let desc_box = Box::new(desc);
tracing::debug!("handle_pw_frame: processing frame, pts={}", frame.pts);
// 3. Allocate raw DRM_PRIME source frame using Video wrapper
// 分配 DRM_PRIME 格式的源帧，将描述符指针挂载到 data[0]
let mut raw_frame = ff::frame::Video::empty();
unsafe {
let raw_ptr = raw_frame.as_mut_ptr();
(*raw_ptr).data[0] = Box::into_raw(desc_box) as *mut u8;
(*raw_ptr).format = ffi::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32;
(*raw_ptr).width = frame.width as i32;
(*raw_ptr).height = frame.height as i32;
}
// 4. Get encoder reference
// 获取编码器引用
let enc = match self.enc.as_mut() {
Some(e) => e,
None => {
// Recover the Box to prevent memory leak of the descriptor
// 编码器未初始化时回收描述符以防止内存泄漏
unsafe {
let desc_ptr = (*raw_frame.as_ptr()).data[0] as *mut ffi::AVDRMFrameDescriptor;
if !desc_ptr.is_null() {
let _ = Box::from_raw(desc_ptr);
}
(*raw_frame.as_mut_ptr()).data[0] = std::ptr::null_mut();
}
bail!("encoder not initialized");
}
Some(enc) => enc,
None => bail!("encoder not initialized"),
};
// 5. Allocate VAAPI hardware target frame
// 分配 VAAPI 硬件帧缓冲区
let mut hw_frame = ff::frame::Video::empty();
let ret = unsafe {
ffi::av_hwframe_get_buffer(enc.frames_rgb().as_ptr(), hw_frame.as_mut_ptr(), 0)
};
if ret < 0 {
// Recover the Box to prevent memory leak of the descriptor
// 分配失败时回收描述符防止内存泄漏
unsafe {
let desc_ptr = (*raw_frame.as_ptr()).data[0] as *mut ffi::AVDRMFrameDescriptor;
if !desc_ptr.is_null() {
let _ = Box::from_raw(desc_ptr);
}
(*raw_frame.as_mut_ptr()).data[0] = std::ptr::null_mut();
}
bail!("av_hwframe_get_buffer failed: error {ret}");
}
// SAFETY: frames_rgb is a live VAAPI frames context configured for capture; frame carries
// valid DMA-BUF fd/format/modifier/stride/offset metadata for the duration of this call.
let mut vaapi_frame = unsafe {
avhw::import_dma_buf_to_vaapi(
enc.frames_rgb().as_ptr(),
frame.fd.as_raw_fd(),
frame.width,
frame.height,
frame.format,
frame.modifier,
frame.stride,
frame.offset,
)
}?;
// 6. Import DMA-BUF into VAAPI via transfer_data
// 通过 DMA-BUF 导入将帧数据从 DRM 传输到 VAAPI 硬件表面
let ret = unsafe {
ffi::av_hwframe_transfer_data(hw_frame.as_mut_ptr(), raw_frame.as_ptr(), 0)
};
if ret < 0 {
// 传输失败时回收描述符防止内存泄漏
unsafe {
let desc_ptr = (*raw_frame.as_ptr()).data[0] as *mut ffi::AVDRMFrameDescriptor;
if !desc_ptr.is_null() {
let _ = Box::from_raw(desc_ptr);
}
(*raw_frame.as_mut_ptr()).data[0] = std::ptr::null_mut();
}
if ret == -(ffi::EINVAL as i32) {
bail!(
"VAAPI does not support DMA-BUF modifier 0x{:X}",
frame.modifier
);
}
bail!("av_hwframe_transfer_data failed: error {ret}");
}
tracing::debug!("handle_pw_frame: DMA-BUF import OK");
// 7. Set PTS — convert PipeWire nanoseconds to encoder frame-number units
let pts = compute_pts(&mut self.first_pts_ns, frame.pts, self.args.fps);
unsafe {
(*hw_frame.as_mut_ptr()).pts = pts;
(*vaapi_frame.as_mut_ptr()).pts = pts;
}
// 8. Recover the Boxed descriptor from raw_frame *before* encoding.
// av_hwframe_transfer_data has already imported the DMA-BUF into the
// VAAPI surface, so FFmpeg no longer references the descriptor struct.
// Doing this before encode_frame ensures the descriptor is reclaimed
// even if encode_frame returns early via `?`.
//
// 在编码前回收描述符内存。
// 此时 DMA-BUF 数据已导入 VAAPI 表面，FFmpeg 不再引用描述符结构体。
// 在 encode_frame 之前回收确保即使编码返回错误也能正确释放内存。
unsafe {
let desc_ptr = (*raw_frame.as_ptr()).data[0] as *mut ffi::AVDRMFrameDescriptor;
if !desc_ptr.is_null() {
let _ = Box::from_raw(desc_ptr);
}
(*raw_frame.as_mut_ptr()).data[0] = std::ptr::null_mut();
}
enc.encode_frame(&vaapi_frame)?;
self.frames_encoded += 1;
tracing::info!("handle_pw_frame: frame #{} encoded OK, pts={}", self.frames_encoded, pts);
let now = std::time::Instant::now();
self.maybe_print_stats(now);
// 9. Encode — safe to early-return via `?` now that descriptor is recovered.
// 编码帧数据（此时描述符已回收，可安全通过 `?` 提前返回）
enc.encode_frame(&hw_frame)?;
// raw_frame and hw_frame drop here via Video::drop → av_frame_free
// raw_frame 和 hw_frame 在此处通过 Video::drop → av_frame_free 释放
Ok(())
}
fn maybe_print_stats(&mut self, now: std::time::Instant) {
let should_print = match self.last_stats_time {
None => true,
Some(last) => now.duration_since(last) >= std::time::Duration::from_secs(2),
};
if should_print {
self.last_stats_time = Some(now);
tracing::info!(
"STATS: received={}, fps_dropped={}, encoded={}",
self.frames_received,
self.frames_fps_dropped,
self.frames_encoded,
);
}
}
/// 关闭状态：刷新编码器并清理资源
///
/// 使用 `enc.take()` 确保编码器只被 flush 一次，即使多次调用也安全（幂等）。
@@ -327,28 +299,21 @@ impl Drop for StatePortal {
}
}
/// 根据 DMA-BUF 帧信息构建 FFmpeg DRM 帧描述符
///
/// 将 PipeWire 提供的 DMA-BUF 参数（fd、偏移量、步长、修饰符等）
/// 转换为 FFmpeg 的 AVDRMFrameDescriptor 结构体，用于零拷贝硬件导入。
fn build_drm_descriptor(frame: &PwDmaBufFrame) -> ffi::AVDRMFrameDescriptor {
let mut desc: ffi::AVDRMFrameDescriptor = unsafe { mem::zeroed() };
fn portal_encode_dimensions(width: u32, height: u32) -> (u32, u32) {
const TARGET_W: u32 = 2560;
const TARGET_H: u32 = 1440;
// DMA-BUF 对象层：一个 fd 对应一个内存对象
desc.nb_objects = 1;
desc.objects[0].fd = frame.fd.as_raw_fd();
desc.objects[0].size = 0; // 大小为 0 表示整个 fd
desc.objects[0].format_modifier = frame.modifier;
if width <= TARGET_W && height <= TARGET_H {
return (width & !1, height & !1);
}
// 像素格式层：单层单平面布局（如 XR24 格式）
desc.nb_layers = 1;
desc.layers[0].format = frame.format;
desc.layers[0].nb_planes = 1;
desc.layers[0].planes[0].object_index = 0;
desc.layers[0].planes[0].offset = frame.offset as isize;
desc.layers[0].planes[0].pitch = frame.stride as isize;
desc
let width_limited_h = ((height as u64) * (TARGET_W as u64) / (width as u64)) as u32;
if width_limited_h <= TARGET_H {
(TARGET_W & !1, width_limited_h & !1)
} else {
let height_limited_w = ((width as u64) * (TARGET_H as u64) / (height as u64)) as u32;
(height_limited_w & !1, TARGET_H & !1)
}
}
/// Convert PipeWire nanosecond PTS to encoder frame-number units.
@@ -372,6 +337,22 @@ fn resolve_drm_device(args: &Args) -> Result<Option<PathBuf>> {
Ok(None)
}
#[cfg(test)]
fn build_drm_descriptor(frame: &PwDmaBufFrame) -> ffmpeg_next::ffi::AVDRMFrameDescriptor {
let mut desc: ffmpeg_next::ffi::AVDRMFrameDescriptor = unsafe { std::mem::zeroed() };
desc.nb_objects = 1;
desc.objects[0].fd = frame.fd.as_raw_fd();
desc.objects[0].size = 0;
desc.objects[0].format_modifier = frame.modifier;
desc.nb_layers = 1;
desc.layers[0].format = frame.format;
desc.layers[0].nb_planes = 1;
desc.layers[0].planes[0].object_index = 0;
desc.layers[0].planes[0].offset = frame.offset as isize;
desc.layers[0].planes[0].pitch = frame.stride as isize;
desc
}
#[cfg(test)]
mod tests {
use super::*;
@@ -426,7 +407,10 @@ mod tests {
port: 0,
};
let result = resolve_drm_device(&args).unwrap();
assert_eq!(result, Some(std::path::PathBuf::from("/dev/dri/renderD128")));
assert_eq!(
result,
Some(std::path::PathBuf::from("/dev/dri/renderD128"))
);
}
#[test]