fix(avhw): derive encoder hw_frames_ctx from filter graph buffersink

src/avhw.rs

@@ -104,15 +104,6 @@ impl AvHwFrameCtx {
         Self::new_inner(hw_dev, w, h, sw_fmt)
     }
 
-    pub fn for_encode(
-        hw_dev: &AvHwDevCtx,
-        w: u32,
-        h: u32,
-        sw_fmt: ff::format::Pixel,
-    ) -> Result<Self> {
-        Self::new_inner(hw_dev, w, h, sw_fmt)
-    }
-
     pub fn as_ptr(&self) -> *mut ffi::AVBufferRef {
         self.ptr
     }
@@ -139,7 +130,6 @@ impl Drop for AvHwFrameCtx {
 pub struct EncState {
     enc_video: ff::codec::encoder::video::Video,
     frames_rgb: AvHwFrameCtx,
-    frames_yuv: AvHwFrameCtx,
     video_filter: ff::filter::Graph,
     hw_device_ctx: AvHwDevCtx,
     octx: ff::format::context::Output,
@@ -170,19 +160,49 @@ impl EncState {
         let hw_device_ctx = AvHwDevCtx::new_vaapi(drm_device)?;
         tracing::debug!("EncState::new: VAAPI device created");
 
-        // 2. Frame contexts (capture=XRGB/RGBZ, encode=NV12)
+        // 2. Frame context for capture (XRGB/RGBZ)
-        // frames_rgb uses original capture dimensions (matches raw framebuffer)
-        // frames_yuv uses encoder dimensions (transposed for 90°/270° rotations)
         let frames_rgb =
             AvHwFrameCtx::for_capture(&hw_device_ctx, width, height, ff::format::Pixel::RGBZ)?;
-        let frames_yuv = AvHwFrameCtx::for_encode(
+
+        // 3. Filter graph — must be built BEFORE encoder config so we can derive
+        //    hw_frames_ctx from the buffersink output (correct surface pool dimensions).
+        tracing::debug!("EncState::new: building filter graph...");
+        let video_filter = build_filter_graph(
             &hw_device_ctx,
+            &frames_rgb,
+            width,
+            height,
             enc_width,
             enc_height,
-            ff::format::Pixel::NV12,
+            fps,
+            transform,
         )?;
 
-        // 3. Find h264_vaapi encoder
+        let mut sink_ctx = video_filter.get("out").unwrap();
+        let sink_hw_frames = unsafe {
+            let raw = ffi::av_buffersink_get_hw_frames_ctx(sink_ctx.as_mut_ptr());
+            if raw.is_null() {
+                bail!("buffersink has no hw_frames_ctx — filter graph may not be configured for hardware output");
+            }
+            let hw_ref = ffi::av_buffer_ref(raw);
+            if hw_ref.is_null() {
+                bail!("av_buffer_ref failed for buffersink hw_frames_ctx — likely out of memory");
+            }
+            hw_ref
+        };
+
+        unsafe {
+            let fc = (*sink_hw_frames).data as *mut ffi::AVHWFramesContext;
+            let actual_w = (*fc).width as u32;
+            let actual_h = (*fc).height as u32;
+            if actual_w != enc_width || actual_h != enc_height {
+                tracing::warn!(
+                    "Filter output dimensions {actual_w}x{actual_h} differ from encoder dimensions {enc_width}x{enc_height}"
+                );
+            }
+        }
+
+        // 4. Find h264_vaapi encoder
         let codec = ff::encoder::find_by_name("h264_vaapi")
             .ok_or_else(|| anyhow::anyhow!("h264_vaapi encoder not found"))?;
 
@@ -208,7 +228,7 @@ impl EncState {
         // SAFETY: Assign hw device and frames ctx to the encoder.
         unsafe {
             (*enc.as_mut_ptr()).hw_device_ctx = hw_device_ctx.ref_clone();
-            (*enc.as_mut_ptr()).hw_frames_ctx = frames_yuv.ref_clone();
+            (*enc.as_mut_ptr()).hw_frames_ctx = sink_hw_frames;
         }
 
         // SAFETY: Set repeat_pps=1 on the encoder so PPS is inserted in every encoded frame.
@@ -227,7 +247,7 @@ impl EncState {
             }
         }
 
-        // 4. Open encoder. Video::open() returns Encoder(Video); .0 extracts the Video.
+        // 5. Open encoder. Video::open() returns Encoder(Video); .0 extracts the Video.
         tracing::debug!("EncState::new: opening encoder...");
         let opened = enc
             .open()
@@ -235,19 +255,6 @@ impl EncState {
         let enc_video = opened.0;
         tracing::debug!("EncState::new: encoder opened");
 
-        // 5. Filter graph (inline)
-        tracing::debug!("EncState::new: building filter graph...");
-        let video_filter = build_filter_graph(
-            &hw_device_ctx,
-            &frames_rgb,
-            width,
-            height,
-            enc_width,
-            enc_height,
-            fps,
-            transform,
-        )?;
-
         // 6. Muxer setup (strict order)
         let output_cstr = CString::new(output_path.to_str().unwrap())?;
         let mut fmt_ctx_ptr: *mut ffi::AVFormatContext = ptr::null_mut();
@@ -322,7 +329,6 @@ impl EncState {
         Ok(Self {
             enc_video,
             frames_rgb,
-            frames_yuv,
             video_filter,
             hw_device_ctx,
             octx,