fix(bsf): remove non-functional BSF pipeline, use encoder repeat_pps option

src/avhw.rs

@@ -7,39 +7,6 @@ use ffmpeg_next as ff;
 use ffmpeg_next::ffi as ffi;
 use ffmpeg_next::packet::Mut as _;
 
-// ---------------------------------------------------------------------------
-// BSF FFI — ffmpeg-sys-next does not expose the BSF API; declare manually.
-// Linked from libavcodec (always present when avcodec feature is enabled).
-// ---------------------------------------------------------------------------
-
-#[repr(C)]
-pub struct AVBitStreamFilter {
-    _opaque: [u8; 0],
-}
-
-#[repr(C)]
-pub struct AVBSFContext {
-    av_class: *const ffi::AVClass,
-    filter: *const AVBitStreamFilter,
-    priv_data: *mut libc::c_void,
-    par_in: *mut ffi::AVCodecParameters,
-    par_out: *mut ffi::AVCodecParameters,
-    time_base_in: ffi::AVRational,
-    time_base_out: ffi::AVRational,
-}
-
-extern "C" {
-    pub fn av_bsf_get_by_name(name: *const libc::c_char) -> *const AVBitStreamFilter;
-    pub fn av_bsf_alloc(
-        filter: *const AVBitStreamFilter,
-        ctx: *mut *mut AVBSFContext,
-    ) -> libc::c_int;
-    pub fn av_bsf_init(ctx: *mut AVBSFContext) -> libc::c_int;
-    pub fn av_bsf_send_packet(ctx: *mut AVBSFContext, pkt: *mut ffi::AVPacket) -> libc::c_int;
-    pub fn av_bsf_receive_packet(ctx: *mut AVBSFContext, pkt: *mut ffi::AVPacket) -> libc::c_int;
-    pub fn av_bsf_free(ctx: *mut *mut AVBSFContext);
-}
-
 // ---------------------------------------------------------------------------
 // AvHwDevCtx
 // ---------------------------------------------------------------------------
@@ -174,7 +141,6 @@ impl Drop for AvHwFrameCtx {
 
 pub struct EncState {
     enc_video: ff::codec::encoder::video::Video,
-    bsf_ctx: *mut AVBSFContext,
     frames_rgb: AvHwFrameCtx,
     frames_yuv: AvHwFrameCtx,
     video_filter: ff::filter::Graph,
@@ -243,72 +209,28 @@ impl EncState {
             (*enc.as_mut_ptr()).hw_frames_ctx = frames_yuv.ref_clone();
         }
 
+        // SAFETY: Set repeat_pps=1 on the encoder so PPS is inserted in every encoded frame.
+        // This ensures decoders can start decoding from any frame (important for WebRTC).
+        // Note: h264_metadata BSF does NOT have repeat_sps/repeat_pps options in any FFmpeg version
+        // (verified on both 6.1.3 and 8.0). The encoder's own repeat_pps option is the correct approach.
+        // For SPS repetition: IDR frames carry SPS by default, controlled by gop_size/idr_interval.
+        {
+            let key = CString::new("repeat_pps").unwrap();
+            let val = CString::new("1").unwrap();
+            let ret = unsafe {
+                ffi::av_opt_set((*enc.as_mut_ptr()).priv_data, key.as_ptr(), val.as_ptr(), 0)
+            };
+            if ret < 0 {
+                bail!("av_opt_set repeat_pps on encoder failed: error {ret}");
+            }
+        }
+
         // 4. Open encoder. Video::open() returns Encoder(Video); .0 extracts the Video.
         let opened = enc.open().map_err(|e| {
             anyhow::anyhow!("Failed to open h264_vaapi encoder: {e}")
         })?;
         let enc_video = opened.0;
 
-        // --- BSF init (after encoder open, before filter graph) ---
-        // SAFETY: av_bsf_get_by_name returns a pointer to a static filter definition.
-        let bsf_name = CString::new("h264_metadata").unwrap();
-        let filter = unsafe { av_bsf_get_by_name(bsf_name.as_ptr()) };
-        if filter.is_null() {
-            bail!("h264_metadata BSF not found in FFmpeg build");
-        }
-
-        let mut bsf_ctx: *mut AVBSFContext = ptr::null_mut();
-        let ret = unsafe { av_bsf_alloc(filter, &mut bsf_ctx) };
-        if ret < 0 {
-            bail!("av_bsf_alloc failed: error {ret}");
-        }
-
-        // SAFETY: avcodec_parameters_from_context copies FROM AVCodecContext TO AVCodecParameters.
-        let ret = unsafe {
-            ffi::avcodec_parameters_from_context((*bsf_ctx).par_in, enc_video.as_ptr())
-        };
-        if ret < 0 {
-            // SAFETY: bsf_ctx was allocated but not yet initialized — safe to free
-            unsafe { av_bsf_free(&mut bsf_ctx) };
-            bail!("avcodec_parameters_from_context for BSF failed: error {ret}");
-        }
-
-        // SAFETY: time_base_in is a plain AVRational field, safe to write
-        unsafe {
-            (*bsf_ctx).time_base_in = (*enc_video.as_ptr()).time_base;
-        }
-
-        // Set repeat_sps=1
-        let key_sps = CString::new("repeat_sps").unwrap();
-        let val_one = CString::new("1").unwrap();
-        let ret = unsafe {
-            ffi::av_opt_set((*bsf_ctx).priv_data, key_sps.as_ptr(), val_one.as_ptr(), 0)
-        };
-        if ret < 0 {
-            // SAFETY: bsf_ctx allocated but not fully initialized — safe to free
-            unsafe { av_bsf_free(&mut bsf_ctx) };
-            bail!("av_opt_set repeat_sps failed: error {ret}");
-        }
-
-        // Set repeat_pps=1
-        let key_pps = CString::new("repeat_pps").unwrap();
-        let ret = unsafe {
-            ffi::av_opt_set((*bsf_ctx).priv_data, key_pps.as_ptr(), val_one.as_ptr(), 0)
-        };
-        if ret < 0 {
-            // SAFETY: bsf_ctx allocated, repeat_sps set but not init'd — safe to free
-            unsafe { av_bsf_free(&mut bsf_ctx) };
-            bail!("av_opt_set repeat_pps failed: error {ret}");
-        }
-
-        // Initialize BSF
-        let ret = unsafe { av_bsf_init(bsf_ctx) };
-        if ret < 0 {
-            // SAFETY: bsf_ctx allocated, params set but init failed — safe to free
-            unsafe { av_bsf_free(&mut bsf_ctx) };
-            bail!("av_bsf_init failed: error {ret}");
-        }
-
         // 5. Filter graph (inline)
         let video_filter =
             build_filter_graph(&hw_device_ctx, &frames_rgb, width, height, fps)?;
@@ -386,7 +308,6 @@ impl EncState {
 
         Ok(Self {
             enc_video,
-            bsf_ctx,
             frames_rgb,
             frames_yuv,
             video_filter,
@@ -469,7 +390,7 @@ impl EncState {
             }
         }
 
-        // SAFETY: Sending null frame signals end of stream.
+        // SAFETY: Sending null frame signals end of stream to encoder.
         unsafe {
             ffi::avcodec_send_frame(self.enc_video.as_mut_ptr(), ptr::null());
         }
@@ -477,34 +398,6 @@ impl EncState {
         let start_ts = self.starting_timestamp.unwrap_or(0);
         self.drain_encoder(start_ts)?;
 
-        // SAFETY: Sending null packet signals end-of-stream to BSF
-        unsafe { av_bsf_send_packet(self.bsf_ctx, ptr::null_mut()) };
-        loop {
-            let mut bsf_pkt = ff::Packet::empty();
-            let ret = unsafe {
-                av_bsf_receive_packet(self.bsf_ctx, bsf_pkt.as_mut_ptr())
-            };
-            if ret < 0 { break; }
-            let enc_tb = self.enc_video.time_base();
-            let stream_tb = unsafe {
-                let streams = (*self.octx.as_ptr()).streams;
-                let st = *streams.add(0);
-                ff::Rational::from((*st).time_base)
-            };
-            bsf_pkt.rescale_ts(enc_tb, stream_tb);
-            if let Some(pts) = bsf_pkt.pts() {
-                bsf_pkt.set_pts(Some(pts - start_ts));
-            }
-            if let Some(dts) = bsf_pkt.dts() {
-                bsf_pkt.set_dts(Some(dts - start_ts));
-            }
-            bsf_pkt.set_stream(0);
-            bsf_pkt.write_interleaved(&mut self.octx).map_err(|e| {
-                anyhow::anyhow!("Failed to write BSF flush packet: {e}")
-            })?;
-            self.frames_written = true;
-        }
-
         // Write trailer only if at least one frame was encoded.
         if self.frames_written {
             self.octx.write_trailer().map_err(|e| {
@@ -516,7 +409,6 @@ impl EncState {
     }
 
     fn drain_encoder(&mut self, start_ts: i64) -> Result<()> {
-        let stream_index: i32 = 0;
         loop {
             let mut pkt = ff::Packet::empty();
             // SAFETY: avcodec_receive_packet retrieves an encoded packet.
@@ -530,72 +422,34 @@ impl EncState {
                 bail!("avcodec_receive_packet failed: error {ret}");
             }
 
-            // SAFETY: av_bsf_send_packet sends the encoded packet through the BSF filter.
+            // Rescale timestamps from encoder time_base to stream time_base
-            // On success, the BSF takes ownership of the packet data (via av_packet_move_ref).
+            let enc_tb = self.enc_video.time_base();
-            let ret = unsafe { av_bsf_send_packet(self.bsf_ctx, pkt.as_mut_ptr()) };
+            let stream_tb = unsafe {
-            if ret == ffi::AVERROR(ffi::EAGAIN) {
+                let streams = (*self.octx.as_ptr()).streams;
-                // BSF buffer full — break and retry next drain cycle
+                let st = *streams.add(0);
-                break;
+                ff::Rational::from((*st).time_base)
+            };
+            pkt.rescale_ts(enc_tb, stream_tb);
+
+            // Offset timestamps so first frame starts at 0
+            if let Some(pts) = pkt.pts() {
+                pkt.set_pts(Some(pts - start_ts));
             }
-            if ret < 0 {
+            if let Some(dts) = pkt.dts() {
-                bail!("av_bsf_send_packet failed: error {ret}");
+                pkt.set_dts(Some(dts - start_ts));
             }
 
-            // Drain all BSF output packets
+            pkt.set_stream(0);
-            loop {
+            pkt.write_interleaved(&mut self.octx).map_err(|e| {
-                let mut bsf_pkt = ff::Packet::empty();
+                anyhow::anyhow!("Failed to write packet: {e}")
-                // SAFETY: av_bsf_receive_packet retrieves a BSF-processed packet.
+            })?;
-                let ret = unsafe {
-                    av_bsf_receive_packet(self.bsf_ctx, bsf_pkt.as_mut_ptr())
-                };
-                if ret == ffi::AVERROR(ffi::EAGAIN) {
-                    break; // No more output yet
-                }
-                if ret == ffi::AVERROR_EOF {
-                    break; // BSF drained
-                }
-                if ret < 0 {
-                    bail!("av_bsf_receive_packet failed: error {ret}");
-                }
 
-                // Rescale and offset on BSF output packet (NOT original pkt)
+            self.frames_written = true;
-                let enc_tb = self.enc_video.time_base();
-                let stream_tb = unsafe {
-                    let streams = (*self.octx.as_ptr()).streams;
-                    let st = *streams.add(0);
-                    ff::Rational::from((*st).time_base)
-                };
-                bsf_pkt.rescale_ts(enc_tb, stream_tb);
-
-                if let Some(pts) = bsf_pkt.pts() {
-                    bsf_pkt.set_pts(Some(pts - start_ts));
-                }
-                if let Some(dts) = bsf_pkt.dts() {
-                    bsf_pkt.set_dts(Some(dts - start_ts));
-                }
-
-                bsf_pkt.set_stream(stream_index as usize);
-                bsf_pkt.write_interleaved(&mut self.octx).map_err(|e| {
-                    anyhow::anyhow!("Failed to write packet: {e}")
-                })?;
-
-                self.frames_written = true;
-            }
         }
         Ok(())
     }
 }
 
-impl Drop for EncState {
-    fn drop(&mut self) {
-        // SAFETY: av_bsf_free releases the BSF context and all associated resources.
-        // It handles null safely (returns immediately if *pctx is null).
-        if !self.bsf_ctx.is_null() {
-            unsafe { av_bsf_free(&mut self.bsf_ctx) };
-        }
-    }
-}
-
 // ---------------------------------------------------------------------------
 // Filter graph (inline)
 // ---------------------------------------------------------------------------