Using MJPEG instead of YUV

This was actually a cool discovery.
It turns out webcams (logitech ones, at least) can also emit MJPEG
frames rather than the standard YUV420p. And on that format, they
support slightly higher resolutions/framerates

src/av/decoder_ctx.rs

@@ -6,6 +6,8 @@ use ffmpeg4_ffi::sys;
 
 use super::utils;
 
+use crate::opts;
+
 pub struct DecoderCtx {
  pub av: *mut sys::AVFormatContext,
  pub video_stream_index: usize,
@@ -16,38 +18,38 @@ pub struct DecoderCtx {
 }
 
 impl DecoderCtx {
- pub fn open(path: &str, framerate: i64, width: i32, height: i32) -> DecoderCtx {
+ pub fn open(path: String, args: &opts::Forward) -> DecoderCtx {
  unsafe {
  let mut av = sys::avformat_alloc_context();
 
  let mut options: *mut sys::AVDictionary = null_mut();
 
  let framerate_key = utils::str_to_c_str("framerate");
- sys::av_dict_set_int(&mut options, framerate_key.as_ptr(), framerate, 0);
+ sys::av_dict_set_int(&mut options, framerate_key.as_ptr(), args.fps, 0);
 
  sys::av_dict_set(
  &mut options,
  utils::str_to_c_str("video_size").as_ptr(),
- utils::string_to_c_str(format!("{}x{}", width, height)).as_ptr(),
+ utils::string_to_c_str(format!("{}x{}", args.width, args.height)).as_ptr(),
  0,
  );
 
  sys::av_dict_set(
  &mut options,
  utils::str_to_c_str("input_format").as_ptr(),
- utils::str_to_c_str("yuyv422").as_ptr(),
+ utils::str_to_c_str("mjpeg").as_ptr(),
  0,
  );
 
  let response = sys::avformat_open_input(
  &mut av,
- utils::str_to_c_str(path).as_ptr(),
+ utils::str_to_c_str(path.as_str()).as_ptr(),
  null_mut(),
  &mut options,
  );
 
  if utils::check_error(response) {
- panic!("could not open {}", path);
+ panic!("could not open {}", path.as_str());
  }
 
  let mut decoder = DecoderCtx {

src/av/encoder_ctx.rs

@@ -11,13 +11,14 @@ pub struct EncoderCtx {
  pub codec_ctx: *mut sys::AVCodecContext,
  pub codec: *mut sys::AVCodec,
  pub stream: *mut sys::AVStream,
+ pub path: String,
 }
 
 unsafe impl Send for EncoderCtx {}
 
 impl EncoderCtx {
- pub fn new(path: &str, format: &str) -> EncoderCtx {
- let path_str = utils::str_to_c_str(path);
+ pub fn new(path: String, format: &str) -> EncoderCtx {
+ let path_str = utils::str_to_c_str(path.as_str());
  let format_str = utils::str_to_c_str(format);
 
  let mut av: *mut sys::AVFormatContext = null_mut();
@@ -33,6 +34,7 @@ impl EncoderCtx {
  codec: null_mut(),
  codec_ctx: null_mut(),
  stream: null_mut(),
+ path: path,
  }
  }
 
@@ -65,8 +67,8 @@ impl EncoderCtx {
  }
  }
 
- pub fn open_file(&mut self, path: &str) {
- let path_str = utils::str_to_c_str(path);
+ pub fn open_file(&mut self) {
+ let path_str = utils::str_to_c_str(self.path.as_str());
 
  let response = unsafe {
  sys::avio_open(

src/canvas/mod.rs

@@ -10,12 +10,9 @@ use sdl2::render::TextureAccess;
 use sdl2::video::Window;
 use sdl2::EventPump;
 
-use std::sync::{Arc, Mutex};
+use crate::types;
 
-pub struct FrameMsg(pub *mut ffmpeg4_ffi::sys::AVFrame);
-unsafe impl Send for FrameMsg {}
-
-pub type ThreadSafeFrame = Arc<Mutex<FrameMsg>>;
+pub type ThreadSafeFrame = types::FrameMsg;
 
 pub fn create(
  width: i32,
@@ -55,7 +52,7 @@ fn render_loop(window: Window, event_pump: &mut EventPump, receiver: Receiver<Th
 
  'running: loop {
  let frame_msg = receiver.recv().expect("Failed to receive frame");
- let frame = frame_msg.lock().unwrap().0;
+ let frame = frame_msg.0;
 
  let data = unsafe { slice::from_raw_parts((*frame).data[0], (width * height) as usize) };
  let linesize = unsafe { (*frame).linesize[0] };

src/cmds/forward.rs

@@ -2,131 +2,44 @@ extern crate crossbeam_channel;
 extern crate sdl2;
 
 use std::path::PathBuf;
-use std::ptr::null_mut;
+use std::time::Instant;
 
 use ffmpeg4_ffi::sys;
 
 use crate::av::decoder_ctx::DecoderCtx;
 use crate::av::encoder_ctx::EncoderCtx;
-use crate::canvas;
-use crate::filter;
-use crate::opts;
-use std::sync::{Arc, Mutex};
-
-const BGR: sys::AVPixelFormat = sys::AVPixelFormat_AV_PIX_FMT_BGR24;
-const YUV: sys::AVPixelFormat = sys::AVPixelFormat_AV_PIX_FMT_YUV420P;
+use crate::{canvas, opts, pipeline};
 
 pub fn run(args: opts::Forward) {
- let width = args.width;
- let height = args.height;
-
  unsafe { sys::avdevice_register_all() };
 
- let input_path = args.input.as_str();
- let output_path = args.output.as_str();
-
- assert!(
- PathBuf::from(input_path).exists(),
- "file {} does not exist",
- input_path
- );
+ let mut decoder = DecoderCtx::open(args.input.clone(), &args);
+ let raw_pix_fmt = unsafe { (*decoder.codec_ctx).pix_fmt };
 
- let mut decoder = DecoderCtx::open(input_path, 30, width, height);
-
- let mut encoder = EncoderCtx::new(output_path, "v4l2");
+ let mut encoder = EncoderCtx::new(args.output.clone(), "v4l2");
  encoder.load_stream(&decoder, sys::AVCodecID_AV_CODEC_ID_RAWVIDEO);
- encoder.open_file(output_path);
+ encoder.open_file();
 
  let (sender, receiver) = crossbeam_channel::unbounded();
 
  if args.preview {
- canvas::create(width, height, receiver);
+ canvas::create(args.width, args.height, receiver);
  }
 
- let frame_raw = unsafe { sys::av_frame_alloc() };
- let frame_bgr = alloc_frame(width, height, BGR);
- let frame_fil = alloc_frame(width, height, BGR);
- let frame_yuv = alloc_frame(width, height, YUV);
-
- let yuv2bgr = sws_alloc(width, height, unsafe { (*decoder.codec_ctx).pix_fmt }, BGR);
- let bgr2yuv = sws_alloc(width, height, BGR, YUV);
-
- let msg = Arc::new(Mutex::new(canvas::FrameMsg(frame_fil)));
+ let mut pipeline = pipeline::Pipeline::new(&args, raw_pix_fmt);
 
  loop {
+ println!("{:?}", raw_pix_fmt);
  if args.preview {
- sender.send(msg.clone()).unwrap();
+ sender.send(pipeline.fil_as_msg()).unwrap();
  }
 
- decoder.read_frame(&frame_raw);
- sws_convert(yuv2bgr, frame_raw, frame_bgr);
- filter::blur(frame_bgr, frame_fil, args.blur);
- sws_convert(bgr2yuv, frame_fil, frame_yuv);
- encoder.encode(&decoder, &frame_yuv);
- }
-}
-
-pub fn alloc_frame(width: i32, height: i32, format: sys::AVPixelFormat) -> *mut sys::AVFrame {
- unsafe {
- let frame = sys::av_frame_alloc();
-
- (*frame).width = width;
- (*frame).height = height;
- (*frame).format = format;
+ println!("reading frame");
+ let now = Instant::now();
+ decoder.read_frame(pipeline.raw_ref());
+ println!("read frame: {}", now.elapsed().as_millis());
 
- sys::av_frame_get_buffer(frame, 0);
-
- let size = sys::avpicture_get_size(format, width, height);
- let buffer = sys::av_malloc(size as usize);
-
- sys::avpicture_fill(
- frame as *mut sys::AVPicture,
- buffer as *mut u8,
- format,
- width,
- height,
- );
-
- (*frame).pts = 0;
-
- frame
- }
-}
-
-pub fn sws_alloc(
- width: i32,
- height: i32,
- from: sys::AVPixelFormat,
- to: sys::AVPixelFormat,
-) -> *mut sys::SwsContext {
- unsafe {
- sys::sws_getContext(
- width,
- height,
- from,
- width,
- height,
- to,
- sys::SWS_BILINEAR as i32,
- null_mut(),
- null_mut(),
- null_mut(),
- )
+ pipeline.process();
+ encoder.encode(&decoder, pipeline.yuv_ref());
  }
 }
-
-pub fn sws_convert(ctx: *mut sys::SwsContext, from: *mut sys::AVFrame, to: *mut sys::AVFrame) {
- unsafe {
- sys::sws_scale(
- ctx,
- (*from).data.as_ptr() as *const *const u8,
- (*from).linesize.as_ptr() as *const i32,
- 0,
- (*from).height,
- (*to).data.as_ptr() as *const *mut u8,
- (*to).linesize.as_ptr() as *const i32,
- );
-
- (*to).pts = (*from).pts;
- };
-}

src/main.rs

@@ -3,6 +3,8 @@ mod canvas;
 mod cmds;
 mod filter;
 mod opts;
+mod pipeline;
+mod types;
 
 use clap::Clap;
 

src/opts.rs

@@ -15,7 +15,7 @@ pub enum SubCommand {
  UI,
 }
 
-#[derive(Clap)]
+#[derive(Clap, Clone)]
 pub struct Forward {
  #[clap(short = "i", long = "input")]
  pub input: String,
@@ -29,4 +29,6 @@ pub struct Forward {
  pub height: i32,
  #[clap(short = "b", long = "blur", default_value = "20")]
  pub blur: i32,
+ #[clap(short = "f", long = "fps", default_value = "20")]
+ pub fps: i64,
 }