-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(video): add video test tool (#298)
was approved by Darius via chat
- Loading branch information
1 parent
be14ae7
commit e610bd6
Showing
17 changed files
with
1,133 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
[package] | ||
name = "opencv-test" | ||
version = "0.1.0" | ||
edition = "2021" | ||
|
||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html | ||
|
||
[dependencies] | ||
anyhow = { workspace = true } | ||
clap = { version = "4.0", features=["derive"] } | ||
opencv = { workspace = true } | ||
tokio = { workspace = true, features=["full"] } | ||
|
||
av-data = { workspace = true } | ||
libaom = { workspace = true } | ||
|
||
openh264 = { git="https://github.com/sdwoodbury/openh264-rs", rev="36abe7e7349c890684457fabaf5d90cb4b716cec", optional = true } | ||
x264 = { version = "0.5", optional = true } | ||
rav1e = { version = "0.6", default-features = false, optional = true } | ||
|
||
[features] | ||
all = ["dep:openh264", "dep:x264", "dep:rav1e"] | ||
default = [] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
# build dependencies | ||
|
||
# testing all codecs | ||
`cargo build --package opencv-test -F all` | ||
|
||
## Mac OS | ||
clang - comes with llvm. `brew install llvm`. symlink $(brew --prefix llvm)/lib/libclang.dylib to wherever is needed | ||
opencv | ||
|
||
## Linux | ||
libopencv-dev | ||
libstdc++-12-dev | ||
clang | ||
libclang-dev | ||
libx264-dev | ||
libaom-dev | ||
|
||
## video file extensions and codecs known to work with opencv | ||
- .avi / MJPG | ||
- .mkv / H264 | ||
- .mp4 / avc1 | ||
|
||
## testing a .mov file | ||
ffmpeg -i <filename.mov> output_%04d.png |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,136 @@ | ||
use crate::{encode::Mode, utils::yuv::*}; | ||
|
||
use super::{Args, EncodingType}; | ||
use anyhow::{bail, Result}; | ||
use av_data::{frame::FrameType, timeinfo::TimeInfo}; | ||
use std::{ | ||
fs::OpenOptions, | ||
io::{BufWriter, Write}, | ||
sync::Arc, | ||
}; | ||
|
||
use libaom::encoder::*; | ||
|
||
use opencv::{ | ||
core::{Mat_AUTO_STEP, CV_32F}, | ||
prelude::*, | ||
videoio, | ||
}; | ||
|
||
pub fn encode_aom(args: Args) -> Result<()> { | ||
let color_scale = args.color_scale.unwrap_or(ColorScale::HdTv); | ||
let optimized_mode = args.mode.unwrap_or(Mode::Normal); | ||
let is_lossy = args | ||
.encoding_type | ||
.map(|t| matches!(t, EncodingType::Lossy)) | ||
.unwrap_or(true); | ||
let multiplier: usize = if is_lossy { 1 } else { 2 }; | ||
|
||
let cam = videoio::VideoCapture::from_file(&args.input, videoio::CAP_ANY)?; | ||
let opened = videoio::VideoCapture::is_opened(&cam)?; | ||
if !opened { | ||
panic!("Unable to open video file!"); | ||
} | ||
|
||
// https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html | ||
let frame_width = cam.get(3)? as u32; | ||
let frame_height = cam.get(4)? as u32; | ||
let _fps = cam.get(5)? as f32; | ||
|
||
let output_file = OpenOptions::new() | ||
.read(false) | ||
.write(true) | ||
.create(true) | ||
.truncate(true) | ||
.open(args.output)?; | ||
let mut writer = BufWriter::new(output_file); | ||
|
||
let mut encoder_config = match AV1EncoderConfig::new_with_usage(AomUsage::RealTime) { | ||
Ok(r) => r, | ||
Err(e) => bail!("failed to get Av1EncoderConfig: {e:?}"), | ||
}; | ||
encoder_config.g_h = frame_height * multiplier as u32; | ||
encoder_config.g_w = frame_width * multiplier as u32; | ||
let mut encoder = match encoder_config.get_encoder() { | ||
Ok(r) => r, | ||
Err(e) => bail!("failed to get Av1Encoder: {e:?}"), | ||
}; | ||
|
||
// this is for testing an optimized version | ||
let color_scale_idx = color_scale.to_idx(); | ||
let mut m = [ | ||
// these scales are for turning RGB to YUV. but the input is in BGR. | ||
Y_SCALE[color_scale_idx], | ||
U_SCALE[color_scale_idx], | ||
V_SCALE[color_scale_idx], | ||
]; | ||
m[0].reverse(); | ||
m[1].reverse(); | ||
m[2].reverse(); | ||
let p = m.as_ptr() as *mut std::ffi::c_void; | ||
let m = unsafe { Mat::new_rows_cols_with_data(3, 3, CV_32F, p, Mat_AUTO_STEP) } | ||
.expect("failed to make xform matrix"); | ||
|
||
let pixel_format = *av_data::pixel::formats::YUV420; | ||
let pixel_format = Arc::new(pixel_format); | ||
for (idx, mut frame) in crate::VideoFileIter::new(cam).enumerate() { | ||
println!("read new frame"); | ||
let sz = frame.size()?; | ||
let width = sz.width as usize; | ||
let height = sz.height as usize; | ||
if width == 0 { | ||
continue; | ||
} | ||
|
||
let yuv = match optimized_mode { | ||
Mode::Faster => bgr_to_yuv420_lossy_faster(frame, &m, width, height, color_scale), | ||
_ => { | ||
let p = frame.data_mut(); | ||
let len = width * height * 3; | ||
let s = std::ptr::slice_from_raw_parts(p, len as _); | ||
let s: &[u8] = unsafe { &*s }; | ||
|
||
if is_lossy { | ||
bgr_to_yuv420_lossy(s, width, height, color_scale) | ||
} else { | ||
bgr_to_yuv420(s, width, height, color_scale) | ||
} | ||
} | ||
}; | ||
|
||
let yuv_buf = YUV420Buf { | ||
data: yuv, | ||
width: width * multiplier, | ||
height: height * multiplier, | ||
}; | ||
|
||
let frame = av_data::frame::Frame { | ||
kind: av_data::frame::MediaKind::Video(av_data::frame::VideoInfo::new( | ||
yuv_buf.width, | ||
yuv_buf.height, | ||
false, | ||
FrameType::I, | ||
pixel_format.clone(), | ||
)), | ||
buf: Box::new(yuv_buf), | ||
t: TimeInfo { | ||
pts: Some(idx as i64 * 60), | ||
..Default::default() | ||
}, | ||
}; | ||
|
||
println!("encoding"); | ||
if let Err(e) = encoder.encode(&frame) { | ||
bail!("encoding error: {e}"); | ||
} | ||
|
||
println!("calling get_packet"); | ||
while let Some(packet) = encoder.get_packet() { | ||
if let AOMPacket::Packet(p) = packet { | ||
let _ = writer.write(&p.data)?; | ||
} | ||
} | ||
} | ||
writer.flush()?; | ||
Ok(()) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,62 @@ | ||
use super::Args; | ||
use anyhow::Result; | ||
|
||
use crate::utils::yuv::*; | ||
use opencv::{prelude::*, videoio}; | ||
use std::{ | ||
fs::OpenOptions, | ||
io::{BufWriter, Write}, | ||
}; | ||
|
||
pub fn encode_h264(args: Args) -> Result<()> { | ||
let color_scale = args.color_scale.unwrap_or(ColorScale::Full); | ||
let cam = videoio::VideoCapture::from_file(&args.input, videoio::CAP_ANY)?; | ||
let opened = videoio::VideoCapture::is_opened(&cam)?; | ||
if !opened { | ||
panic!("Unable to open video file!"); | ||
} | ||
|
||
// https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html | ||
let frame_width = cam.get(3)? as u32; | ||
let frame_height = cam.get(4)? as u32; | ||
let fps = cam.get(5)? as _; | ||
|
||
let output_file = OpenOptions::new() | ||
.read(false) | ||
.write(true) | ||
.create(true) | ||
.truncate(true) | ||
.open(args.output)?; | ||
let mut writer = BufWriter::new(output_file); | ||
|
||
let config = openh264::encoder::EncoderConfig::new(frame_width * 2, frame_height * 2) | ||
.max_frame_rate(fps); //.rate_control_mode(openh264::encoder::RateControlMode::Timestamp); | ||
|
||
let mut encoder = openh264::encoder::Encoder::with_config(config)?; | ||
|
||
for mut frame in crate::VideoFileIter::new(cam) { | ||
let sz = frame.size()?; | ||
let width = sz.width as usize; | ||
let height = sz.height as usize; | ||
if width == 0 { | ||
continue; | ||
} | ||
let p = frame.data_mut(); | ||
let len = width * height * 3; | ||
let s = std::ptr::slice_from_raw_parts(p, len as _); | ||
let s: &[u8] = unsafe { &*s }; | ||
|
||
let yuv = bgr_to_yuv420(s, width, height, color_scale); | ||
|
||
let yuv_buf = YUV420Buf { | ||
data: yuv, | ||
width: width * 2, | ||
height: height * 2, | ||
}; | ||
|
||
let encoded_stream = encoder.encode(&yuv_buf)?; | ||
encoded_stream.write(&mut writer)?; | ||
} | ||
writer.flush()?; | ||
Ok(()) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
mod aom; | ||
#[cfg(feature = "all")] | ||
mod h264; | ||
#[cfg(feature = "all")] | ||
mod rav1e; | ||
#[cfg(feature = "all")] | ||
mod x264; | ||
pub use crate::encode::aom::encode_aom; | ||
#[cfg(feature = "all")] | ||
pub use crate::encode::h264::encode_h264; | ||
#[cfg(feature = "all")] | ||
pub use crate::encode::rav1e::encode_rav1e; | ||
#[cfg(feature = "all")] | ||
pub use crate::encode::x264::encode_x264; | ||
use crate::utils::yuv::ColorScale; | ||
|
||
use clap::Parser; | ||
|
||
// transforms the input file to h264 | ||
#[derive(Parser, Debug)] | ||
pub struct Args { | ||
/// an mp4 file generated by opencv | ||
pub input: String, | ||
/// name of the file to save | ||
pub output: String, | ||
/// The codec to use | ||
pub codec: CodecTypes, | ||
/// Optional parameter. defaults to Lossy. | ||
/// NotLossy doubles the size of each frame so that converting to YUV420 | ||
/// doesn't lose any chromiance information. | ||
pub encoding_type: Option<EncodingType>, | ||
///specifies the RGB to YUV transformation matrix | ||
pub color_scale: Option<ColorScale>, | ||
/// use optimized version. currently only works for aom in lossy mode. | ||
/// if set, overrides encoding_type | ||
pub mode: Option<Mode>, | ||
} | ||
|
||
#[derive(Debug, Clone, clap::ValueEnum)] | ||
pub enum CodecTypes { | ||
#[cfg(feature = "all")] | ||
/// OpenH264 | ||
H264, | ||
#[cfg(feature = "all")] | ||
/// x264 | ||
X264, | ||
#[cfg(feature = "all")] | ||
/// av1 (rav1e) | ||
RAV1E, | ||
/// av1 (aom) | ||
AOM, | ||
} | ||
|
||
#[derive(Debug, Clone, clap::ValueEnum)] | ||
pub enum EncodingType { | ||
/// convert from BGR24 to YUV420 | ||
Lossy, | ||
/// expand BGR24 image before converting to YUV420 | ||
NotLossy, | ||
} | ||
|
||
#[derive(Debug, Clone, Copy, clap::ValueEnum)] | ||
pub enum Mode { | ||
Normal, | ||
/// attempt to use opencv matrix multiplication | ||
/// to speed up the conversion from BGR to YUV | ||
Faster, | ||
} |
Oops, something went wrong.