feat(video): add video test tool (#298)

was approved by Darius via chat

Cargo.toml

@@ -70,8 +70,6 @@ uuid = { version = "1", features = ["serde", "v4"] }
 derive_more = "0.99"
 paste = "1.0"
 libc = "0.2"
-mp4 = { git="https://github.com/sdwoodbury/mp4-rust", rev="9887024513f59e5fcda9f44442b2a3a2cdfe9706" }
-opus = { git = "https://github.com/sdwoodbury/opus-rs", rev="3f07443dac1907dbcc847181a0d40d561592072b" }
 sata = { git = "https://github.com/Satellite-im/Sata", rev = "9b5a5d441c816968bc75ea8b99335f728be259ef" }
 tracing = { version = "0.1" }
 tracing-futures = { version = "0.2" }
@@ -81,3 +79,11 @@ void = "1"
 
 #ipfs dependency
 rust-ipfs = "=0.3.19"
+
+# Blink related crates
+# av-data is needed to use libaom. need to ensure that Warp and libaom use the same version of av-data
+av-data = "*"
+libaom = { git="https://github.com/rust-av/aom-rs", rev="7aca932ea5f6cc1f0e9eb54f7aabcce234f18864" }
+mp4 = { git="https://github.com/sdwoodbury/mp4-rust", rev="9887024513f59e5fcda9f44442b2a3a2cdfe9706" }
+opus = { git = "https://github.com/sdwoodbury/opus-rs", rev="3f07443dac1907dbcc847181a0d40d561592072b" }
+opencv = { version = "0.82", features=["clang-runtime"] }

extensions/warp-ipfs/examples/identity-interface.rs

@@ -2,7 +2,6 @@ use clap::Parser;
 use comfy_table::Table;
 use futures::prelude::*;
 use rustyline_async::{Readline, ReadlineError};
-use warp_ipfs::WarpIpfsBuilder;
 use std::io::Write;
 use std::path::PathBuf;
 use std::str::FromStr;
@@ -13,6 +12,7 @@ use warp::multipass::identity::{Identifier, IdentityStatus, IdentityUpdate};
 use warp::multipass::MultiPass;
 use warp::tesseract::Tesseract;
 use warp_ipfs::config::{Config, Discovery};
+use warp_ipfs::WarpIpfsBuilder;
 
 #[derive(Debug, Parser)]
 #[clap(name = "identity-interface")]

extensions/warp-ipfs/src/lib.rs

@@ -135,7 +135,7 @@ impl WarpIpfsBuilder {
 
         let mp = Box::new(instance.clone()) as Box<_>;
         let rg = Box::new(instance.clone()) as Box<_>;
-        let fs = Box::new(instance.clone()) as Box<_>;
+        let fs = Box::new(instance) as Box<_>;
 
         Ok((mp, rg, fs))
     }

extensions/warp-ipfs/src/store/mod.rs

@@ -19,7 +19,7 @@ use ipfs::{Multiaddr, PeerId, Protocol, PublicKey};
 use warp::{
     crypto::{
         cipher::Cipher,
-        did_key::{Generate, ECDH, CoreSign},
+        did_key::{CoreSign, Generate, ECDH},
         hash::sha256_hash,
         zeroize::Zeroizing,
         DIDKey, Ed25519KeyPair, KeyMaterial, DID,

tools/blink-webrtc-cli/src/main.rs

@@ -20,7 +20,10 @@ use warp::crypto::DID;
 use warp::multipass::identity::Identity;
 
 use warp::tesseract::Tesseract;
-use warp_ipfs::{config::{Config, UpdateEvents}, WarpIpfsBuilder};
+use warp_ipfs::{
+    config::{Config, UpdateEvents},
+    WarpIpfsBuilder,
+};
 
 mod logger;
 

tools/opencv-test/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "opencv-test"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = { workspace = true }
+clap = { version = "4.0", features=["derive"] }
+opencv = { workspace = true }
+tokio = { workspace = true, features=["full"] }
+
+av-data = { workspace = true }
+libaom = { workspace = true }
+
+openh264 = { git="https://github.com/sdwoodbury/openh264-rs", rev="36abe7e7349c890684457fabaf5d90cb4b716cec", optional = true }
+x264 = { version = "0.5", optional = true }
+rav1e = { version = "0.6", default-features = false, optional = true }
+
+[features]
+all = ["dep:openh264", "dep:x264", "dep:rav1e"]
+default = []

tools/opencv-test/README.md

@@ -0,0 +1,24 @@
+# build dependencies
+
+# testing all codecs
+`cargo build --package opencv-test -F all`
+
+## Mac OS
+clang  - comes with llvm. `brew install llvm`. symlink $(brew --prefix llvm)/lib/libclang.dylib to wherever is needed
+opencv
+
+## Linux
+libopencv-dev
+libstdc++-12-dev
+clang
+libclang-dev
+libx264-dev
+libaom-dev
+
+## video file extensions and codecs known to work with opencv
+- .avi / MJPG
+- .mkv / H264
+- .mp4 / avc1
+
+## testing a .mov file
+ffmpeg -i <filename.mov> output_%04d.png

tools/opencv-test/src/encode/aom.rs

@@ -0,0 +1,136 @@
+use crate::{encode::Mode, utils::yuv::*};
+
+use super::{Args, EncodingType};
+use anyhow::{bail, Result};
+use av_data::{frame::FrameType, timeinfo::TimeInfo};
+use std::{
+    fs::OpenOptions,
+    io::{BufWriter, Write},
+    sync::Arc,
+};
+
+use libaom::encoder::*;
+
+use opencv::{
+    core::{Mat_AUTO_STEP, CV_32F},
+    prelude::*,
+    videoio,
+};
+
+pub fn encode_aom(args: Args) -> Result<()> {
+    let color_scale = args.color_scale.unwrap_or(ColorScale::HdTv);
+    let optimized_mode = args.mode.unwrap_or(Mode::Normal);
+    let is_lossy = args
+        .encoding_type
+        .map(|t| matches!(t, EncodingType::Lossy))
+        .unwrap_or(true);
+    let multiplier: usize = if is_lossy { 1 } else { 2 };
+
+    let cam = videoio::VideoCapture::from_file(&args.input, videoio::CAP_ANY)?;
+    let opened = videoio::VideoCapture::is_opened(&cam)?;
+    if !opened {
+        panic!("Unable to open video file!");
+    }
+
+    // https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html
+    let frame_width = cam.get(3)? as u32;
+    let frame_height = cam.get(4)? as u32;
+    let _fps = cam.get(5)? as f32;
+
+    let output_file = OpenOptions::new()
+        .read(false)
+        .write(true)
+        .create(true)
+        .truncate(true)
+        .open(args.output)?;
+    let mut writer = BufWriter::new(output_file);
+
+    let mut encoder_config = match AV1EncoderConfig::new_with_usage(AomUsage::RealTime) {
+        Ok(r) => r,
+        Err(e) => bail!("failed to get Av1EncoderConfig: {e:?}"),
+    };
+    encoder_config.g_h = frame_height * multiplier as u32;
+    encoder_config.g_w = frame_width * multiplier as u32;
+    let mut encoder = match encoder_config.get_encoder() {
+        Ok(r) => r,
+        Err(e) => bail!("failed to get Av1Encoder: {e:?}"),
+    };
+
+    // this is for testing an optimized version
+    let color_scale_idx = color_scale.to_idx();
+    let mut m = [
+        // these scales are for turning RGB to YUV. but the input is in BGR.
+        Y_SCALE[color_scale_idx],
+        U_SCALE[color_scale_idx],
+        V_SCALE[color_scale_idx],
+    ];
+    m[0].reverse();
+    m[1].reverse();
+    m[2].reverse();
+    let p = m.as_ptr() as *mut std::ffi::c_void;
+    let m = unsafe { Mat::new_rows_cols_with_data(3, 3, CV_32F, p, Mat_AUTO_STEP) }
+        .expect("failed to make xform matrix");
+
+    let pixel_format = *av_data::pixel::formats::YUV420;
+    let pixel_format = Arc::new(pixel_format);
+    for (idx, mut frame) in crate::VideoFileIter::new(cam).enumerate() {
+        println!("read new frame");
+        let sz = frame.size()?;
+        let width = sz.width as usize;
+        let height = sz.height as usize;
+        if width == 0 {
+            continue;
+        }
+
+        let yuv = match optimized_mode {
+            Mode::Faster => bgr_to_yuv420_lossy_faster(frame, &m, width, height, color_scale),
+            _ => {
+                let p = frame.data_mut();
+                let len = width * height * 3;
+                let s = std::ptr::slice_from_raw_parts(p, len as _);
+                let s: &[u8] = unsafe { &*s };
+
+                if is_lossy {
+                    bgr_to_yuv420_lossy(s, width, height, color_scale)
+                } else {
+                    bgr_to_yuv420(s, width, height, color_scale)
+                }
+            }
+        };
+
+        let yuv_buf = YUV420Buf {
+            data: yuv,
+            width: width * multiplier,
+            height: height * multiplier,
+        };
+
+        let frame = av_data::frame::Frame {
+            kind: av_data::frame::MediaKind::Video(av_data::frame::VideoInfo::new(
+                yuv_buf.width,
+                yuv_buf.height,
+                false,
+                FrameType::I,
+                pixel_format.clone(),
+            )),
+            buf: Box::new(yuv_buf),
+            t: TimeInfo {
+                pts: Some(idx as i64 * 60),
+                ..Default::default()
+            },
+        };
+
+        println!("encoding");
+        if let Err(e) = encoder.encode(&frame) {
+            bail!("encoding error: {e}");
+        }
+
+        println!("calling get_packet");
+        while let Some(packet) = encoder.get_packet() {
+            if let AOMPacket::Packet(p) = packet {
+                let _ = writer.write(&p.data)?;
+            }
+        }
+    }
+    writer.flush()?;
+    Ok(())
+}

tools/opencv-test/src/encode/h264.rs

@@ -0,0 +1,62 @@
+use super::Args;
+use anyhow::Result;
+
+use crate::utils::yuv::*;
+use opencv::{prelude::*, videoio};
+use std::{
+    fs::OpenOptions,
+    io::{BufWriter, Write},
+};
+
+pub fn encode_h264(args: Args) -> Result<()> {
+    let color_scale = args.color_scale.unwrap_or(ColorScale::Full);
+    let cam = videoio::VideoCapture::from_file(&args.input, videoio::CAP_ANY)?;
+    let opened = videoio::VideoCapture::is_opened(&cam)?;
+    if !opened {
+        panic!("Unable to open video file!");
+    }
+
+    // https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html
+    let frame_width = cam.get(3)? as u32;
+    let frame_height = cam.get(4)? as u32;
+    let fps = cam.get(5)? as _;
+
+    let output_file = OpenOptions::new()
+        .read(false)
+        .write(true)
+        .create(true)
+        .truncate(true)
+        .open(args.output)?;
+    let mut writer = BufWriter::new(output_file);
+
+    let config = openh264::encoder::EncoderConfig::new(frame_width * 2, frame_height * 2)
+        .max_frame_rate(fps); //.rate_control_mode(openh264::encoder::RateControlMode::Timestamp);
+
+    let mut encoder = openh264::encoder::Encoder::with_config(config)?;
+
+    for mut frame in crate::VideoFileIter::new(cam) {
+        let sz = frame.size()?;
+        let width = sz.width as usize;
+        let height = sz.height as usize;
+        if width == 0 {
+            continue;
+        }
+        let p = frame.data_mut();
+        let len = width * height * 3;
+        let s = std::ptr::slice_from_raw_parts(p, len as _);
+        let s: &[u8] = unsafe { &*s };
+
+        let yuv = bgr_to_yuv420(s, width, height, color_scale);
+
+        let yuv_buf = YUV420Buf {
+            data: yuv,
+            width: width * 2,
+            height: height * 2,
+        };
+
+        let encoded_stream = encoder.encode(&yuv_buf)?;
+        encoded_stream.write(&mut writer)?;
+    }
+    writer.flush()?;
+    Ok(())
+}

tools/opencv-test/src/encode/mod.rs

@@ -0,0 +1,68 @@
+mod aom;
+#[cfg(feature = "all")]
+mod h264;
+#[cfg(feature = "all")]
+mod rav1e;
+#[cfg(feature = "all")]
+mod x264;
+pub use crate::encode::aom::encode_aom;
+#[cfg(feature = "all")]
+pub use crate::encode::h264::encode_h264;
+#[cfg(feature = "all")]
+pub use crate::encode::rav1e::encode_rav1e;
+#[cfg(feature = "all")]
+pub use crate::encode::x264::encode_x264;
+use crate::utils::yuv::ColorScale;
+
+use clap::Parser;
+
+// transforms the input file to h264
+#[derive(Parser, Debug)]
+pub struct Args {
+    /// an mp4 file generated by opencv
+    pub input: String,
+    /// name of the file to save
+    pub output: String,
+    /// The codec to use
+    pub codec: CodecTypes,
+    /// Optional parameter. defaults to Lossy.
+    /// NotLossy doubles the size of each frame so that converting to YUV420
+    /// doesn't lose any chromiance information.
+    pub encoding_type: Option<EncodingType>,
+    ///specifies the RGB to YUV transformation matrix
+    pub color_scale: Option<ColorScale>,
+    /// use optimized version. currently only works for aom in lossy mode.
+    /// if set, overrides encoding_type
+    pub mode: Option<Mode>,
+}
+
+#[derive(Debug, Clone, clap::ValueEnum)]
+pub enum CodecTypes {
+    #[cfg(feature = "all")]
+    /// OpenH264
+    H264,
+    #[cfg(feature = "all")]
+    /// x264
+    X264,
+    #[cfg(feature = "all")]
+    /// av1 (rav1e)
+    RAV1E,
+    /// av1 (aom)
+    AOM,
+}
+
+#[derive(Debug, Clone, clap::ValueEnum)]
+pub enum EncodingType {
+    /// convert from BGR24 to YUV420
+    Lossy,
+    /// expand BGR24 image before converting to YUV420
+    NotLossy,
+}
+
+#[derive(Debug, Clone, Copy, clap::ValueEnum)]
+pub enum Mode {
+    Normal,
+    /// attempt to use opencv matrix multiplication
+    /// to speed up the conversion from BGR to YUV
+    Faster,
+}