Skip to content

Commit

Permalink
Add support for new ALVR VRCFaceTracking module
Browse files Browse the repository at this point in the history
  • Loading branch information
@zmerp
zmerp committed Jul 28, 2023
1 parent 9e2c59f commit b1be8db
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 51 deletions.
4 changes: 2 additions & 2 deletions alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,12 +277,12 @@ pub fn eye_face_tracking_schema() -> PresetSchemaNode {
content: None,
},
HigherOrderChoiceOption {
display_name: "VRCFaceTracking OSC".into(),
display_name: "VRCFaceTracking".into(),
modifiers: vec![
bool_modifier("session_settings.headset.face_tracking.enabled", true),
string_modifier(
"session_settings.headset.face_tracking.content.sink.variant",
"VrcFaceTrackingOsc",
"VrcFaceTracking",
),
],
content: None,
Expand Down
4 changes: 2 additions & 2 deletions alvr/server/src/connection.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -682,7 +682,7 @@ fn try_connect(mut client_ips: HashMap<IpAddr, String>) -> ConResult {
let tracking_receive_thread = thread::spawn({
let tracking_manager = Arc::clone(&tracking_manager);
move || {
let face_tracking_sink =
let mut face_tracking_sink =
settings
.headset
.face_tracking
Expand Down Expand Up @@ -762,7 +762,7 @@ fn try_connect(mut client_ips: HashMap<IpAddr, String>) -> ConResult {
}
}

if let Some(sink) = &face_tracking_sink {
if let Some(sink) = &mut face_tracking_sink {
let mut face_data = tracking.face_data;
face_data.eye_gazes = local_eye_gazes;

Expand Down
69 changes: 27 additions & 42 deletions alvr/server/src/face_tracking.rs
View file
Original file line number Diff line number Diff line change
@@ -1,29 +1,37 @@
use alvr_common::{glam::EulerRot, prelude::*};
use alvr_packets::FaceData;
use alvr_session::FaceTrackingSinkConfig;
use bytes::{BufMut, BytesMut};
use rosc::{OscMessage, OscPacket, OscType};
use std::{f32::consts::PI, net::UdpSocket};

const RAD_TO_DEG: f32 = 180.0 / PI;

const VRCFT_PORT: u16 = 0xA1F7;

pub struct FaceTrackingSink {
config: FaceTrackingSinkConfig,
socket: UdpSocket,
packet_buffer: BytesMut,
}

impl FaceTrackingSink {
pub fn new(config: FaceTrackingSinkConfig, local_osc_port: u16) -> StrResult<Self> {
let port = match config {
FaceTrackingSinkConfig::VrchatEyeOsc { port } => port,
FaceTrackingSinkConfig::VrcFaceTrackingOsc { port } => port,
FaceTrackingSinkConfig::VrcFaceTracking => VRCFT_PORT,
};

let socket = UdpSocket::bind(format!("127.0.0.1:{local_osc_port}")).map_err(err!())?;
socket
.connect(format!("127.0.0.1:{port}"))
.map_err(err!())?;

Ok(Self { config, socket })
Ok(Self {
config,
socket,
packet_buffer: BytesMut::new(),
})
}

fn send_osc_message(&self, path: &str, args: Vec<OscType>) {
Expand All @@ -38,7 +46,15 @@ impl FaceTrackingSink {
.ok();
}

pub fn send_tracking(&self, face_data: FaceData) {
fn append_packet_vrcft(&mut self, prefix: &[u8; 8], data: &[f32]) {
self.packet_buffer.put(prefix.as_slice());

for val in data {
self.packet_buffer.put_f32_le(*val);
}
}

pub fn send_tracking(&mut self, face_data: FaceData) {
match self.config {
FaceTrackingSinkConfig::VrchatEyeOsc { .. } => {
if let [Some(left), Some(right)] = face_data.eye_gazes {
Expand Down Expand Up @@ -88,53 +104,22 @@ impl FaceTrackingSink {
);
}
}
FaceTrackingSinkConfig::VrcFaceTrackingOsc { .. } => {
if let Some(pose) = face_data.eye_gazes[0] {
self.send_osc_message(
"/tracking/eye/left/Quat",
vec![
OscType::Float(pose.orientation.w),
OscType::Float(pose.orientation.x),
OscType::Float(pose.orientation.y),
OscType::Float(pose.orientation.z),
],
);
} else {
self.send_osc_message("/tracking/eye/left/Active", vec![OscType::Bool(false)]);
}
if let Some(pose) = face_data.eye_gazes[1] {
self.send_osc_message(
"/tracking/eye/right/Quat",
vec![
OscType::Float(pose.orientation.w),
OscType::Float(pose.orientation.x),
OscType::Float(pose.orientation.y),
OscType::Float(pose.orientation.z),
],
);
} else {
self.send_osc_message("/tracking/eye/right/Active", vec![OscType::Bool(false)]);
}
FaceTrackingSinkConfig::VrcFaceTracking { .. } => {
self.packet_buffer.clear();

if let Some(arr) = face_data.fb_face_expression {
self.send_osc_message(
"/tracking/face_fb",
arr.into_iter().map(OscType::Float).collect(),
);
self.append_packet_vrcft(b"FaceFb\0\0", &arr);
}

if let Some(arr) = face_data.htc_eye_expression {
self.send_osc_message(
"/tracking/eye_htc",
arr.into_iter().map(OscType::Float).collect(),
);
self.append_packet_vrcft(b"EyesHtc\0", &arr);
}

if let Some(arr) = face_data.htc_lip_expression {
self.send_osc_message(
"/tracking/lip_htc",
arr.into_iter().map(OscType::Float).collect(),
);
self.append_packet_vrcft(b"LipHtc\0\0", &arr);
}

self.socket.send(&self.packet_buffer).ok();
}
}
}
Expand Down
7 changes: 2 additions & 5 deletions alvr/session/src/settings.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -588,8 +588,8 @@ pub struct FaceTrackingSources {
pub enum FaceTrackingSinkConfig {
#[schema(strings(display_name = "VRChat Eye OSC"))]
VrchatEyeOsc { port: u16 },
#[schema(strings(display_name = "VRCFaceTracking OSC"))]
VrcFaceTrackingOsc { port: u16 },
#[schema(strings(display_name = "VRCFaceTracking"))]
VrcFaceTracking,
}

#[derive(SettingsSchema, Serialize, Deserialize, Clone)]
Expand Down Expand Up @@ -1162,9 +1162,6 @@ pub fn session_settings_default() -> SettingsDefault {
},
sink: FaceTrackingSinkConfigDefault {
VrchatEyeOsc: FaceTrackingSinkConfigVrchatEyeOscDefault { port: 9000 },
VrcFaceTrackingOsc: FaceTrackingSinkConfigVrcFaceTrackingOscDefault {
port: 9620,
},
variant: FaceTrackingSinkConfigDefaultVariant::VrchatEyeOsc,
},
},
Expand Down

0 comments on commit b1be8db

Please sign in to comment.