diff --git a/code/cameras/exploration_scripts/pupil_synch.py b/code/cameras/exploration_scripts/pupil_synch.py index e280cbb..dcd4807 100644 --- a/code/cameras/exploration_scripts/pupil_synch.py +++ b/code/cameras/exploration_scripts/pupil_synch.py @@ -14,6 +14,7 @@ from pathlib import Path from typing import Dict, List, Tuple +import cv2 import numpy as np @@ -26,6 +27,7 @@ def __init__(self, folder_path: Path): self.raw_videos_path = folder_path / "raw_videos" self.synched_videos_path = folder_path / "synchronized_videos" + self.output_path = folder_path / "basler_pupil_synchronized" self.basler_timestamp_mapping_file_name = "timestamp_mapping.json" basler_timestamp_mapping_file = ( @@ -230,33 +232,6 @@ def get_utc_timestamp_per_camera(self) -> Dict[int, int]: ]["camera_timestamps"].items() } - # def find_pupil_starting_offsets_in_frames(self) -> Dict[str, int]: - # # find pupil frame number where timestamp is >= the first basler frame - # starting_offsets_in_frames = { - # "eye0": np.where( - # self.pupil_eye0_timestamps_utc >= self.basler_first_synched_timestamp - # )[0][0], - # "eye1": np.where( - # self.pupil_eye1_timestamps_utc >= self.basler_first_synched_timestamp - # )[0][0], - # } - # print(starting_offsets_in_frames) - # return starting_offsets_in_frames - - # def find_pupil_ending_offsets_in_frames( - # self, pupil_starting_offsets: Dict[str, int] - # ) -> Dict[str, int]: - # ending_offsets_in_frames = { - # "eye0": np.where( - # self.pupil_eye0_timestamps_utc >= self.basler_last_synched_timestamp - # )[0][0], - # "eye1": np.where( - # self.pupil_eye1_timestamps_utc >= self.basler_last_synched_timestamp - # )[0][0], - # } - # print(ending_offsets_in_frames) - # return ending_offsets_in_frames - def find_starting_offsets_in_frames(self) -> Dict[str, int]: starting_offsets_in_frames = { cam_name: np.where( @@ -292,8 +267,80 @@ def find_ending_offsets_in_frames(self) -> Dict[str, int]: print(f"ending offsets in frames: {ending_offsets_in_frames}") return ending_offsets_in_frames + + def save_corrected_timestamps(self): + if self.corrected_timestamps is None: + raise ValueError("corrected_timestamps is None, this method should only be called from synchronize(), it should not be called directly") + for cam_name, timestamps in self.corrected_timestamps.items(): + print(f"cam {cam_name} timestamps shape: {timestamps.shape}") + np.save(f"{self.output_path}/cam_{cam_name}_corrected_timestamps.npy", timestamps) + + def trim_single_video(self, + start_frame: int, + end_frame: int, + input_video_pathstring: str, + output_video_pathstring: str, + ): + frame_list = list(range(start_frame, end_frame+1)) + cap = cv2.VideoCapture(input_video_pathstring) + + framerate = cap.get(cv2.CAP_PROP_FPS) + framesize = ( + int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), + int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)), + ) + fourcc = cv2.VideoWriter.fourcc(*"mp4v") + + video_writer_object = cv2.VideoWriter( + output_video_pathstring, fourcc, framerate, framesize + ) + + current_frame = 0 + written_frames = 0 + + while True: + ret, frame = cap.read() + if not ret: + break + + if current_frame in frame_list: + video_writer_object.write(frame) + written_frames += 1 + + if written_frames == len(frame_list): + break + + current_frame += 1 + + cap.release() + video_writer_object.release() + + def trim_videos(self, starting_offsets_frames: Dict[str, int], ending_offsets_frames: Dict[str, int]): + for cam_name in self.basler_camera_names: + # TODO: get input video path (self.synched_videos_path/SERIALNUMBER.mp4) + self.trim_single_video( + starting_offsets_frames[cam_name], + ending_offsets_frames[cam_name], + str(inputpath), + str(self.output_path / "basler_cam{cam_name}_SERIALNUMBER.mp4"), + ) + + self.trim_single_video( + starting_offsets_frames["eye0"], + ending_offsets_frames["eye0"], + str(self.pupil_eye0_video_path), + str(self.output_path / "eye0.mp4"), + ) + + self.trim_single_video( + starting_offsets_frames["eye1"], + ending_offsets_frames["eye1"], + str(self.pupil_eye1_video_path), + str(self.output_path / "eye1.mp4"), + ) def synchronize(self): + self.output_path.mkdir(parents=True, exist_ok=True) print(f"latest synched start utc: {self.latest_synched_start_utc}") print(f"earliest synched end utc: {self.earliest_synched_end_utc}") starting_offsets_frames = self.find_starting_offsets_in_frames() @@ -312,70 +359,16 @@ def synchronize(self): starting_offsets_frames["eye1"] : ending_offsets_frames["eye1"] ] - for cam_name, timestamps in self.corrected_timestamps.items(): - print(f"cam {cam_name} timestamps shape: {timestamps.shape}") - # TODO: getting different shapes for the corrected pupil timestamps?? + self.save_corrected_timestamps() + + # TODO: trim videos + + # TODO: save everything to a "pupil_basler_synched" folder self.plot_timestamps( starting_offsets=starting_offsets_frames, ending_offsets=ending_offsets_frames ) - - # if ( - # self.basler_first_synched_timestamp_utc - # > self.pupil_first_synched_timestamp_utc - # ): - # # pupil data starts before basler data - # pass - # else: - # # basler data starts before pupil data - # pass - - # if ( - # self.basler_last_synched_timestamp_utc - # < self.pupil_last_synched_timestamp_utc - # ): - # # pupil data ends after basler data - # pass - # else: - # # basler data ends after pupil data - # pass - - def old_synchronize(self): - pupil_starting_offsets = self.find_pupil_starting_offsets_in_frames() - pupil_ending_offsets = self.find_pupil_ending_offsets_in_frames( - pupil_starting_offsets=pupil_starting_offsets - ) - - corrected_pupil_timestamps = { - "eye0": self.pupil_eye0_timestamps_utc[ - pupil_starting_offsets["eye0"] : pupil_ending_offsets["eye0"] - ], - "eye1": self.pupil_eye1_timestamps_utc[ - pupil_starting_offsets["eye1"] : pupil_ending_offsets["eye1"] - ], - } - - print( - f"corrected timestamp shapes - eye0: {corrected_pupil_timestamps['eye0'].shape} eye1: {corrected_pupil_timestamps['eye1'].shape}" - ) - print( - f"starting timestamps - eye0: {self.pupil_eye0_timestamps_utc[pupil_starting_offsets['eye0']]} eye1: {self.pupil_eye0_timestamps_utc[pupil_starting_offsets['eye1']]}" - ) - print( - f"ending timestamps - eye0: {self.pupil_eye0_timestamps_utc[pupil_ending_offsets['eye0']]} eye1: {self.pupil_eye0_timestamps_utc[pupil_ending_offsets['eye1']]}" - ) - print( - f"starting timestamp difference: {self.pupil_eye0_timestamps_utc[pupil_starting_offsets['eye0']] - self.pupil_eye0_timestamps_utc[pupil_starting_offsets['eye1']]}" - ) - print( - f"ending timestamp difference: {self.pupil_eye0_timestamps_utc[pupil_ending_offsets['eye0']] - self.pupil_eye0_timestamps_utc[pupil_ending_offsets['eye1']]}" - ) - - # self.plot_timestamps( - # starting_offsets=pupil_starting_offsets, - # ending_offsets=pupil_ending_offsets, - # ) def plot_timestamps( self, @@ -383,7 +376,7 @@ def plot_timestamps( ending_offsets: Dict[str, int], ): """plot some diagnostics to assess quality of camera sync""" - + # TODO: swap time and frame number, so x axis shows synching # opportunistic load of matplotlib to avoid startup time costs from matplotlib import pyplot as plt @@ -401,7 +394,7 @@ def plot_timestamps( for i, cam_name in enumerate(self.basler_camera_names): ax1.plot( self.synched_basler_timestamps_utc[i, :], - label=cam_name, + label=f"basler {cam_name}", ) ax1.plot(self.pupil_eye0_timestamps_utc, label="eye0") ax1.plot(self.pupil_eye1_timestamps_utc, label="eye1") diff --git a/code/cameras/multicamera_recording.py b/code/cameras/multicamera_recording.py index 0b4a209..2d66c43 100644 --- a/code/cameras/multicamera_recording.py +++ b/code/cameras/multicamera_recording.py @@ -93,10 +93,16 @@ def open_camera_array(self): if not self.camera_array.IsOpen(): self.camera_array.Open() + index_to_serial_number_mapping = {} + for index, camera in enumerate(self.camera_array): camera_serial = camera.DeviceInfo.GetSerialNumber() logger.info(f"set context {index} for camera {camera_serial}") camera.SetCameraContext(index) # this gives us an easy to enumerate camera id, but we may prefer using serial number + dictionaries + index_to_serial_number_mapping[index] = camera_serial + + with open(self.output_path / "index_to_serial_number_mapping.json", mode="x") as f: + json.dump(index_to_serial_number_mapping, f, indent=4) def close_camera_array(self): self.camera_array.Close()