Video Remixer: Find Break Frame in Scene Splitter

config.yaml

@@ -119,6 +119,8 @@ remixer_settings:
   - "qt"
   - "webm"
   - "wmv"
+  find_break_stride: 128
+  find_break_threshold: 1
   gif_end_delay: 1.0
   gif_factor: 10
   labeled_ffmpeg_video: -vf "drawtext=<LABEL>" -c:v libx264 -crf <CRF>
@@ -144,6 +146,7 @@ remixer_settings:
   raise_on_error: False
   scale_type_up: "lanczos"
   scale_type_down: "area"
+  skip_break_threshold: 3
   source_audio_crf: 28
   thumb_scale: 0.5
   use_tiling_over: 921600

tabs/video_remixer_ui.py

@@ -24,6 +24,8 @@
 from video_remixer_project import VideoRemixerProject
 from video_remixer_reports import VideoRemixerReports
 import cv2
+from webui_utils.image_utils import get_average_lightness
+from typing import Literal
 
 class VideoRemixer(TabBase):
     """Encapsulates UI elements and events for the Video Remixer Feature"""
@@ -610,12 +612,15 @@ def render_tab(self):
                                             label="Secondary Split Position", minimum=0.0,
                                             maximum=100.0, step=0.1, container=False, scale=2,
                                             info="Earliest split is performed first")
+                                    with gr.Row(variant="compact", equal_height=False):
+                                        prev_break_702 = gr.Button(value="< Find Break Frame" + SimpleIcons.SLOW_SYMBOL, size="sm")
+                                        next_break_702 = gr.Button(value="Find Break Frame"  + SimpleIcons.SLOW_SYMBOL + " >", size="sm")
                                     with gr.Row(variant="compact", equal_height=False):
                                         set_view_hint_702 = gr.Textbox(placeholder="View Hint such as {V:200%}",
                                                                     max_lines=1, show_label=False,
-                                                                    min_width=100, container=False)
+                                                                    min_width=100, container=False, scale=2)
                                         preview_view_hint_702 = gr.Button(value="Visualize View Hint",
-                                                                      size="sm", min_width=40)
+                                                                      size="sm", min_width=40, scale=1)
 
                             with gr.Column():
                                 preview_image702 = gr.Image(type="filepath",
@@ -1364,12 +1369,26 @@ def render_tab(self):
                                 inputs=[scene_id_702, split_percent_702, go_to_f_702],
                                 outputs=split_percent_702, show_progress=False)
 
-        split_button702.click(self.split_button702,
-                              inputs=[scene_id_702, split_percent_702, use_alt_split_702,
-                                      split_percent_alt_702],
-                              outputs=[tabs_video_remixer, message_box702, use_alt_split_702,
-                                       split_percent_alt_702, scene_index, scene_name,
-                                       scene_image, scene_state, scene_info, set_scene_label])
+
+        use_alt_split_702.change(self.use_alt_split_change,
+                                inputs=[use_alt_split_702, split_percent_702, split_percent_alt_702],
+                                outputs=[split_percent_702, split_percent_alt_702],
+                                show_progress=False)
+
+        prev_break_702.click(self.prev_break_702,
+                                inputs=[scene_id_702, split_percent_702],
+                                outputs=split_percent_702, show_progress=True)
+        next_break_702.click(self.next_break_702,
+                                inputs=[scene_id_702, split_percent_702],
+                                outputs=split_percent_702, show_progress=True)
+
+        set_view_hint_702.submit(self.set_view_hint_702,
+                                 inputs=[scene_id_702, split_percent_702, set_view_hint_702],
+                                 outputs=[preview_image702, scene_info_702], show_progress=False)
+
+        preview_view_hint_702.click(self.preview_view_hint_702,
+                                    inputs=[scene_id_702, split_percent_702, set_view_hint_702],
+                                    outputs=[preview_image702, scene_info_702], show_progress=False)
 
         split_keep_before_702.click(self.split_keep_before_702,
                                 inputs=[scene_id_702, split_percent_702, use_alt_split_702,
@@ -1385,21 +1404,15 @@ def render_tab(self):
                                             split_percent_alt_702, scene_index, scene_name,
                                             scene_image, scene_state, scene_info, set_scene_label])
 
-        use_alt_split_702.change(self.use_alt_split_change,
-                                inputs=[use_alt_split_702, split_percent_702, split_percent_alt_702],
-                                outputs=[split_percent_702, split_percent_alt_702],
-                                show_progress=False)
+        split_button702.click(self.split_button702,
+                              inputs=[scene_id_702, split_percent_702, use_alt_split_702,
+                                      split_percent_alt_702],
+                              outputs=[tabs_video_remixer, message_box702, use_alt_split_702,
+                                       split_percent_alt_702, scene_index, scene_name,
+                                       scene_image, scene_state, scene_info, set_scene_label])
 
         back_button702.click(self.back_button702, outputs=tabs_video_remixer)
 
-        set_view_hint_702.submit(self.set_view_hint_702,
-                                 inputs=[scene_id_702, split_percent_702, set_view_hint_702],
-                                 outputs=[preview_image702, scene_info_702], show_progress=False)
-
-        preview_view_hint_702.click(self.preview_view_hint_702,
-                                    inputs=[scene_id_702, split_percent_702, set_view_hint_702],
-                                    outputs=[preview_image702, scene_info_702], show_progress=False)
-
         export_project_703.click(self.export_project_703,
                                  inputs=[export_path_703, project_name_703],
                                  outputs=[message_box703, result_box703, open_result703])
@@ -2720,6 +2733,112 @@ def use_alt_split_change(self, use_alt_split, split_percent, split_percent_alt):
         else:
             return split_percent_alt, split_percent_alt
 
+    def find_break_frame_type(self, frame_file) -> Literal["break", "skip", "find"]:
+        find_break_stride = self.config.remixer_settings["find_break_stride"]
+        find_break_threshold = self.config.remixer_settings["find_break_threshold"]
+        skip_break_threshold = self.config.remixer_settings["skip_break_threshold"]
+        l = get_average_lightness(frame_file, find_break_stride)
+
+        break_type : str
+        if l <= find_break_threshold:
+            break_type = "break"
+        elif l <= skip_break_threshold:
+            break_type = "skip"
+        else:
+            break_type = "find"
+
+        return break_type
+
+    def next_break_702(self, scene_index, split_percent):
+        scene_index = int(scene_index)
+        num_scenes = len(self.state.scene_names)
+        last_scene = num_scenes - 1
+        if scene_index < 0 or scene_index > last_scene:
+            return split_percent
+
+        scene_name = self.state.scene_names[scene_index]
+        _, num_frames, _, _, split_frame = self.state.compute_scene_split(scene_name, split_percent)
+
+        last_frame = num_frames - 1
+        last_starting_search_frame = last_frame - 1
+        if split_frame > last_starting_search_frame:
+            return split_percent
+        starting_search_frame = split_frame + 1
+        search_frame_index : int = starting_search_frame
+
+        frame_files = self.state.get_split_scene_cache(scene_index)
+
+        frame_file = frame_files[search_frame_index]
+        frame_type = self.find_break_frame_type(frame_file)
+        if frame_type == "skip":
+            # skip frames until either a break frame or a find frame
+            for search_frame_index in range(search_frame_index + 1, last_frame + 1):
+                frame_file = frame_files[search_frame_index]
+                frame_type = self.find_break_frame_type(frame_file)
+                if frame_type == "break" or frame_type == "find":
+                    break
+
+        if search_frame_index > last_starting_search_frame:
+            return split_percent
+
+        if frame_type != "break":
+            for search_frame_index in range(search_frame_index + 1, last_frame + 1):
+                frame_file = frame_files[search_frame_index]
+                frame_type = self.find_break_frame_type(frame_file)
+                if frame_type == "break":
+                    break
+
+        if frame_type == "break": # and search_frame_index != starting_search_frame:
+            new_split_percent = 100.0 * (search_frame_index * 1.0 / num_frames)
+            return new_split_percent
+
+        return split_percent
+
+    def prev_break_702(self, scene_index, split_percent):
+        scene_index = int(scene_index)
+        num_scenes = len(self.state.scene_names)
+        last_scene = num_scenes - 1
+        if scene_index < 0 or scene_index > last_scene:
+            return split_percent
+
+        scene_name = self.state.scene_names[scene_index]
+        _, num_frames, _, _, split_frame = self.state.compute_scene_split(scene_name, split_percent)
+
+        last_frame = 0
+        last_starting_search_frame = last_frame + 1
+        if split_frame - 1 < last_starting_search_frame:
+            return split_percent
+        starting_search_frame = split_frame - 2 # split frame is after the split, now going in rev.
+        search_frame_index : int = starting_search_frame
+
+        frame_files = self.state.get_split_scene_cache(scene_index)
+
+        frame_file = frame_files[starting_search_frame]
+        frame_type = self.find_break_frame_type(frame_file)
+        if frame_type == "skip":
+            # skip frames until either a break frame or a find frame
+            for search_frame_index in range(search_frame_index - 1, last_frame - 1, -1):
+                frame_file = frame_files[search_frame_index]
+                frame_type = self.find_break_frame_type(frame_file)
+                if frame_type == "break" or frame_type == "find":
+                    break
+
+        if search_frame_index < last_starting_search_frame:
+            return split_percent
+
+        if frame_type != "break":
+            for search_frame_index in range(search_frame_index - 1, last_frame - 1, -1):
+                frame_file = frame_files[search_frame_index]
+                frame_type = self.find_break_frame_type(frame_file)
+                if frame_type == "break":
+                    break
+
+        if frame_type == "break":
+            new_split_percent = 100.0 * (search_frame_index * 1.0 / num_frames)
+            return new_split_percent
+
+        return split_percent
+
     def back_button702(self):
         return gr.update(selected=self.TAB_CHOOSE_SCENES)
 

video_remixer.py

@@ -609,14 +609,16 @@ def compute_preview_frame(self, scene_index, split_percent):
 
         scene_name = self.scene_names[scene_index]
         _, num_frames, _, _, split_frame = self.compute_scene_split(scene_name, split_percent)
-        original_scene_path = os.path.join(self.scenes_path, scene_name)
-        frame_files = self.valid_split_scene_cache(scene_index)
-        if not frame_files:
-            # optimize to uncompile only the first time it's needed
-            self.uncompile_scenes()
 
-            frame_files = sorted(get_files(original_scene_path))
-            self.fill_split_scene_cache(scene_index, frame_files)
+        # original_scene_path = os.path.join(self.scenes_path, scene_name)
+        # frame_files = self.valid_split_scene_cache(scene_index)
+        # if not frame_files:
+        #     # optimize to uncompile only the first time it's needed
+        #     self.uncompile_scenes()
+
+        #     frame_files = sorted(get_files(original_scene_path))
+        #     self.fill_split_scene_cache(scene_index, frame_files)
+        frame_files = self.get_split_scene_cache(scene_index)
 
         num_frame_files = len(frame_files)
         if num_frame_files != num_frames:
@@ -878,3 +880,15 @@ def fill_split_scene_cache(self, scene_index, data):
     def invalidate_split_scene_cache(self):
         self.split_scene_cache = []
         self.split_scene_cached_index = -1
+
+    def get_split_scene_cache(self, scene_index):
+        entries = self.valid_split_scene_cache(scene_index)
+        if not entries:
+            # optimize to uncompile only the first time it's needed
+            self.uncompile_scenes()
+
+            scene_name = self.scene_names[scene_index]
+            original_scene_path = os.path.join(self.scenes_path, scene_name)
+            entries = sorted(get_files(original_scene_path))
+            self.fill_split_scene_cache(scene_index, entries)
+        return entries

webui_utils/image_utils.py

@@ -43,3 +43,19 @@ def gif_frame_count(filepath : str):
     gif = Image.open(filepath)
     if gif:
         return gif.n_frames
+
+def get_average_lightness(image_path : str, stride : int = 1) -> int:
+    with Image.open(image_path) as img:
+        img = img.convert('L')
+        pixels = img.getdata()
+        total = 0
+        pixel_count = 0
+        pixels = list(pixels)
+        for pixel in pixels[::stride]:
+            # assume the sampled pixel is an average representative of the stride range
+            total += pixel * stride
+            pixel_count += 1
+
+        average = int(total / (pixel_count * stride))
+        return average
+