From ad131abd1d271a7f9373e41e8fa129d64ecf118b Mon Sep 17 00:00:00 2001 From: Jerry Hogsett Date: Fri, 25 Oct 2024 07:38:27 -0700 Subject: [PATCH 1/4] wip --- split_scenes.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/split_scenes.py b/split_scenes.py index 574b022d..830b35d5 100644 --- a/split_scenes.py +++ b/split_scenes.py @@ -69,7 +69,7 @@ def __init__(self, if not self.type in valid_types: raise ValueError(f"'type' must be one of {', '.join([t for t in valid_types])}") - def split_scenes(self, format : str="png"): + def split_scenes(self, format : str="png", move : bool=False): files = sorted(glob.glob(os.path.join(self.input_path, f"*.{self.file_ext}"))) num_files = len(files) num_width = len(str(num_files)) @@ -162,22 +162,18 @@ def split_breaks(self, format : str="jpg"): return group_paths - def split(self, type : str="png") -> list: + def split(self, type : str="png", move : bool=False) -> list: """Invoke the Split Scenes feature""" - # files = sorted(glob.glob(os.path.join(self.input_path, f"*.{self.file_ext}"))) - # num_files = len(files) - # num_width = len(str(num_files)) - if self.type == "scene": if self.scene_threshold < 0.0 or self.scene_threshold > 1.0: raise ValueError("'scene_threshold' must be between 0.0 and 1.0") - self.split_scenes(type) + self.split_scenes(type, move) else: if self.break_duration < 0.0: raise ValueError("'break_duration' >= 0.0") if self.break_ratio < 0.0 or self.break_ratio > 1.0: raise ValueError("'break_ratio' must be between 0.0 and 1.0") - self.split_breaks(type) + self.split_breaks(type, move) if self.dry_run: print(f"[Dry Run] Creating base output path {self.output_path}") From dab0ca3484e08f8bfbb1a76c98fde71702f85d1a Mon Sep 17 00:00:00 2001 From: Jerry Hogsett Date: Sat, 26 Oct 2024 08:41:21 -0700 Subject: [PATCH 2/4] fix issues with group resequence --- resequence_files.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/resequence_files.py b/resequence_files.py index 69ea99e0..ae1c6711 100644 --- a/resequence_files.py +++ b/resequence_files.py @@ -8,7 +8,7 @@ from webui_utils.simple_log import SimpleLog from webui_utils.simple_utils import create_sample_set from webui_utils.mtqdm import Mtqdm -from webui_utils.file_utils import get_directories, check_for_name_clash, get_files +from webui_utils.file_utils import get_directories, check_for_name_clash, get_files, create_directory def main(): """Use the Resequence Files feature from the command line""" @@ -94,13 +94,22 @@ def __init__(self, def resequence_groups(self, group_names : list, contiguous=True, ignore_name_clash=True): """Resequence files contained in the specified directory names at the input path. Returns a string with any errors.""" + + # count the original files across groups all_files_count = 0 for group_name in group_names: - check_path = self.input_path if self.rename else self.output_path - group_check_path = os.path.join(check_path, group_name) + group_check_path = os.path.join(self.input_path, group_name) try: group_files = glob.glob(os.path.join(group_check_path, "*." + self.file_type)) all_files_count += len(group_files) + except ValueError as error: + return str(error) + + # check for name clashes in the output paths + for group_name in group_names: + group_check_path = os.path.join(self.output_path, group_name) + try: + group_files = glob.glob(os.path.join(group_check_path, "*." + self.file_type)) if not ignore_name_clash: check_for_name_clash(group_files, self.file_type, self.new_base_filename) except ValueError as error: @@ -109,6 +118,7 @@ def resequence_groups(self, group_names : list, contiguous=True, ignore_name_cla if self.zero_fill == ResequenceFiles.ZERO_FILL_AUTO_DETECT: max_index_num = all_files_count * self.index_step batch_zero_fill = len(str(max_index_num)) + print(batch_zero_fill, self.index_step) else: batch_zero_fill = self.zero_fill @@ -119,6 +129,7 @@ def resequence_groups(self, group_names : list, contiguous=True, ignore_name_cla for group_name in group_names: group_input_path = os.path.join(self.input_path, group_name) group_output_path = os.path.join(self.output_path, group_name) + create_directory(group_output_path) try: if contiguous: group_start = running_start @@ -140,7 +151,7 @@ def resequence_groups(self, group_names : list, contiguous=True, ignore_name_cla self.log_fn, group_output_path, self.reverse).resequence(ignore_name_clash=ignore_name_clash, - skip_if_not_required=False) + skip_if_not_required=not contiguous) except ValueError as error: errors.append(f"Error handling directory {group_name}: " + str(error)) Mtqdm().update_bar(bar) From 674e8dc15b2d0f0198f0f6838226979ff3848541 Mon Sep 17 00:00:00 2001 From: Jerry Hogsett Date: Sat, 26 Oct 2024 11:16:02 -0700 Subject: [PATCH 3/4] use resequencer to split source frames --- resequence_files.py | 27 ++++++++++++------ split_frames.py | 43 +++++++--------------------- split_scenes.py | 61 ++++++++++++++++++++-------------------- tabs/video_remixer_ui.py | 2 +- video_remixer_ingest.py | 12 ++++---- 5 files changed, 66 insertions(+), 79 deletions(-) diff --git a/resequence_files.py b/resequence_files.py index ae1c6711..5afac6a5 100644 --- a/resequence_files.py +++ b/resequence_files.py @@ -92,7 +92,7 @@ def __init__(self, ZERO_FILL_AUTO_DETECT = -1 - def resequence_groups(self, group_names : list, contiguous=True, ignore_name_clash=True): + def resequence_groups(self, group_names : list, contiguous=True, ignore_name_clash=True, move_files=False): """Resequence files contained in the specified directory names at the input path. Returns a string with any errors.""" # count the original files across groups @@ -151,24 +151,26 @@ def resequence_groups(self, group_names : list, contiguous=True, ignore_name_cla self.log_fn, group_output_path, self.reverse).resequence(ignore_name_clash=ignore_name_clash, - skip_if_not_required=not contiguous) + skip_if_not_required=not contiguous, + move_files=move_files) except ValueError as error: errors.append(f"Error handling directory {group_name}: " + str(error)) Mtqdm().update_bar(bar) if errors: return "\r\n".join(errors) - def resequence_batch(self, contiguous=True, ignore_name_clash=True): + def resequence_batch(self, contiguous=True, ignore_name_clash=True, move_files=False): """Resequence groups of files. Returns a string with any errors.""" group_names = sorted(get_directories(self.input_path), reverse=self.reverse) return self.resequence_groups(group_names, contiguous=contiguous, - ignore_name_clash=ignore_name_clash) + ignore_name_clash=ignore_name_clash, + move_files=False) - def resequence(self, ignore_name_clash=True, skip_if_not_required=True) -> None: + def resequence(self, ignore_name_clash=True, skip_if_not_required=True, move_files=False, file_list=None) -> None: """Resesequence files in the directory per settings. Returns a count of the files resequenced. Raises ValueError on name clash.""" - files = sorted(glob.glob(os.path.join(self.input_path, "*." + self.file_type)), - reverse=self.reverse) + files = file_list or \ + sorted(glob.glob(os.path.join(self.input_path, "*." + self.file_type)), reverse=self.reverse) num_files = len(files) # if renaming files in place, check to see that they are not already in proper sequence @@ -191,7 +193,11 @@ def resequence(self, ignore_name_clash=True, skip_if_not_required=True) -> None: running_index = self.start_index sample_set = create_sample_set(files, self.sample_offset, self.sample_stride) - pbar_title = "Resequence Rename" if self.rename else "Resequence Copy" + + if self.rename: + pbar_title = "Resequence Rename" + else: + pbar_title = "Resequence Move" if move_files else "Resequence Copy" with Mtqdm().open_bar(total=len(sample_set), desc=pbar_title) as bar: for file in sample_set: new_filename = \ @@ -204,7 +210,10 @@ def resequence(self, ignore_name_clash=True, skip_if_not_required=True) -> None: os.replace(old_filepath, new_filepath) else: new_filepath = os.path.join(self.output_path, new_filename) - shutil.copy(old_filepath, new_filepath) + if move_files: + shutil.move(old_filepath, new_filepath) + else: + shutil.copy(old_filepath, new_filepath) running_index += self.index_step Mtqdm().update_bar(bar) diff --git a/split_frames.py b/split_frames.py index 27bf0049..b6185c96 100644 --- a/split_frames.py +++ b/split_frames.py @@ -184,41 +184,18 @@ def split(self) -> list: else: create_directory(group_path) - desc = "Copying" if self.action == "copy" else "Moving" - with Mtqdm().open_bar(total=num_group_files, desc=desc) as file_bar: - for file in group_files: - from_filepath = file - _, filename, ext = split_filepath(file) - to_filepath = os.path.join(group_path, filename + ext) - if self.dry_run: - print(f"[Dry Run] Copying {from_filepath} to {to_filepath}") - else: - shutil.copy(from_filepath, to_filepath) - Mtqdm().update_bar(file_bar) + base_filename = f"{group_name}-{self.type}-split-frame" + ResequenceFiles(self.input_path, + self.file_ext, + base_filename, + 0, 1, 1, 0, + num_width, + False, + self.log, + output_path=group_path).resequence(move_files=self.action == "move", + file_list=group_files) Mtqdm().update_bar(group_bar) - if add_resynthesis_frames or add_inflation_frame: - if self.dry_run: - print(f"[Dry Run] Resequencing files in {group_path}") - else: - base_filename = f"{group_name}-{self.type}-split-frame" - ResequenceFiles(group_path, - self.file_ext, - base_filename, - 0, 1, 1, 0, num_width, - True, - self.log).resequence() - - if self.action != "copy": - with Mtqdm().open_bar(total=num_files, desc="Deleting") as bar: - for file in files: - if os.path.exists(file): - if self.dry_run: - print(f"[Dry Run] Deleting {file}") - else: - os.remove(file) - Mtqdm().update_bar(bar) - return group_paths def log(self, message : str) -> None: diff --git a/split_scenes.py b/split_scenes.py index 830b35d5..10c3734a 100644 --- a/split_scenes.py +++ b/split_scenes.py @@ -8,6 +8,7 @@ from webui_utils.file_utils import create_directory, is_safe_path, split_filepath from webui_utils.video_utils import get_detected_scenes, get_detected_breaks, scene_list_to_ranges from webui_utils.mtqdm import Mtqdm +from resequence_files import ResequenceFiles def main(): """Use the Split Scenes feature from the command line""" @@ -69,7 +70,7 @@ def __init__(self, if not self.type in valid_types: raise ValueError(f"'type' must be one of {', '.join([t for t in valid_types])}") - def split_scenes(self, format : str="png", move : bool=False): + def split_scenes(self, format : str="png", move_files : bool=False): files = sorted(glob.glob(os.path.join(self.input_path, f"*.{self.file_ext}"))) num_files = len(files) num_width = len(str(num_files)) @@ -98,24 +99,23 @@ def split_scenes(self, format : str="png", move : bool=False): else: create_directory(group_path) - desc = "Copying" - with Mtqdm().open_bar(total=group_size, desc=desc) as file_bar: - for index in range(first_index, last_index+1): - frame_file = files[index] - from_filepath = frame_file - _, filename, ext = split_filepath(frame_file) - to_filepath = os.path.join(group_path, filename + ext) - - if self.dry_run: - print(f"[Dry Run] Copying {from_filepath} to {to_filepath}") - else: - shutil.copy(from_filepath, to_filepath) - Mtqdm().update_bar(file_bar) + group_files = [files[index] for index in range(first_index, last_index+1)] + base_filename = f"{group_name}-{self.type}-split-frame" + ResequenceFiles(self.input_path, + self.file_ext, + base_filename, + 0, 1, 1, 0, + num_width, + False, + self.log, + output_path=group_path).resequence(move_files=move_files, + file_list=group_files) + Mtqdm().update_bar(scene_bar) return group_paths - def split_breaks(self, format : str="jpg"): + def split_breaks(self, format : str="jpg", move_files : bool=False): files = sorted(glob.glob(os.path.join(self.input_path, f"*.{self.file_ext}"))) num_files = len(files) num_width = len(str(num_files)) @@ -145,35 +145,34 @@ def split_breaks(self, format : str="jpg"): else: create_directory(group_path) - desc = "Copying" - with Mtqdm().open_bar(total=group_size, desc=desc) as file_bar: - for index in range(first_index, last_index+1): - frame_file = files[index] - from_filepath = frame_file - _, filename, ext = split_filepath(frame_file) - to_filepath = os.path.join(group_path, filename + ext) - - if self.dry_run: - print(f"[Dry Run] Copying {from_filepath} to {to_filepath}") - else: - shutil.copy(from_filepath, to_filepath) - Mtqdm().update_bar(file_bar) + group_files = [files[index] for index in range(first_index, last_index+1)] + base_filename = f"{group_name}-{self.type}-split-frame" + ResequenceFiles(self.input_path, + self.file_ext, + base_filename, + 0, 1, 1, 0, + num_width, + False, + self.log, + output_path=group_path).resequence(move_files=move_files, + file_list=group_files) + Mtqdm().update_bar(scene_bar) return group_paths - def split(self, type : str="png", move : bool=False) -> list: + def split(self, type : str="png", move_files : bool=False) -> list: """Invoke the Split Scenes feature""" if self.type == "scene": if self.scene_threshold < 0.0 or self.scene_threshold > 1.0: raise ValueError("'scene_threshold' must be between 0.0 and 1.0") - self.split_scenes(type, move) + self.split_scenes(type, move_files) else: if self.break_duration < 0.0: raise ValueError("'break_duration' >= 0.0") if self.break_ratio < 0.0 or self.break_ratio > 1.0: raise ValueError("'break_ratio' must be between 0.0 and 1.0") - self.split_breaks(type, move) + self.split_breaks(type, move_files) if self.dry_run: print(f"[Dry Run] Creating base output path {self.output_path}") diff --git a/tabs/video_remixer_ui.py b/tabs/video_remixer_ui.py index d3c2de75..d46afd27 100644 --- a/tabs/video_remixer_ui.py +++ b/tabs/video_remixer_ui.py @@ -2010,7 +2010,7 @@ def _next_button2(self, thumbnail_type, min_frames_per_scene, skip_detection, re self.state.save() # split frames into scenes - error = self.state.ingest.split_scenes(prevent_overwrite=False) + error = self.state.ingest.split_scenes(prevent_overwrite=False, move_files=remove_source) if error: raise ValueError(f"There was an error splitting the source video: {error}") diff --git a/video_remixer_ingest.py b/video_remixer_ingest.py index 1796409c..64232590 100644 --- a/video_remixer_ingest.py +++ b/video_remixer_ingest.py @@ -279,7 +279,7 @@ def create_source_audio(self, crf, prevent_overwrite=True, skip_mp4=True): global_options=self.state.global_options) Mtqdm().update_bar(bar) - def split_scenes(self, prevent_overwrite=False): + def split_scenes(self, prevent_overwrite=False, move_files=False): if prevent_overwrite and self.scenes_present(): return None try: @@ -293,7 +293,8 @@ def split_scenes(self, prevent_overwrite=False): self.state.scene_threshold, 0.0, 0.0, - self.state.log_fn).split(type=self.state.frame_format) + self.state.log_fn).split(type=self.state.frame_format, + move_files=move_files) Mtqdm().update_bar(bar) elif self.state.split_type == "Break": @@ -306,7 +307,8 @@ def split_scenes(self, prevent_overwrite=False): 0.0, float(self.state.break_duration), float(self.state.break_ratio), - self.state.log_fn).split(type=self.state.frame_format) + self.state.log_fn).split(type=self.state.frame_format, + move_files=move_files) Mtqdm().update_bar(bar) elif self.state.split_type == "Time": # split by seconds @@ -317,7 +319,7 @@ def split_scenes(self, prevent_overwrite=False): "precise", 0, self.state.split_frames, - "copy", + "move" if move_files else "copy", False, self.state.log_fn).split() else: @@ -329,7 +331,7 @@ def split_scenes(self, prevent_overwrite=False): "precise", 1, 0, - "copy", + "move" if move_files else "copy", False, self.state.log_fn).split() return None From 74d64aa3e6f808350f79b92703cd188d6659573b Mon Sep 17 00:00:00 2001 From: Jerry Hogsett Date: Sat, 26 Oct 2024 15:53:42 -0700 Subject: [PATCH 4/4] leftover --- resequence_files.py | 1 - 1 file changed, 1 deletion(-) diff --git a/resequence_files.py b/resequence_files.py index 5afac6a5..37ab08d2 100644 --- a/resequence_files.py +++ b/resequence_files.py @@ -118,7 +118,6 @@ def resequence_groups(self, group_names : list, contiguous=True, ignore_name_cla if self.zero_fill == ResequenceFiles.ZERO_FILL_AUTO_DETECT: max_index_num = all_files_count * self.index_step batch_zero_fill = len(str(max_index_num)) - print(batch_zero_fill, self.index_step) else: batch_zero_fill = self.zero_fill