From 71a72a994ca95eae1366769873185d20624eaeee Mon Sep 17 00:00:00 2001 From: sashacmc Date: Wed, 1 May 2024 19:14:38 +0200 Subject: [PATCH] Move progressbar start to utils. Fix pylint notes. --- .github/workflows/pylint.yml | 2 +- src/mmdiary/notion/uploader.py | 19 +++------------ .../telegrambot/telegrambot_service.py | 3 ++- src/mmdiary/transcriber/transcriber.py | 18 ++------------ src/mmdiary/utils/medialib.py | 6 ++--- src/mmdiary/utils/progressbar.py | 17 +++++++++++++ src/mmdiary/video/processor.py | 23 ++++-------------- src/mmdiary/video/uploader.py | 24 ++++--------------- 8 files changed, 36 insertions(+), 76 deletions(-) create mode 100644 src/mmdiary/utils/progressbar.py diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index d70ed30..bd81679 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -21,4 +21,4 @@ jobs: pip install . - name: Analysing the code with pylint run: | - pylint $(git ls-files '*.py') --disable missing-function-docstring,missing-module-docstring,missing-class-docstring,broad-exception-caught + pylint $(git ls-files '*.py') --disable missing-function-docstring,missing-module-docstring,missing-class-docstring,broad-exception-caught,duplicate-code diff --git a/src/mmdiary/notion/uploader.py b/src/mmdiary/notion/uploader.py index 1e5a21a..3c5ba6d 100755 --- a/src/mmdiary/notion/uploader.py +++ b/src/mmdiary/notion/uploader.py @@ -1,17 +1,16 @@ -# pylint: disable=too-many-arguments #!/usr/bin/python3 +# pylint: disable=too-many-arguments import argparse import logging import os -import progressbar from notion.block import AudioBlock, CalloutBlock, TextBlock from notion.client import NotionClient from notion.collection import CollectionRowBlock from notion_client import Client -from mmdiary.utils import log, medialib +from mmdiary.utils import log, medialib, progressbar from mmdiary.notion import cachedb @@ -201,19 +200,7 @@ def process_list(self, fileslist): fileslist = list(filter(self.filter_existing, fileslist)) self.__status["total"] = len(fileslist) - pbar = progressbar.ProgressBar( - maxval=len(fileslist), - widgets=[ - "Uploading: ", - progressbar.SimpleProgress(), - " (", - progressbar.Percentage(), - ") ", - progressbar.Bar(), - " ", - progressbar.ETA(), - ], - ).start() + pbar = progressbar.start("Uploading", len(fileslist)) for af in fileslist: try: diff --git a/src/mmdiary/telegrambot/telegrambot_service.py b/src/mmdiary/telegrambot/telegrambot_service.py index 0768afd..dad71ef 100755 --- a/src/mmdiary/telegrambot/telegrambot_service.py +++ b/src/mmdiary/telegrambot/telegrambot_service.py @@ -19,9 +19,10 @@ from mmdiary.utils import log, medialib -g_audiofiles = medialib.MediaLib().get_processed() MAX_MESSAGE_SIZE = 1024 +g_audiofiles = medialib.MediaLib(os.getenv("AUDIO_NOTES_ROOT")).get_processed() + class DateSelector: EXIT = 0 diff --git a/src/mmdiary/transcriber/transcriber.py b/src/mmdiary/transcriber/transcriber.py index 6ae26f8..ea3e8ec 100755 --- a/src/mmdiary/transcriber/transcriber.py +++ b/src/mmdiary/transcriber/transcriber.py @@ -5,12 +5,10 @@ import os from datetime import datetime -import progressbar import whisper from photo_importer import fileprop -from mmdiary.utils import log -from mmdiary.utils import medialib +from mmdiary.utils import log, medialib, progressbar from mmdiary.utils.medialib import TIME_OUT_FORMAT from mmdiary.transcriber.verifier import check_text @@ -106,19 +104,7 @@ def main(): tr = Transcriber("medium") print("done") - pbar = progressbar.ProgressBar( - maxval=len(fileslist), - widgets=[ - "Transcribe: ", - progressbar.SimpleProgress(), - " (", - progressbar.Percentage(), - ") ", - progressbar.Bar(), - " ", - progressbar.ETA(), - ], - ).start() + pbar = progressbar.start("Transcribe", len(fileslist)) for af in fileslist: try: diff --git a/src/mmdiary/utils/medialib.py b/src/mmdiary/utils/medialib.py index 2b55bd0..d259a38 100755 --- a/src/mmdiary/utils/medialib.py +++ b/src/mmdiary/utils/medialib.py @@ -62,9 +62,7 @@ def __str__(self): class MediaLib: - def __init__(self, root=None): - if root is None: - root = os.getenv("AUDIO_NOTES_ROOT") # TODO: made proper init + def __init__(self, root): self.__root = root self.__supported_exts = [] @@ -126,7 +124,7 @@ def get_date_from_timestring(time): def main(): - lib = MediaLib() + lib = MediaLib(os.getenv("AUDIO_NOTES_ROOT")) for f in lib.get_new(): print(f) diff --git a/src/mmdiary/utils/progressbar.py b/src/mmdiary/utils/progressbar.py new file mode 100644 index 0000000..e33f496 --- /dev/null +++ b/src/mmdiary/utils/progressbar.py @@ -0,0 +1,17 @@ +import progressbar + + +def start(text, maxval): + return progressbar.ProgressBar( + maxval=maxmal, + widgets=[ + f"{text}: ", + progressbar.SimpleProgress(), + " (", + progressbar.Percentage(), + ") ", + progressbar.Bar(), + ' ', + progressbar.ETA(), + ], + ).start() diff --git a/src/mmdiary/video/processor.py b/src/mmdiary/video/processor.py index 5dbfe99..57171b3 100755 --- a/src/mmdiary/video/processor.py +++ b/src/mmdiary/video/processor.py @@ -8,9 +8,8 @@ from datetime import datetime import mixvideoconcat -import progressbar -from mmdiary.utils import log, datelib +from mmdiary.utils import log, datelib, progressbar from mmdiary.utils.medialib import TIME_OUT_FORMAT @@ -76,27 +75,13 @@ def process_all(self): nonprocessed = list(self.__lib.get_nonprocessed()) random.shuffle(nonprocessed) - pbar = progressbar.ProgressBar( - maxval=len(nonprocessed), - widgets=[ - "Process: ", - progressbar.SimpleProgress(), - " (", - progressbar.Percentage(), - ") ", - progressbar.Bar(), - ' ', - progressbar.ETA(), - ], - ).start() - + pbar = progressbar.start("Process", len(nonprocessed)) for date, _ in nonprocessed: try: self.process_date(date) except Exception: logging.exception("Video processing failed") pbar.increment() - pbar.finish() @@ -121,11 +106,11 @@ def main(): for date in args.dates: vp.process_date(date) - logging.info("Done.") + logging.info("Processor done.") if __name__ == "__main__": try: main() except Exception: - logging.exception("Main failed") + logging.exception("Video processor main failed") diff --git a/src/mmdiary/video/uploader.py b/src/mmdiary/video/uploader.py index d1c9a0c..9cf617b 100755 --- a/src/mmdiary/video/uploader.py +++ b/src/mmdiary/video/uploader.py @@ -1,4 +1,5 @@ #!/usr/bin/python3 +# pylint: disable=import-outside-toplevel,no-member # because of false positive on Resource import argparse import json @@ -7,9 +8,8 @@ from datetime import datetime import googleapiclient.discovery -import progressbar -from mmdiary.utils import log, datelib +from mmdiary.utils import log, datelib, progressbar from mmdiary.utils.medialib import TIME_OUT_FORMAT, split_large_text YOUTUBE_MAX_DESCRIPTION = 5000 @@ -206,20 +206,7 @@ def process_date(self, date): def process_all(self): converted = list(self.__lib.get_converted()) - pbar = progressbar.ProgressBar( - maxval=len(converted), - widgets=[ - "Upload: ", - progressbar.SimpleProgress(), - " (", - progressbar.Percentage(), - ") ", - progressbar.Bar(), - ' ', - progressbar.ETA(), - ], - ).start() - + pbar = progressbar.start("Upload", len(converted)) for date, _ in converted: try: if not self.process_date(date): @@ -227,7 +214,6 @@ def process_all(self): except Exception: logging.exception("Video uploading failed") pbar.increment() - pbar.finish() @@ -251,11 +237,11 @@ def main(): for date in args.dates: vup.process_date(date) - logging.info("Done.") + logging.info("Uploader done.") if __name__ == "__main__": try: main() except Exception: - logging.exception("Main failed") + logging.exception("Video uploader main failed")