be - preprocess - add ETA, save the latest guild and channel names an…

…d icons, less verbose logs, remove old channel cache, remove unused sha256 json hash

src/dcef/backend/preprocess/AssetProcessor.py

@@ -8,7 +8,7 @@
 
 from FileFinder import FileFinder
 from MongoDatabase import MongoDatabase
-from helpers import pad_id
+from Formatters import Formatters
 
 print = functools.partial(print, flush=True)
 

src/dcef/backend/preprocess/Eta.py

@@ -0,0 +1,31 @@
+import time
+
+from Formatters import Formatters
+
+
+class Eta:
+    def __init__(self, max_value, count):
+        self.bulgarian_constant = 50000
+        self.max_value = max_value + count * self.bulgarian_constant
+        self.current_value = 0
+        self.time_started = time.time()
+        self.last_increment_time = self.time_started
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        if exc_type is not None:
+            raise exc_type(exc_value).with_traceback(exc_traceback)
+        return True
+
+    def calculate_eta(self):
+        if self.current_value == 0:
+            return "N/A"
+        else:
+            eta = (time.time() - self.time_started) / self.current_value * (self.max_value - self.current_value) - (time.time() - self.last_increment_time)
+        return Formatters.human_time(eta)
+
+    def increment(self, add_value):
+        self.current_value += add_value + self.bulgarian_constant
+        self.last_increment_time = time.time()

src/dcef/backend/preprocess/Formatters.py

@@ -0,0 +1,30 @@
+class Formatters:
+    @staticmethod
+    def human_file_size(file_size_bytes: int) -> str:
+        """
+        returns file size in human readable format
+        """
+        # round to 2 decimal places
+        if file_size_bytes < 1024:
+            return f'{file_size_bytes} B'
+        elif file_size_bytes < 1024 * 1024:
+            return f'{round(file_size_bytes / 1024, 2)} KB'
+        elif file_size_bytes < 1024 * 1024 * 1024:
+            return f'{round(file_size_bytes / 1024 / 1024, 2)} MB'
+        else:
+            return f'{round(file_size_bytes / 1024 / 1024 / 1024, 2)} GB'
+
+    @staticmethod
+    def human_time(seconds_total: int) -> str:
+        """
+        returns time in format HH:MM:SS
+        """
+        hours, remainder = divmod(seconds_total, 3600)
+        minutes, seconds = divmod(remainder, 60)
+        return f"{int(hours):02}:{int(minutes):02}:{int(seconds):02}"
+
+    @staticmethod
+    def pad_id(id: int) -> str:
+        if id == None:
+            return None
+        return str(id).zfill(24)

src/dcef/backend/preprocess/JsonFileStreamer.py

@@ -1,9 +1,8 @@
 import functools
+from Formatters import Formatters
 import ijson
 import datetime
 
-from helpers import human_file_size
-
 print = functools.partial(print, flush=True)
 
 
@@ -26,7 +25,7 @@ def get_file_size_human(self) -> str:
 		"""
 		file_size_bytes = self.get_file_size()
 
-		return human_file_size(file_size_bytes)
+		return Formatters.human_file_size(file_size_bytes)
 
 	def get_file_pointer_position(self) -> int:
 		return self.file.tell()
@@ -83,7 +82,7 @@ def get_exported_at(self) -> str:
 			# print(prefix, event, value)
 			if prefix == 'exportedAt':
 				exportedAt = value
-				print('    found exportedAt field')
+				# print('    found exportedAt field')
 				return exportedAt
 
 			if prefix == 'messages.item.timestamp' or prefix == 'messages.item.timestampEdited':