From 950d4de3ffae2a97c6779d271287e6bc2be31316 Mon Sep 17 00:00:00 2001 From: George Zorinyants Date: Tue, 1 Oct 2024 12:57:27 +0000 Subject: [PATCH] Removed singleton_main prototype. Moved importing SingletonBoto class to the if statement, as we only need it in s3. Removed the client from the config --- make_dirs_main.py | 3 +-- singleton_main.py | 31 ------------------------------- src/pipeline.py | 8 ++++---- 3 files changed, 5 insertions(+), 37 deletions(-) delete mode 100644 singleton_main.py diff --git a/make_dirs_main.py b/make_dirs_main.py index 0165b6645..4eac6e5a4 100644 --- a/make_dirs_main.py +++ b/make_dirs_main.py @@ -14,7 +14,6 @@ def run_make_dirs(): # root = "s3a://onscdp-dev-data01-5320d6ca/bat/res_dev/project_data" - global config root = "/user/anne.griffith/res_dev/2022_surveys" # root = "/bat/res_dev/project_data" @@ -86,7 +85,7 @@ def run_make_dirs(): }, } - dir_list = tree_to_list(tree, path_list=[],prefix=root) + dir_list = tree_to_list(tree, path_list=[], prefix=root) for s in dir_list: print(s) mods.rd_mkdir(s) diff --git a/singleton_main.py b/singleton_main.py deleted file mode 100644 index d8725ef6b..000000000 --- a/singleton_main.py +++ /dev/null @@ -1,31 +0,0 @@ -''' -Example of using a singleton class that initialises the boto3 client -This class will be: - Accessible globally throughout the codebase - Only one instance will exist, and this instance is used every time -''' -import os - -os.chdir('../../../home/cdsw/research-and-development') -print(f"Current working directory is {os.getcwd()}") - -from src.utils.singleton_boto import SingletonBoto - - -def main(): - - my_client1 = SingletonBoto.get_client() - - if not (my_client1 is None): - print("Client created") - - my_client2 = SingletonBoto.get_client() - - if my_client1 is my_client2: - print("same client") - else: - print("different clients") - - -if __name__ == "__main__": - main() diff --git a/src/pipeline.py b/src/pipeline.py index 95d11624c..4ffed727a 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -20,7 +20,6 @@ from src.estimation.estimation_main import run_estimation from src.site_apportionment.site_apportionment_main import run_site_apportionment from src.outputs.outputs_main import run_outputs -from src.utils.singleton_boto import SingletonBoto MainLogger = logging.getLogger(__name__) @@ -50,18 +49,19 @@ def run_pipeline(user_config_path, dev_config_path): if platform == "s3": #create singletion boto3 client object & pass in bucket string + from src.utils.singleton_boto import SingletonBoto boto3_client = SingletonBoto.get_client() from src.utils import s3_mods as mods # Creating boto3 client and adding it to the config dict - config["client"] = boto3_client + # config["client"] = boto3_client elif platform == "network": # If the platform is "network" or "hdfs", there is no need for a client. # Adding a client = None for consistency. - config["client"] = None + # config["client"] = None from src.utils import local_file_mods as mods elif platform == "hdfs": - config["client"] = None + # config["client"] = None from src.utils import hdfs_mods as mods else: MainLogger.error(f"The selected platform {platform} is wrong")