Removed singleton_main prototype. Moved importing SingletonBoto class…

… to the if statement, as we only need it in s3. Removed the client from the config

make_dirs_main.py

@@ -14,7 +14,6 @@
 def run_make_dirs():
     # root = "s3a://onscdp-dev-data01-5320d6ca/bat/res_dev/project_data"
 
-    global config
     root = "/user/anne.griffith/res_dev/2022_surveys"
     # root = "/bat/res_dev/project_data"
 
@@ -86,7 +85,7 @@ def run_make_dirs():
         },
     }
 
-    dir_list = tree_to_list(tree, path_list=[],prefix=root)
+    dir_list = tree_to_list(tree, path_list=[], prefix=root)
     for s in dir_list:
         print(s)
         mods.rd_mkdir(s)

src/pipeline.py

@@ -20,7 +20,6 @@
 from src.estimation.estimation_main import run_estimation
 from src.site_apportionment.site_apportionment_main import run_site_apportionment
 from src.outputs.outputs_main import run_outputs
-from src.utils.singleton_boto import SingletonBoto
 
 MainLogger = logging.getLogger(__name__)
 
@@ -50,18 +49,19 @@ def run_pipeline(user_config_path, dev_config_path):
 
     if platform == "s3":
         #create singletion boto3 client object & pass in bucket string
+        from src.utils.singleton_boto import SingletonBoto
         boto3_client = SingletonBoto.get_client()
 
         from src.utils import s3_mods as mods
         # Creating boto3 client and adding it to the config dict
-        config["client"] = boto3_client
+        # config["client"] = boto3_client
     elif platform == "network":
         # If the platform is "network" or "hdfs", there is no need for a client.
         # Adding a client = None for consistency.
-        config["client"] = None
+        # config["client"] = None
         from src.utils import local_file_mods as mods
     elif platform == "hdfs":
-        config["client"] = None
+        # config["client"] = None
         from src.utils import hdfs_mods as mods
     else:
         MainLogger.error(f"The selected platform {platform} is wrong")