Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Create make_network_dirs_main.py #368

Open
wants to merge 6 commits into
base: develop
Choose a base branch
from
+173 −0
Open

Create make_network_dirs_main.py #368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
faa4540
Create make_network_dirs_main.py
Ryan2Y79 Nov 5, 2024
b875df0
updated pathing
Ryan2Y79 Nov 5, 2024
5c67446
moved script locations
Ryan2Y79 Nov 5, 2024
c6e517a
renamed script
Ryan2Y79 Nov 5, 2024
198a457
corrected dicts
Ryan2Y79 Nov 5, 2024
ac79103
fixed issue where script wasn't creating subfolders
Ryan2Y79 Nov 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 171 additions & 0 deletions helpers/make_network_dirs_main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
"""Script that creates all directories"""
import sys
sys.path.append("D:/research-and-development")
from src.utils.helpers import tree_to_list
from src.utils.local_file_mods import rd_file_exists
import os

# Change to the project repository location
my_wd = os.getcwd()
my_repo = "R:\\"
if not my_wd.endswith(my_repo):
os.chdir(my_repo)


def make_dir(dir):
"""Creates a directory bassed on the provided folder_path."""
# create directory if it does not exist

if rd_file_exists(dir, raise_error=False) is False:
os.makedirs(dir)
print(f"Created {dir}")
else:
print(f"Directory {dir} already exists.")


def run_make_dirs():
"""Creates all directories based on the `tree` structure.

tree (dict) converted to list of lists by function src.utils.helpers.tree_to_list.
make_dir function is called for item in the list of directories.

"""

root = "BERD Results System Development 2023/DAP_emulation"

tree = {
"2022_surveys": {
"PNP": {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a way to do this where you don't have to hard code survey (PNP / BERD) or year, and you can put these in at the top?

"01_staging": {
"feather": {},
"staging_qa": {
"full_responses_qa": {},
"postcode_validation": {},
"Postcodes for QA": {},
},
},
"02_freezing": {
"changes_to_review": {},
"freezing_updates": {},
"frozen_data_staged": {},
},
"03_northern_ireland": {
"2022": {},
"ni_staging_qa": {},
},
"04_construction": {
"manual_construction": {},
},
"05_mapping": {
"mapping_qa": {},
},
"06_imputation": {
"backdata_output": {},
"imputation_qa": {},
"manual_trimming": {},
},
"07_outliers": {
"auto_outliers": {},
"manual_outliers": {},
"outliers_qa": {},
},
"08_estimation": {
"estimation_qa": {},
},
"09_apportionment": {
"apportionment_qa": {},
},
"10_outputs": {
"output_fte_total_qa": {},
"output_gb_sas": {},
"output_intram_by_civil_defence": {},
"output_intram_by_pg_gb": {},
"output_intram_by_pg_uk": {},
"output_intram_by_sic": {},
"output_intram_gb_itl1": {},
"output_intram_gb_itl2": {},
"output_intram_uk_itl1": {},
"output_intram_uk_itl2": {},
"output_long_form": {},
"output_ni_sas": {},
"output_short_form": {},
"output_status_filtered_qa": {},
"output_tau": {},
},
},
"mappers": {
"v1": {},
}
},
"2023_surveys": {
"PNP": {
"01_staging": {
"feather": {},
"staging_qa": {
"full_responses_qa": {},
"postcode_validation": {},
"Postcodes for QA": {},
},
},
"02_freezing": {
"changes_to_review": {},
"freezing_updates": {},
"frozen_data_staged": {},
},
"03_northern_ireland": {
"2023": {},
"ni_staging_qa": {},
},
"04_construction": {
"manual_construction": {},
},
"05_mapping": {
"mapping_qa": {},
},
"06_imputation": {
"backdata_output": {},
"imputation_qa": {},
"manual_trimming": {},
},
"07_outliers": {
"auto_outliers": {},
"manual_outliers": {},
"outliers_qa": {},
},
"08_estimation": {
"estimation_qa": {},
},
"09_apportionment": {
"apportionment_qa": {},
},
"10_outputs": {
"output_fte_total_qa": {},
"output_gb_sas": {},
"output_intram_by_civil_defence": {},
"output_intram_by_pg_gb": {},
"output_intram_by_pg_uk": {},
"output_intram_by_sic": {},
"output_intram_gb_itl1": {},
"output_intram_gb_itl2": {},
"output_intram_uk_itl1": {},
"output_intram_uk_itl2": {},
"output_long_form": {},
"output_ni_sas": {},
"output_short_form": {},
"output_status_filtered_qa": {},
"output_tau": {},
},
},
"mappers": {
"v1": {},
}
},
}

dir_list = tree_to_list(tree, prefix=root, path_list=[])
for s in dir_list:
make_dir(s)


if __name__ == "__main__":
run_make_dirs()
2 changes: 2 additions & 0 deletions make_dirs_main.py → helpers/make_s3_dirs_main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""Script that creates all directories"""
import os
import sys
sys.path.append("D:/research-and-development")

# Change to the project repository location
my_wd = os.getcwd()
Expand Down
Loading