From f9ab4725b320eaae9ea91752d48793bb09a5d9c5 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Fri, 6 Sep 2024 07:07:37 +0000 Subject: [PATCH 1/9] in & MC v in config --- .../lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml index 1ce9c418a..769ef9090 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml @@ -1,6 +1,7 @@ directories: workspace_dir: "/fefs/aswg/workspace/elisa.visentin/auto_MCP_PR/" # Output directory where all the data products will be saved. - + MC_version: "" #MCP version used to process MC; if not provided, current MCP version will be used + real_input_version: "" #MCP version used to process input real data (to be used in case you want to run only the last steps of the analysis, using lower-level data previously processed); if not provided, current MCP version will be used data_selection: source_name_database: "CrabNebula" # MUST BE THE SAME AS IN THE DATABASE; Set to null to process all sources in the given time range. From 6c6d32fafc9088cbe6a048a430923a94cac7eaed Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Fri, 6 Sep 2024 07:07:53 +0000 Subject: [PATCH 2/9] in v: merging MAGIC --- .../lst1_magic/semi_automatic_scripts/merging_runs.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merging_runs.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merging_runs.py index ca80d7d56..a2cc884ef 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merging_runs.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merging_runs.py @@ -33,7 +33,7 @@ logger.setLevel(logging.INFO) -def merge(target_dir, MAGIC_runs, env_name, source, cluster): +def merge(target_dir, MAGIC_runs, env_name, source, cluster, version): """ This function creates the bash scripts to run merge_hdf_files.py for real data @@ -50,11 +50,13 @@ def merge(target_dir, MAGIC_runs, env_name, source, cluster): Target name cluster : str Cluster system + version : str + Version of the input (DL1 MAGIC subruns) data """ process_name = f"merging_{source}" - MAGIC_DL1_dir = f"{target_dir}/v{__version__}/{source}/DL1/" + MAGIC_DL1_dir = f"{target_dir}/v{version}/{source}/DL1/" if cluster != "SLURM": logger.warning( @@ -120,6 +122,9 @@ def main(): source_in = config["data_selection"]["source_name_database"] source = config["data_selection"]["source_name_output"] cluster = config["general"]["cluster"] + in_version = config["directories"]["real_input_version"] + if in_version =="": + in_version == __version__ if source_in is None: source_list = joblib.load("list_sources.dat") @@ -144,6 +149,7 @@ def main(): env_name, source_name, cluster, + in_version, ) # generating the bash script to merge the subruns print("***** Running merge_hdf_files.py on the MAGIC data files...") From adb7abf47c4a90b349df13802b35bee760e34340 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Fri, 6 Sep 2024 07:14:13 +0000 Subject: [PATCH 3/9] in v: coincidence --- .../semi_automatic_scripts/coincident_events.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py index 542c5ba67..40f207655 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py @@ -73,7 +73,7 @@ def configfile_coincidence(target_dir, source_name, config_file): yaml.dump(conf, f, default_flow_style=False) -def linking_bash_lst(target_dir, LST_runs, source_name, LST_version, env_name, cluster): +def linking_bash_lst(target_dir, LST_runs, source_name, LST_version, env_name, cluster, version): """ This function links the LST data paths to the working directory and creates bash scripts. @@ -92,11 +92,13 @@ def linking_bash_lst(target_dir, LST_runs, source_name, LST_version, env_name, c Name of the conda environment cluster : str Cluster system + version : str + Version of the input (DL1 MAGIC runs) data """ coincidence_DL1_dir = f"{target_dir}/v{__version__}/{source_name}" - MAGIC_DL1_dir = f"{target_dir}/v{__version__}/{source_name}/DL1" + MAGIC_DL1_dir = f"{target_dir}/v{version}/{source_name}/DL1" dates = [os.path.basename(x) for x in glob.glob(f"{MAGIC_DL1_dir}/Merged/[0-9]*")] if cluster != "SLURM": @@ -199,6 +201,10 @@ def main(): source = config["data_selection"]["source_name_output"] cluster = config["general"]["cluster"] + in_version = config["directories"]["real_input_version"] + if in_version =="": + in_version == __version__ + if source_in is None: source_list = joblib.load("list_sources.dat") From b1660d146478841cf0477ec37d0c7dd508cf5350 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Fri, 6 Sep 2024 07:14:26 +0000 Subject: [PATCH 4/9] in v: fix directories --- .../lst1_magic/semi_automatic_scripts/merging_runs.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merging_runs.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merging_runs.py index a2cc884ef..fd22215d4 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merging_runs.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merging_runs.py @@ -56,7 +56,8 @@ def merge(target_dir, MAGIC_runs, env_name, source, cluster, version): process_name = f"merging_{source}" - MAGIC_DL1_dir = f"{target_dir}/v{version}/{source}/DL1/" + MAGIC_in_dir = f"{target_dir}/v{version}/{source}/DL1/" + MAGIC_out_dir= f"{target_dir}/v{__version__}/{source}/DL1/ if cluster != "SLURM": logger.warning( @@ -67,18 +68,18 @@ def merge(target_dir, MAGIC_runs, env_name, source, cluster, version): queue="short", job_name=process_name, mem="2g", - out_name=f"{MAGIC_DL1_dir}/Merged/logs/slurm-%x.%j", + out_name=f"{MAGIC_out_dir}/Merged/logs/slurm-%x.%j", ) - os.makedirs(f"{MAGIC_DL1_dir}/Merged/logs", exist_ok=True) + os.makedirs(f"{MAGIC_out_dir}/Merged/logs", exist_ok=True) with open(f"{source}_Merge_MAGIC.sh", "w") as f: f.writelines(lines) for magic in [1, 2]: for i in MAGIC_runs: # Here is a difference w.r.t. original code. If only one telescope data are available they will be merged now for this telescope - indir = f"{MAGIC_DL1_dir}/M{magic}/{i[0]}/{i[1]}" + indir = f"{MAGIC_in_dir}/M{magic}/{i[0]}/{i[1]}" if os.path.exists(f"{indir}"): - outdir = f"{MAGIC_DL1_dir}/Merged/{i[0]}" + outdir = f"{MAGIC_out_dir}/Merged/{i[0]}" os.makedirs(f"{outdir}/logs", exist_ok=True) f.write( From 27ed17343f8c1ee601e1bd5bba71831d96ebe4d1 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Fri, 6 Sep 2024 07:18:27 +0000 Subject: [PATCH 5/9] minor bug --- .../lst1_magic/semi_automatic_scripts/coincident_events.py | 1 + 1 file changed, 1 insertion(+) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py index 40f207655..e49d3c1b2 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py @@ -233,6 +233,7 @@ def main(): LST_version, env_name, cluster, + in_version, ) # linking the data paths to current working directory print("***** Submitting processess to the cluster...") From 9ad19049a90d28dce104bca0356087ca4915c898 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Fri, 6 Sep 2024 07:18:37 +0000 Subject: [PATCH 6/9] in v: stereo --- .../semi_automatic_scripts/stereo_events.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/stereo_events.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/stereo_events.py index a2f8a4eea..4988e5426 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/stereo_events.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/stereo_events.py @@ -63,7 +63,7 @@ def configfile_stereo(target_dir, source_name, config_file): yaml.dump(conf, f, default_flow_style=False) -def bash_stereo(target_dir, source, env_name, cluster): +def bash_stereo(target_dir, source, env_name, cluster, version): """ This function generates the bashscripts for running the stereo analysis. @@ -78,11 +78,13 @@ def bash_stereo(target_dir, source, env_name, cluster): Name of the environment cluster : str Cluster system + version : str + Version of the input (coincident) data """ process_name = source - coincidence_DL1_dir = f"{target_dir}/v{__version__}/{source}" + coincidence_DL1_dir = f"{target_dir}/v{version}/{source}" listOfNightsLST = np.sort(glob.glob(f"{coincidence_DL1_dir}/DL1Coincident/*")) if cluster != "SLURM": @@ -92,7 +94,7 @@ def bash_stereo(target_dir, source, env_name, cluster): return for nightLST in listOfNightsLST: night = nightLST.split("/")[-1] - stereoDir = f"{coincidence_DL1_dir}/DL1Stereo/{night}" + stereoDir = f"{target_dir}/v{__version__}/{source}/DL1Stereo/{night}" os.makedirs(f"{stereoDir}/logs", exist_ok=True) if not os.listdir(f"{nightLST}"): continue @@ -166,6 +168,9 @@ def main(): source = config["data_selection"]["source_name_output"] cluster = config["general"]["cluster"] + in_version = config["directories"]["real_input_version"] + if in_version =="": + in_version == __version__ if source_in is None: source_list = joblib.load("list_sources.dat") @@ -182,7 +187,7 @@ def main(): # Below we run the analysis on the real data print("***** Generating the bashscript...") - bash_stereo(target_dir, source_name, env_name, cluster) + bash_stereo(target_dir, source_name, env_name, cluster, in_version) print("***** Submitting processess to the cluster...") print(f"Process name: {source_name}_stereo") From 52369e5eb85076a8f8f3a7a4777003fdbc2003ac Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Fri, 6 Sep 2024 07:21:54 +0000 Subject: [PATCH 7/9] in v: merge stereo --- .../semi_automatic_scripts/merge_stereo.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merge_stereo.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merge_stereo.py index 1833498fc..495dde4fb 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merge_stereo.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merge_stereo.py @@ -27,7 +27,7 @@ logger.setLevel(logging.INFO) -def MergeStereo(target_dir, env_name, source, cluster): +def MergeStereo(target_dir, env_name, source, cluster, version): """ This function creates the bash scripts to run merge_hdf_files.py in all DL1Stereo subruns. @@ -41,10 +41,12 @@ def MergeStereo(target_dir, env_name, source, cluster): Name of the target cluster : str Cluster system + version : str + Version of the input (stereo subruns) data """ process_name = source - stereo_DL1_dir = f"{target_dir}/v{__version__}/{source}" + stereo_DL1_dir = f"{target_dir}/v{version}/{source}" listOfNightsLST = np.sort(glob.glob(f"{stereo_DL1_dir}/DL1Stereo/*")) if cluster != "SLURM": logger.warning( @@ -53,7 +55,7 @@ def MergeStereo(target_dir, env_name, source, cluster): return for nightLST in listOfNightsLST: night = nightLST.split("/")[-1] - stereoMergeDir = f"{stereo_DL1_dir}/DL1Stereo/Merged/{night}" + stereoMergeDir = f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/{night}" os.makedirs(f"{stereoMergeDir}/logs", exist_ok=True) if len(glob.glob(f"{nightLST}/dl1_stereo*.h5")) < 1: @@ -109,6 +111,9 @@ def main(): source_in = config["data_selection"]["source_name_database"] source = config["data_selection"]["source_name_output"] cluster = config["general"]["cluster"] + in_version = config["directories"]["real_input_version"] + if in_version =="": + in_version == __version__ if source_in is None: source_list = joblib.load("list_sources.dat") @@ -121,7 +126,7 @@ def main(): for source_name in source_list: print("***** Merging DL1Stereo files run-wise...") - MergeStereo(target_dir, env_name, source_name, cluster) + MergeStereo(target_dir, env_name, source_name, cluster, in_version) list_of_merge = glob.glob(f"{source_name}_StereoMerge_*.sh") if len(list_of_merge) < 1: From 1f9db8971ffc05cf008bdb6f6daf04adcd63475a Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Fri, 6 Sep 2024 07:25:26 +0000 Subject: [PATCH 8/9] linter --- .../lst1_magic/semi_automatic_scripts/coincident_events.py | 7 ++++--- .../lst1_magic/semi_automatic_scripts/merge_stereo.py | 6 ++++-- .../lst1_magic/semi_automatic_scripts/merging_runs.py | 4 ++-- .../lst1_magic/semi_automatic_scripts/stereo_events.py | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py index e49d3c1b2..edabea6e5 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py @@ -73,7 +73,9 @@ def configfile_coincidence(target_dir, source_name, config_file): yaml.dump(conf, f, default_flow_style=False) -def linking_bash_lst(target_dir, LST_runs, source_name, LST_version, env_name, cluster, version): +def linking_bash_lst( + target_dir, LST_runs, source_name, LST_version, env_name, cluster, version +): """ This function links the LST data paths to the working directory and creates bash scripts. @@ -202,10 +204,9 @@ def main(): cluster = config["general"]["cluster"] in_version = config["directories"]["real_input_version"] - if in_version =="": + if in_version == "": in_version == __version__ - if source_in is None: source_list = joblib.load("list_sources.dat") diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merge_stereo.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merge_stereo.py index 495dde4fb..cf8b90736 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merge_stereo.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merge_stereo.py @@ -55,7 +55,9 @@ def MergeStereo(target_dir, env_name, source, cluster, version): return for nightLST in listOfNightsLST: night = nightLST.split("/")[-1] - stereoMergeDir = f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/{night}" + stereoMergeDir = ( + f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/{night}" + ) os.makedirs(f"{stereoMergeDir}/logs", exist_ok=True) if len(glob.glob(f"{nightLST}/dl1_stereo*.h5")) < 1: @@ -112,7 +114,7 @@ def main(): source = config["data_selection"]["source_name_output"] cluster = config["general"]["cluster"] in_version = config["directories"]["real_input_version"] - if in_version =="": + if in_version == "": in_version == __version__ if source_in is None: diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merging_runs.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merging_runs.py index fd22215d4..27c1cb143 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merging_runs.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merging_runs.py @@ -57,7 +57,7 @@ def merge(target_dir, MAGIC_runs, env_name, source, cluster, version): process_name = f"merging_{source}" MAGIC_in_dir = f"{target_dir}/v{version}/{source}/DL1/" - MAGIC_out_dir= f"{target_dir}/v{__version__}/{source}/DL1/ + MAGIC_out_dir = f"{target_dir}/v{__version__}/{source}/DL1/" if cluster != "SLURM": logger.warning( @@ -124,7 +124,7 @@ def main(): source = config["data_selection"]["source_name_output"] cluster = config["general"]["cluster"] in_version = config["directories"]["real_input_version"] - if in_version =="": + if in_version == "": in_version == __version__ if source_in is None: diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/stereo_events.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/stereo_events.py index 4988e5426..dcdbdcf5b 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/stereo_events.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/stereo_events.py @@ -169,7 +169,7 @@ def main(): cluster = config["general"]["cluster"] in_version = config["directories"]["real_input_version"] - if in_version =="": + if in_version == "": in_version == __version__ if source_in is None: From 9f01945441153954c7cde89954a27f59bfef997a Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Mon, 16 Sep 2024 08:47:52 +0000 Subject: [PATCH 9/9] bug fix --- .../lst1_magic/semi_automatic_scripts/coincident_events.py | 5 ++++- .../lst1_magic/semi_automatic_scripts/stereo_events.py | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py index edabea6e5..5736f570e 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py @@ -66,7 +66,10 @@ def configfile_coincidence(target_dir, source_name, config_file): "event_coincidence": config_dict["event_coincidence"], } - file_name = f"{target_dir}/v{__version__}/{source_name}/config_coincidence.yaml" + conf_dir = f"{target_dir}/v{__version__}/{source_name}" + os.makedirs(conf_dir, exist_ok=True) + + file_name = f"{conf_dir}/config_coincidence.yaml" with open(file_name, "w") as f: diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/stereo_events.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/stereo_events.py index dcdbdcf5b..4e80a3a8d 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/stereo_events.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/stereo_events.py @@ -57,7 +57,12 @@ def configfile_stereo(target_dir, source_name, config_file): "mc_tel_ids": config_dict["mc_tel_ids"], "stereo_reco": config_dict["stereo_reco"], } - file_name = f"{target_dir}/v{__version__}/{source_name}/config_stereo.yaml" + + conf_dir = f"{target_dir}/v{__version__}/{source_name}" + os.makedirs(conf_dir, exist_ok=True) + + file_name = f"{conf_dir}/config_stereo.yaml" + with open(file_name, "w") as f: yaml.dump(conf, f, default_flow_style=False)