In_out_versions

magicctapipe/scripts/lst1_magic/semi_automatic_scripts/coincident_events.py

@@ -66,14 +66,19 @@ def configfile_coincidence(target_dir, source_name, config_file):
         "event_coincidence": config_dict["event_coincidence"],
     }
 
-    file_name = f"{target_dir}/v{__version__}/{source_name}/config_coincidence.yaml"
+    conf_dir = f"{target_dir}/v{__version__}/{source_name}"
+    os.makedirs(conf_dir, exist_ok=True)
+
+    file_name = f"{conf_dir}/config_coincidence.yaml"
 
     with open(file_name, "w") as f:
 
         yaml.dump(conf, f, default_flow_style=False)
 
 
-def linking_bash_lst(target_dir, LST_runs, source_name, LST_version, env_name, cluster):
+def linking_bash_lst(
+    target_dir, LST_runs, source_name, LST_version, env_name, cluster, version
+):
 
     """
     This function links the LST data paths to the working directory and creates bash scripts.
@@ -92,11 +97,13 @@ def linking_bash_lst(target_dir, LST_runs, source_name, LST_version, env_name, c
         Name of the conda environment
     cluster : str
         Cluster system
+    version : str
+        Version of the input (DL1 MAGIC runs) data
     """
 
     coincidence_DL1_dir = f"{target_dir}/v{__version__}/{source_name}"
 
-    MAGIC_DL1_dir = f"{target_dir}/v{__version__}/{source_name}/DL1"
+    MAGIC_DL1_dir = f"{target_dir}/v{version}/{source_name}/DL1"
 
     dates = [os.path.basename(x) for x in glob.glob(f"{MAGIC_DL1_dir}/Merged/[0-9]*")]
     if cluster != "SLURM":
@@ -199,6 +206,9 @@ def main():
     source = config["data_selection"]["source_name_output"]
 
     cluster = config["general"]["cluster"]
+    in_version = config["directories"]["real_input_version"]
+    if in_version == "":
+        in_version == __version__
 
     if source_in is None:
         source_list = joblib.load("list_sources.dat")
@@ -227,6 +237,7 @@ def main():
                 LST_version,
                 env_name,
                 cluster,
+                in_version,
             )  # linking the data paths to current working directory
 
             print("***** Submitting processess to the cluster...")

magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml

@@ -1,6 +1,7 @@
 directories:
     workspace_dir: "/fefs/aswg/workspace/elisa.visentin/auto_MCP_PR/"  # Output directory where all the data products will be saved.
-
+    MC_version: "" #MCP version used to process MC; if not provided, current MCP version will be used
+    real_input_version: ""  #MCP version used to process input real data (to be used in case you want to run only the last steps of the analysis, using lower-level data previously processed); if not provided, current MCP version will be used 
 
 data_selection:
     source_name_database: "CrabNebula"  # MUST BE THE SAME AS IN THE DATABASE; Set to null to process all sources in the given time range.

magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merge_stereo.py

@@ -27,7 +27,7 @@
 logger.setLevel(logging.INFO)
 
 
-def MergeStereo(target_dir, env_name, source, cluster):
+def MergeStereo(target_dir, env_name, source, cluster, version):
     """
     This function creates the bash scripts to run merge_hdf_files.py in all DL1Stereo subruns.
 
@@ -41,10 +41,12 @@ def MergeStereo(target_dir, env_name, source, cluster):
         Name of the target
     cluster : str
         Cluster system
+    version : str
+        Version of the input (stereo subruns) data
     """
 
     process_name = source
-    stereo_DL1_dir = f"{target_dir}/v{__version__}/{source}"
+    stereo_DL1_dir = f"{target_dir}/v{version}/{source}"
     listOfNightsLST = np.sort(glob.glob(f"{stereo_DL1_dir}/DL1Stereo/*"))
     if cluster != "SLURM":
         logger.warning(
@@ -53,7 +55,9 @@ def MergeStereo(target_dir, env_name, source, cluster):
         return
     for nightLST in listOfNightsLST:
         night = nightLST.split("/")[-1]
-        stereoMergeDir = f"{stereo_DL1_dir}/DL1Stereo/Merged/{night}"
+        stereoMergeDir = (
+            f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/{night}"
+        )
         os.makedirs(f"{stereoMergeDir}/logs", exist_ok=True)
 
         if len(glob.glob(f"{nightLST}/dl1_stereo*.h5")) < 1:
@@ -109,6 +113,9 @@ def main():
     source_in = config["data_selection"]["source_name_database"]
     source = config["data_selection"]["source_name_output"]
     cluster = config["general"]["cluster"]
+    in_version = config["directories"]["real_input_version"]
+    if in_version == "":
+        in_version == __version__
 
     if source_in is None:
         source_list = joblib.load("list_sources.dat")
@@ -121,7 +128,7 @@ def main():
     for source_name in source_list:
 
         print("***** Merging DL1Stereo files run-wise...")
-        MergeStereo(target_dir, env_name, source_name, cluster)
+        MergeStereo(target_dir, env_name, source_name, cluster, in_version)
 
         list_of_merge = glob.glob(f"{source_name}_StereoMerge_*.sh")
         if len(list_of_merge) < 1:

magicctapipe/scripts/lst1_magic/semi_automatic_scripts/merging_runs.py

@@ -33,7 +33,7 @@
 logger.setLevel(logging.INFO)
 
 
-def merge(target_dir, MAGIC_runs, env_name, source, cluster):
+def merge(target_dir, MAGIC_runs, env_name, source, cluster, version):
 
     """
     This function creates the bash scripts to run merge_hdf_files.py for real data
@@ -50,11 +50,14 @@ def merge(target_dir, MAGIC_runs, env_name, source, cluster):
         Target name
     cluster : str
         Cluster system
+    version : str
+        Version of the input (DL1 MAGIC subruns) data
     """
 
     process_name = f"merging_{source}"
 
-    MAGIC_DL1_dir = f"{target_dir}/v{__version__}/{source}/DL1/"
+    MAGIC_in_dir = f"{target_dir}/v{version}/{source}/DL1/"
+    MAGIC_out_dir = f"{target_dir}/v{__version__}/{source}/DL1/"
 
     if cluster != "SLURM":
         logger.warning(
@@ -65,18 +68,18 @@ def merge(target_dir, MAGIC_runs, env_name, source, cluster):
         queue="short",
         job_name=process_name,
         mem="2g",
-        out_name=f"{MAGIC_DL1_dir}/Merged/logs/slurm-%x.%j",
+        out_name=f"{MAGIC_out_dir}/Merged/logs/slurm-%x.%j",
     )
-    os.makedirs(f"{MAGIC_DL1_dir}/Merged/logs", exist_ok=True)
+    os.makedirs(f"{MAGIC_out_dir}/Merged/logs", exist_ok=True)
 
     with open(f"{source}_Merge_MAGIC.sh", "w") as f:
         f.writelines(lines)
         for magic in [1, 2]:
             for i in MAGIC_runs:
                 # Here is a difference w.r.t. original code. If only one telescope data are available they will be merged now for this telescope
-                indir = f"{MAGIC_DL1_dir}/M{magic}/{i[0]}/{i[1]}"
+                indir = f"{MAGIC_in_dir}/M{magic}/{i[0]}/{i[1]}"
                 if os.path.exists(f"{indir}"):
-                    outdir = f"{MAGIC_DL1_dir}/Merged/{i[0]}"
+                    outdir = f"{MAGIC_out_dir}/Merged/{i[0]}"
                     os.makedirs(f"{outdir}/logs", exist_ok=True)
 
                     f.write(
@@ -120,6 +123,9 @@ def main():
     source_in = config["data_selection"]["source_name_database"]
     source = config["data_selection"]["source_name_output"]
     cluster = config["general"]["cluster"]
+    in_version = config["directories"]["real_input_version"]
+    if in_version == "":
+        in_version == __version__
 
     if source_in is None:
         source_list = joblib.load("list_sources.dat")
@@ -144,6 +150,7 @@ def main():
             env_name,
             source_name,
             cluster,
+            in_version,
         )  # generating the bash script to merge the subruns
 
         print("***** Running merge_hdf_files.py on the MAGIC data files...")

magicctapipe/scripts/lst1_magic/semi_automatic_scripts/stereo_events.py

@@ -57,13 +57,18 @@ def configfile_stereo(target_dir, source_name, config_file):
         "mc_tel_ids": config_dict["mc_tel_ids"],
         "stereo_reco": config_dict["stereo_reco"],
     }
-    file_name = f"{target_dir}/v{__version__}/{source_name}/config_stereo.yaml"
+
+    conf_dir = f"{target_dir}/v{__version__}/{source_name}"
+    os.makedirs(conf_dir, exist_ok=True)
+
+    file_name = f"{conf_dir}/config_stereo.yaml"
+
     with open(file_name, "w") as f:
 
         yaml.dump(conf, f, default_flow_style=False)
 
 
-def bash_stereo(target_dir, source, env_name, cluster):
+def bash_stereo(target_dir, source, env_name, cluster, version):
 
     """
     This function generates the bashscripts for running the stereo analysis.
@@ -78,11 +83,13 @@ def bash_stereo(target_dir, source, env_name, cluster):
         Name of the environment
     cluster : str
         Cluster system
+    version : str
+        Version of the input (coincident) data
     """
 
     process_name = source
 
-    coincidence_DL1_dir = f"{target_dir}/v{__version__}/{source}"
+    coincidence_DL1_dir = f"{target_dir}/v{version}/{source}"
 
     listOfNightsLST = np.sort(glob.glob(f"{coincidence_DL1_dir}/DL1Coincident/*"))
     if cluster != "SLURM":
@@ -92,7 +99,7 @@ def bash_stereo(target_dir, source, env_name, cluster):
         return
     for nightLST in listOfNightsLST:
         night = nightLST.split("/")[-1]
-        stereoDir = f"{coincidence_DL1_dir}/DL1Stereo/{night}"
+        stereoDir = f"{target_dir}/v{__version__}/{source}/DL1Stereo/{night}"
         os.makedirs(f"{stereoDir}/logs", exist_ok=True)
         if not os.listdir(f"{nightLST}"):
             continue
@@ -166,6 +173,9 @@ def main():
     source = config["data_selection"]["source_name_output"]
 
     cluster = config["general"]["cluster"]
+    in_version = config["directories"]["real_input_version"]
+    if in_version == "":
+        in_version == __version__
 
     if source_in is None:
         source_list = joblib.load("list_sources.dat")
@@ -182,7 +192,7 @@ def main():
         # Below we run the analysis on the real data
 
         print("***** Generating the bashscript...")
-        bash_stereo(target_dir, source_name, env_name, cluster)
+        bash_stereo(target_dir, source_name, env_name, cluster, in_version)
 
         print("***** Submitting processess to the cluster...")
         print(f"Process name: {source_name}_stereo")