Merge branch 'master' into 20240909_v0.10.12_nsbgrid

README.rst

@@ -116,7 +116,7 @@ To request a MC analysis:
 
 * a lstchain config file (please provide an exhaustive config that will help others and provide a more explicit provenance information)
 * a lstmcpipe config file (to generate it, please refer to the documentation)
-* a readme with a short description of why you require this analysis to be run. **Do not add information that should not appear publicly** (such as source names) here. If you are requesting a production for a specific new source, please edit `this table on LST wiki <https://www.lst1.iac.es/wiki/index.php/MC_analysis_and_IRF_production#lstmcpipe_productions>`_. Also add the command line to generate the lstmcpipe config, that will help debugging.
+* a readme with a short description of why you require this analysis to be run. **Do not add information that should not appear publicly** (such as source names) here. If you are requesting a production for a specific new source, please edit `this table on LST wiki <https://www.lst1.iac.es/mediawiki/index.php?title=MC_analysis_and_IRF_production>`_. Also add the command line to generate the lstmcpipe config, that will help debugging.
 
 The proposed configuration will be tested for validity by the continuous integration tests and we will interact with you to run the analysis on the cluster at La Palma.
 

codemeta.json

@@ -4,16 +4,16 @@
     "license": "https://spdx.org/licenses/MIT",
     "codeRepository": "git+https://github.com/cta-observatory/lstmcpipe.git",
     "dateCreated": "2019-09-18",
-    "datePublished": "2024-09-18",
+    "datePublished": "2024-10-18",
     "downloadUrl": "https://github.com/cta-observatory/lstmcpipe/archive/refs/tags/v0.11.1.tar.gz",
     "issueTracker": "https://github.com/cta-observatory/lstmcpipe/issues",
     "name": "lstmcpipe",
-    "version": "0.11.1",
-    "softwareVersion": "0.11.1",
+    "version": "0.11.3",
+    "softwareVersion": "0.11.3",
     "readme": "https://github.com/cta-observatory/lstmcpipe/blob/master/README.rst",
     "description": "Scripts to ease the reduction of MC data on the LST cluster at La Palma. With this package, the analysis/creation of R1/DL0/DL1/DL2/IRFs can be orchestrated.",
     "applicationCategory": "CTA, LST",
-    "releaseNotes": "v0.11.1 includes fixes for the NSB-grid prod. For bug fixes, see the **full changelog**: https://github.com/cta-observatory/lstmcpipe/compare/v0.11.0...v0.11.1",
+    "releaseNotes": "lstmcpipe v0.11.3 makes a hotfix about failed jobs directory. For bug fixes, see the **full changelog**: https://github.com/cta-observatory/lstmcpipe/compare/v0.11.2...v0.11.3",
     "funding": "824064",
     "developmentStatus": "active",
     "isPartOf": "https://www.cta-observatory.org/",

docs/pipeline.rst

@@ -264,7 +264,10 @@ In this case, you should produce your own lstchain config file using lstchain to
 
 The workflow then starts from a base production (the `source_prod_id`), produces new tuned DL1 (dl1ab steps) and trains a new set of models.
 
-To prepare the lstmcpipe config, you typically want to run **on the cluster**:
+To prepare the lstmcpipe config, you want to:
+
+- find a base production to start with (see the list of productions in the `documentation <productions>` and look for the latest "base" or "unuted" one) 
+- run **on the cluster**:
 
 .. code-block::
 

lstmcpipe/lstmcpipe_start.py

@@ -194,11 +194,11 @@ def main():
         update_scancel_file(scancel_file, jobs_from_splitting)
         all_job_ids.update({"train_test_split": jobs_from_splitting})
     else:
-        jobs_from_splitting = ""
+        jobs_from_splitting = None
 
     # 2.2 STAGE --> Merge DL1 files
-    if jobs_from_splitting != "":
-        merge_wait_jobs = ",".join([jobs_from_dl1_processing, jobs_from_splitting])
+    if jobs_from_splitting:
+        merge_wait_jobs = jobs_from_splitting
     else:
         merge_wait_jobs = jobs_from_dl1_processing
 

lstmcpipe/scripts/generate_nsb_levels_configs.py

@@ -2,8 +2,10 @@
 import json
 import logging
 import argparse
-from lstchain.io.config import get_mc_config
 from pathlib import Path
+from packaging import version
+from lstchain.io.config import get_mc_config
+from lstchain import __version__ as lstchain_version
 
 BASE_LSTCHAIN_MC_CONFIG = get_mc_config()
 
@@ -55,10 +57,10 @@ def build_argparser():
 
 def lstchain_config_name(nsb_tuning):
     """
-    Generate the name of the lstchain configuration file based on the given nsb_tuning_ratio.
+    Generate the name of the lstchain configuration file based on the given nsb_tuning_rate_GHz.
 
     Parameters:
-        nsb_tuning_ratio (float): The nsb tuning ratio.
+        nsb_tuning_rate_GHz (float): The nsb tuning rate in GHz.
 
     Returns:
         str: The name of the lstchain configuration file.
@@ -68,17 +70,19 @@ def lstchain_config_name(nsb_tuning):
 
 def dump_lstchain_nsb_config(nsb_tuning, outdir="."):
     """
-    Dump the lstchain configuration file with the given nsb_tuning_ratio.
+    Dump the lstchain configuration file with the given nsb_tuning_rate_GHz.
 
     Parameters:
-        nsb_tuning_ratio (float): The nsb tuning ratio.
+        nsb_tuning_rate_GHz (float): The nsb tuning rate in GHz.
     """
     new_config = BASE_LSTCHAIN_MC_CONFIG.copy()
     if nsb_tuning == 0 or nsb_tuning is None:
         new_config["waveform_nsb_tuning"]["nsb_tuning"] = False
     else:
         new_config["waveform_nsb_tuning"]["nsb_tuning"] = True
-        new_config["waveform_nsb_tuning"]["nsb_tuning_ratio"] = nsb_tuning
+        # The parameter name changed in lstchain v0.10.12 !
+        parameter_name = "nsb_tuning_rate_GHz" if version.parse(lstchain_version) >= version.parse("0.10.12") else "nsb_tuning_ratio"
+        new_config["waveform_nsb_tuning"][parameter_name] = nsb_tuning
     json_filename = Path(outdir) / lstchain_config_name(nsb_tuning)
     with open(json_filename, 'w') as f:
         json.dump(new_config, f, indent=4)
@@ -87,10 +91,10 @@ def dump_lstchain_nsb_config(nsb_tuning, outdir="."):
 
 def prod_id(nsb_tuning, prefix=None):
     """
-    Generate the prod ID based on the given nsb_tuning_ratio.
+    Generate the prod ID based on the given nsb_tuning_rate_GHz.
 
     Parameters:
-        nsb_tuning_ratio (float): The nsb tuning ratio.
+        nsb_tuning_rate_GHz (float): The nsb tuning rate in GHz.
         prefix (str): The prefix of the prod ID. Example: "20240918_v0.10.12"
     Returns:
         str: The prod ID.
@@ -101,10 +105,10 @@ def prod_id(nsb_tuning, prefix=None):
 
 def lstmcpipe_config_filename(nsb_tuning, outdir="."):
     """
-    Generate the name of the lstmcpipe configuration file based on the given nsb_tuning_ratio.
+    Generate the name of the lstmcpipe configuration file based on the given nsb_tuning_rate_GHz.
 
     Parameters:
-        nsb_tuning_ratio (float): The nsb tuning ratio.
+        nsb_tuning_rate_GHz (float): The nsb tuning rate in GHz.
 
     Returns:
         str: The name of the lstmcpipe configuration file.
@@ -114,7 +118,7 @@ def lstmcpipe_config_filename(nsb_tuning, outdir="."):
 
 def main():
     """
-    Dump the lstchain and lstmcpipe configuration files for the given nsb_tuning_ratios.
+    Dump the lstchain and lstmcpipe configuration files for the given nsb_tuning_rate_GHz.
     """
     parser = build_argparser()
     args = parser.parse_args()

lstmcpipe/scripts/script_batch_filelist_lst.py

@@ -21,15 +21,15 @@ def main():
     parser.add_argument(
         "--output_dir",
         "-o",
-        type=str,
+        type=Path,
         dest="output_dir",
         help="lstchain_mc_r0_to_dl1 output directory argument.",
         required=True,
     )
     parser.add_argument(
         "--config_file",
         "-c",
-        type=str,
+        type=Path,
         dest="config_file",
         help="lstchain_mc_r0_to_dl1 configuration file argument.",
         required=True,
@@ -57,7 +57,7 @@ def main():
             if args.config_file:
                 cmd.append("--config={}".format(args.config_file))
 
-            outfile = file.parent.joinpath('dl1_' + file.name.replace('.simtel.gz', '.h5')).as_posix()
+            outfile = args.output_dir.joinpath('dl1_' + file.name.replace('.simtel.gz', '.h5')).as_posix()
             rerun_cmd(cmd, outfile, max_ntry=2)
 
 

lstmcpipe/scripts/script_batch_filelist_lst_dl1ab.py

@@ -2,7 +2,8 @@
 
 import argparse
 from os import environ
-from os.path import join, basename
+from os.path import basename
+from pathlib import Path
 from lstmcpipe.utils import rerun_cmd
 
 
@@ -22,15 +23,15 @@ def main():
     parser.add_argument(
         "--output_dir",
         "-o",
-        type=str,
+        type=Path,
         dest="output_dir",
         help="lstchain_mc_r0_to_dl1 output directory argument.",
         required=True,
     )
     parser.add_argument(
         "--config_file",
         "-c",
-        type=str,
+        type=Path,
         dest="config_file",
         help="lstchain_mc_r0_to_dl1 configuration file argument.",
         required=True,
@@ -49,10 +50,7 @@ def main():
     with open(file_for_this_job, "r") as filelist:
         for file in filelist:
             file = file.strip("\n")
-            # TODO: We need ways to set the pedestal cleaning!
-            # or do we? is that not only for observed data?
-            output = join(args.output_dir, basename(file))
-            # TODO --no-image should be passed by default. To be configurable from lstmcpipe config ?
+            output = args.output_dir.joinpath(basename(file))
             cmd = [
                 "lstchain_dl1ab",
                 "--no-image",

lstmcpipe/tests/test_utils.py

@@ -34,23 +34,24 @@ def test_rerun_cmd():
     with tempfile.TemporaryDirectory() as tmp_dir:
         file, filename = tempfile.mkstemp(dir=tmp_dir)
         cmd = f'echo "1" >> {filename}; rm nonexistingfile'
+
+        filename = Path(filename)
 
         # first test: the cmd fails 3 times but the outfile stays in place
-        subdir_failures = ""
+        failed_jobs_dir = Path(tmp_dir) / ""
         try:
-            n_tries = rerun_cmd(cmd, filename, max_ntry=3, subdir_failures=subdir_failures, shell=True)
-            filename = Path(filename)
-            filename = Path(tmp_dir).joinpath(subdir_failures, filename.name)
+            n_tries = rerun_cmd(cmd, filename, max_ntry=3, failed_jobs_dir=failed_jobs_dir, shell=True)
+            filename = failed_jobs_dir.joinpath(filename.name)
             assert open(filename).read() == "1\n1\n1\n"
             assert n_tries == 3
         except Exception as e:
             assert isinstance(e, RuntimeError)
 
         # 2nd test: the cmd fails and the outfile is moved in subdir
-        subdir_failures = "fail"
+        failed_jobs_dir = filename.parent.joinpath("fail")
         try:
-            rerun_cmd(cmd, filename, max_ntry=3, subdir_failures=subdir_failures, shell=True)
-            filename = filename.parent.joinpath(subdir_failures).joinpath(filename.name)
+            rerun_cmd(cmd, filename, max_ntry=3, failed_jobs_dir=failed_jobs_dir, shell=True)
+            filename = failed_jobs_dir.joinpath(filename.name)
             assert open(filename).read() == "1\n"
             assert filename.exists()
         except Exception as e:
@@ -66,7 +67,7 @@ def test_rerun_cmd_lstchain_mc_r0_to_dl1(mc_gamma_testfile):
         ntry = rerun_cmd(cmd, outfile, max_ntry=3)
         assert ntry == 1
         # second try should fail because the outfile already exists
-        ntry = rerun_cmd(cmd, outfile, max_ntry=3, subdir_failures='failed_outputs')
+        ntry = rerun_cmd(cmd, outfile, max_ntry=3, failed_jobs_dir=Path(tmp_dir) / 'failed_outputs')
         assert ntry == 2
         assert Path(tmp_dir, 'failed_outputs', outfilename).exists()
 

lstmcpipe/utils.py

@@ -12,6 +12,8 @@
 from copy import deepcopy
 from deepdiff import DeepDiff
 
+from . import prod_logs
+
 log = logging.getLogger(__name__)
 
 
@@ -109,7 +111,7 @@ def batch_mc_production_check(
     return jobid
 
 
-def rerun_cmd(cmd, outfile, max_ntry=2, subdir_failures="failed_outputs", **run_kwargs):
+def rerun_cmd(cmd, outfile, max_ntry=2, failed_jobs_dir=prod_logs/"failed_outputs", **run_kwargs):
     """
     Rerun the command up to max_ntry times.
     If all attempts fail, raise an exception.
@@ -122,7 +124,7 @@ def rerun_cmd(cmd, outfile, max_ntry=2, subdir_failures="failed_outputs", **run_
         Path to the cmd output file
     max_ntry: int
         Maximum number of attempts to run the command
-    subdir_failures: str
+    failed_jobs_dir: Path or str
         Subdirectory to move failed output files to
     run_kwargs: kwargs
         Additional keyword arguments for subprocess.run
@@ -133,18 +135,18 @@ def rerun_cmd(cmd, outfile, max_ntry=2, subdir_failures="failed_outputs", **run_
         If the command fails after all retry attempts
     """
     outfile = Path(outfile)
+    failed_jobs_dir = Path(failed_jobs_dir)
     for ntry in range(1, max_ntry + 1):
         result = sp.run(cmd, **run_kwargs, capture_output=True, text=True, check=False)
 
         if result.returncode == 0:
             return ntry  # Success, return the number of tries it took
 
         # Command failed, handle the error
-        failed_jobs_subdir = outfile.parent.joinpath(subdir_failures)
         if outfile.exists():
-            failed_jobs_subdir.mkdir(exist_ok=True)
-            outfile_target = failed_jobs_subdir.joinpath(outfile.name)
-            print(f"Move failed output file from {outfile} to {outfile_target}. try #{ntry}")
+            failed_jobs_dir.mkdir(exist_ok=True)
+            outfile_target = failed_jobs_dir.joinpath(outfile.name)
+            print(f"Try #{ntry} - move failed output file from {outfile} to {outfile_target}")
             shutil.move(outfile, outfile_target)
 
         # If this was the last try, raise an exception

production_configs/20240917_v0.10.12_src15_dec_min_2924_tuned/README.md

@@ -0,0 +1,19 @@
+# PROD 20240917_v0.10.12_src15_dec_min_2924_tuned
+
+I need this production to analyse the LST-1 data of src15 (see https://www.lst1.iac.es/wiki/index.php/MC_analysis_and_IRF_production#lstmcpipe_productions) with dec 29.24 deg. The median of the NSB standard deviation for this dataset is approximately 2.308 p.e.
+
+'''
+lstmcpipe_generate_config PathConfigAllSkyFullDL1ab --dec_list dec_min_2924 --prod_id 20240917_v0.10.12_src15_dec_min_2924_tuned --kwargs source_prod_id=20240131_allsky_v0.10.5_all_dec_base
+'''
+
+Config for the NSB tuning:
+
+"image_modifier": {
+    "increase_nsb": true,
+    "extra_noise_in_dim_pixels": 3.843,
+    "extra_bias_in_dim_pixels": 1.07,
+    "transition_charge": 8,
+    "extra_noise_in_bright_pixels": 5.178
+  }
+
+Contact: Lisa N