From 2f239ae4f7e8ab100f8bc9d12bf33c52d83efaef Mon Sep 17 00:00:00 2001 From: Ryan Lyttle Date: Mon, 6 Jan 2025 19:10:16 +0000 Subject: [PATCH 1/5] Cleaning issues related to config file dataclasses Removed some commented out lines, removed unused functions and their unit tests in PPConfigParser.py, made sure all config variables are outputted to log and a unit test has been made for function PrintConfigsToLog. --- benchmarks/bench_cProfile.py | 4 +- src/sorcha/modules/PPConfigParser.py | 935 ------------------- src/sorcha/sorcha.py | 2 +- src/sorcha/utilities/diffTestUtils.py | 1 - src/sorcha/utilities/sorchaConfigs.py | 39 +- src/sorcha_cmdline/run.py | 4 - tests/data/test_PPPrintConfigsToLog.txt | 276 +++++- tests/ephemeris/test_ephemeris_generation.py | 1 - tests/ephemeris/test_pixdict.py | 1 - tests/sorcha/test_PPConfigParser.py | 283 ------ tests/sorcha/test_sorchaConfigs.py | 50 +- 11 files changed, 318 insertions(+), 1278 deletions(-) diff --git a/benchmarks/bench_cProfile.py b/benchmarks/bench_cProfile.py index 71e759af..813e8c14 100644 --- a/benchmarks/bench_cProfile.py +++ b/benchmarks/bench_cProfile.py @@ -8,7 +8,7 @@ from pstats import SortKey from sorcha.sorcha import runLSSTSimulation # noqa: F401 -from sorcha.modules.PPConfigParser import PPConfigFileParser +from sorcha.utilities.sorchaConfigs import sorchaConfigs from sorcha.utilities.sorchaArguments import sorchaArguments import argparse @@ -39,7 +39,7 @@ args_obj = sorchaArguments(cmd_args_dict) - configs = PPConfigFileParser(os.path.join(path_to_sorcha, "benchmarks/test_bench_config.ini"), "LSST") + configs = sorchaConfigs(os.path.join(path_to_sorcha, "benchmarks/test_bench_config.ini"), "LSST") cProfile.run("runLSSTSimulation(args_obj, configs)", os.path.join(path_to_sorcha, "tests/out/restats")) diff --git a/src/sorcha/modules/PPConfigParser.py b/src/sorcha/modules/PPConfigParser.py index 96f6e216..c586ed7e 100755 --- a/src/sorcha/modules/PPConfigParser.py +++ b/src/sorcha/modules/PPConfigParser.py @@ -1,247 +1,6 @@ import logging import os import sys -import numpy as np -import configparser - -from sorcha.lightcurves.lightcurve_registration import LC_METHODS -from sorcha.activity.activity_registration import CA_METHODS - - -def log_error_and_exit(message: str) -> None: - """Log a message to the error output file and terminal, then exit. - - Parameters - ---------- - message : string - The error message to be logged to the error output file. - - - Returns - -------- - None - """ - - logging.error(message) - sys.exit(message) - - -def PPGetOrExit(config, section, key, message): - """ - Checks to see if the config file parser has a key. If it does not, this - function errors out and the code stops. - - Parameters - ----------- - config : ConfigParser - ConfigParser object containing configs. - - section : string - Section of the key being checked. - - key : string) - The key being checked. - - message : string - The message to log and display if the key is not found. - - Returns - ---------- - None. - - """ - - if config.has_option(section, key): - return config[section][key] - else: - logging.error(message) - sys.exit(message) - - -def PPGetFloatOrExit(config, section, key, message): - """ - Checks to see if a key in the config parser is present and can be read as a - float. If it cannot, this function errors out and the code stops. - - Parameters - ----------- - config : ConfigParser - ConfigParser object containing configs. - - section : string - section of the key being checked. - - key : string - The key being checked. - - message : string - The message to log and display if the key is not found. - - Returns - ---------- - None. - - """ - - if config.has_option(section, key): - try: - val = config.getfloat(section, key) - return val - except ValueError: - logging.error( - "ERROR: expected a float for config parameter {}. Check value in config file.".format(key) - ) - sys.exit( - "ERROR: expected a float for config parameter {}. Check value in config file.".format(key) - ) - else: - logging.error(message) - sys.exit(message) - - -def PPGetIntOrExit(config, section, key, message): - """ - Checks to see if a key in the config parser is present and can be read as an - int. If it cannot, this function errors out and the code stops. - - Parameters - ----------- - config : ConfigParser - ConfigParser object containing configs. - - section : string - Section of the key being checked. - - key : string - The key being checked. - - message : string - The message to log and display if the key is not found. - - Returns - ---------- - None. - - """ - - if config.has_option(section, key): - try: - val = config.getint(section, key) - return val - except ValueError: - logging.error( - "ERROR: expected an int for config parameter {}. Check value in config file.".format(key) - ) - sys.exit( - "ERROR: expected an int for config parameter {}. Check value in config file.".format(key) - ) - else: - logging.error(message) - sys.exit(message) - - -def PPGetBoolOrExit(config, section, key, message): - """ - Checks to see if a key in the config parser is present and can be read as a - Boolean. If it cannot, this function errors out and the code stops. - - Parameters - ----------- - config : ConfigParser object - ConfigParser object containing configs. - - section : string - Section of the key being checked. - - key : string - The key being checked. - - message : string - The message to log and display if the key is not found. - - Returns - ---------- - None. - - """ - - if config.has_option(section, key): - try: - val = config.getboolean(section, key) - return val - except ValueError: - logging.error(f"ERROR: {key} could not be converted to a Boolean.") - sys.exit(f"ERROR: {key} could not be converted to a Boolean.") - else: - logging.error(message) - sys.exit(message) - - -def PPGetValueAndFlag(config, section, key, type_wanted): - """ - Obtains a value from the config flag, forcing it to be the specified - type and error-handling if it can't be forced. If the value is not present - in the config fie, the flag is set to False; if it is, the flag is True. - - Parameters - ----------- - config : ConfigParser - ConfigParser object containing configs. - - section : string - Section of the key being checked. - - key : string - The key being checked. - - type_wanted : string - The type the value should be forced to. - Accepts int, float, none (for no type-forcing). - - Returns - ---------- - value : any type - The value of the key, with type dependent on type_wanted. - Will be None if the key is not present. - - flag : boolean - Will be False if the key is not present in the config file - and True if it is. - - """ - - if type_wanted == "int": - try: - value = config.getint(section, key, fallback=None) - except ValueError: - logging.error( - "ERROR: expected an int for config parameter {}. Check value in config file.".format(key) - ) - sys.exit( - "ERROR: expected an int for config parameter {}. Check value in config file.".format(key) - ) - elif type_wanted == "float": - try: - value = config.getfloat(section, key, fallback=None) - except ValueError: - logging.error( - "ERROR: expected a float for config parameter {}. Check value in config file.".format(key) - ) - sys.exit( - "ERROR: expected a float for config parameter {}. Check value in config file.".format(key) - ) - elif type_wanted == "none": - value = config.get(section, key, fallback=None) - else: - logging.error("ERROR: internal error: type not recognised.") - sys.exit("ERROR: internal error: type not recognised.") - - if value is None: - flag = False - else: - flag = True - - return value, flag def PPFindFileOrExit(arg_fn, argname): @@ -297,697 +56,3 @@ def PPFindDirectoryOrExit(arg_fn, argname): else: pplogger.error("ERROR: filepath {} supplied for {} argument does not exist.".format(arg_fn, argname)) sys.exit("ERROR: filepath {} supplied for {} argument does not exist.".format(arg_fn, argname)) - - -def PPCheckFiltersForSurvey(survey_name, observing_filters): - """ - When given a list of filters, this function checks to make sure they exist in the - user-selected survey, and if the filters given in the config file do not match the - survey filters, the function exits the program with an error. - - Parameters - ----------- - survey_name : string - Survey name. Currently only "LSST", "lsst" accepted. - - observing_filters: list of strings - Observation filters of interest. - - Returns - ---------- - None. - - Notes - ------- - Currently only has options for LSST, but can be expanded upon later. - - """ - - pplogger = logging.getLogger(__name__) - - if survey_name in ["rubin_sim", "RUBIN_SIM"]: - lsst_filters = ["u", "g", "r", "i", "z", "y"] - filters_ok = all(elem in lsst_filters for elem in observing_filters) - - if not filters_ok: - bad_list = np.setdiff1d(observing_filters, lsst_filters) - pplogger.error( - "ERROR: Filter(s) {} given in config file are not recognised filters for {} survey.".format( - bad_list, survey_name - ) - ) - pplogger.error("Accepted {} filters: {}".format("LSST", lsst_filters)) - pplogger.error("Change observing_filters in config file or select another survey.") - sys.exit( - "ERROR: Filter(s) {} given in config file are not recognised filters for {} survey.".format( - bad_list, survey_name - ) - ) - - -def PPConfigFileParser(configfile, survey_name): - """ - Parses the config file, error-handles, then assigns the values into a single - dictionary, which is passed out. - - Parameters - ----------- - configfile : string - Filepath/name of config file. - - survey_name : string - Survey name. Currently only "LSST", "lsst" accepted. - - Returns - ---------- - config_dict : dictionary - Dictionary of config file variables. - - Notes - ------- - We chose not to use the original ConfigParser object for readability: it's a dict of - dicts, so calling the various values can become quite unwieldy. - - """ - pplogger = logging.getLogger(__name__) - - # Save a raw copy of the configuration to the logs as a backup. - with open(configfile, "r") as file: - pplogger.info(f"Copy of configuration file {configfile}:\n{file.read()}") - - config = configparser.ConfigParser() - config.read(configfile) - - config_dict = {} - - # INPUT - - config_dict["eph_format"] = PPGetOrExit( - config, "INPUT", "eph_format", "ERROR: no ephemerides file format is specified." - ).lower() - if config_dict["eph_format"] not in ["csv", "whitespace", "hdf5"]: - pplogger.error("ERROR: eph_format should be either csv, whitespace, or hdf5.") - sys.exit("ERROR: eph_format should be either either csv, whitespace, or hdf5.") - - config_dict["aux_format"] = PPGetOrExit( - config, "INPUT", "aux_format", "ERROR: no auxiliary data format specified." - ).lower() - if config_dict["aux_format"] not in ["comma", "whitespace", "csv"]: - pplogger.error("ERROR: aux_format should be either comma, csv, or whitespace.") - sys.exit("ERROR: aux_format should be either comma, csv, or whitespace.") - - config_dict["ephemerides_type"] = PPGetOrExit( - config, "INPUT", "ephemerides_type", "ERROR: no ephemerides type provided." - ).lower() - if config_dict["ephemerides_type"] not in ["ar", "external"]: - pplogger.error('ERROR: ephemerides_type not recognised - expecting either "ar" or "external".') - sys.exit('ERROR: ephemerides_type not recognised - expecting either "ar" or "external".') - - config_dict["size_serial_chunk"] = PPGetIntOrExit( - config, "INPUT", "size_serial_chunk", "ERROR: size_serial_chunk not specified." - ) - if config_dict["size_serial_chunk"] < 1: - pplogger.error("ERROR: size_serial_chunk is zero or negative.") - sys.exit("ERROR: size_serial_chunk is zero or negative.") - - config_dict["pointing_sql_query"] = PPGetOrExit( - config, "INPUT", "pointing_sql_query", "ERROR: no pointing database SQLite3 query provided." - ) - - # ACTIVITY - - config_dict["comet_activity"] = config.get("ACTIVITY", "comet_activity", fallback=None) - config_dict["comet_activity"] = ( - None if config_dict["comet_activity"] == "none" else config_dict["comet_activity"] - ) - - # If the user defined a specific comet_activity value, but the model has not been registered, exit the program. - if config_dict["comet_activity"] and config_dict["comet_activity"] not in CA_METHODS: - log_error_and_exit( - f"The requested comet activity model, '{config_dict['comet_activity']}', is not registered. Available comet activity models are: {list(CA_METHODS.keys())}" - ) - - # FILTERS - - obsfilters = PPGetOrExit( - config, "FILTERS", "observing_filters", "ERROR: observing_filters config file variable not provided." - ) - config_dict["observing_filters"] = [e.strip() for e in obsfilters.split(",")] - - PPCheckFiltersForSurvey(survey_name, config_dict["observing_filters"]) - - # SATURATION - - bright_limits, config_dict["bright_limit_on"] = PPGetValueAndFlag( - config, "SATURATION", "bright_limit", "none" - ) - - if config_dict["bright_limit_on"]: - try: - bright_list = [float(e.strip()) for e in bright_limits.split(",")] - except ValueError: - pplogger.error("ERROR: could not parse brightness limits. Check formatting and try again.") - sys.exit("ERROR: could not parse brightness limits. Check formatting and try again.") - - if len(bright_list) == 1: - config_dict["bright_limit"] = bright_list[0] - else: - if len(bright_list) != len(config_dict["observing_filters"]): - pplogger.error( - "ERROR: list of saturation limits is not the same length as list of observing filters." - ) - sys.exit( - "ERROR: list of saturation limits is not the same length as list of observing filters." - ) - config_dict["bright_limit"] = bright_list - - # PHASECURVES - - config_dict["phase_function"] = PPGetOrExit( - config, "PHASECURVES", "phase_function", "ERROR: phase function not defined." - ) - - # FOV - - config_dict["camera_model"] = PPGetOrExit( - config, "FOV", "camera_model", "ERROR: camera model not defined." - ) - - if config_dict["camera_model"] not in ["circle", "footprint", "none"]: - pplogger.error('ERROR: camera_model should be either "circle" or "footprint".') - sys.exit('ERROR: camera_model should be either "circle" or "footprint".') - - elif config_dict["camera_model"] == "footprint": - config_dict["footprint_path"], external_file = PPGetValueAndFlag( - config, "FOV", "footprint_path", "none" - ) - if external_file: - PPFindFileOrExit(config_dict["footprint_path"], "footprint_path") - elif survey_name.lower() not in ["lsst", "rubin_sim"]: - log_error_and_exit( - "a default detector footprint is currently only provided for LSST; please provide your own footprint file." - ) - - config_dict["footprint_edge_threshold"], _ = PPGetValueAndFlag( - config, "FOV", "footprint_edge_threshold", "float" - ) - - if config.has_option("FOV", "fill_factor"): - pplogger.error('ERROR: fill factor supplied in config file but camera model is not "circle".') - sys.exit('ERROR: fill factor supplied in config file but camera model is not "circle".') - elif config.has_option("FOV", "circle_radius"): - pplogger.error('ERROR: circle radius supplied in config file but camera model is not "circle".') - sys.exit('ERROR: circle radius supplied in config file but camera model is not "circle".') - - elif (config_dict["camera_model"]) == "circle": - config_dict["fill_factor"], _ = PPGetValueAndFlag(config, "FOV", "fill_factor", "float") - config_dict["circle_radius"], _ = PPGetValueAndFlag(config, "FOV", "circle_radius", "float") - - if not config_dict["fill_factor"] and not config_dict["circle_radius"]: - pplogger.error( - 'ERROR: either "fill_factor" or "circle_radius" must be specified for circular footprint.' - ) - sys.exit( - 'ERROR: either "fill_factor" or "circle_radius" must be specified for circular footprint.' - ) - elif config_dict["fill_factor"]: - if config_dict["fill_factor"] < 0.0 or config_dict["fill_factor"] > 1.0: - pplogger.error("ERROR: fill_factor out of bounds. Must be between 0 and 1.") - sys.exit("ERROR: fill_factor out of bounds. Must be between 0 and 1.") - elif config_dict["circle_radius"]: - if config_dict["circle_radius"] < 0.0: - pplogger.error("ERROR: circle_radius is negative.") - sys.exit("ERROR: circle_radius is negative.") - - if config.has_option("FOV", "footprint_edge_threshold"): - pplogger.error( - 'ERROR: footprint edge threshold supplied in config file but camera model is not "footprint".' - ) - sys.exit( - 'ERROR: footprint edge threshold supplied in config file but camera model is not "footprint".' - ) - - # FADINGFUNCTION - - config_dict["fading_function_on"] = PPGetBoolOrExit( - config, "FADINGFUNCTION", "fading_function_on", "ERROR: fading_function_on flag not present." - ) - - if config_dict["fading_function_on"]: - config_dict["fading_function_width"] = PPGetFloatOrExit( - config, - "FADINGFUNCTION", - "fading_function_width", - "ERROR: fading function is on but no fading_function_width supplied.", - ) - config_dict["fading_function_peak_efficiency"] = PPGetFloatOrExit( - config, - "FADINGFUNCTION", - "fading_function_peak_efficiency", - "ERROR: fading function is on but no fading_function_peak_efficiency supplied.", - ) - - if config_dict["fading_function_width"] <= 0.0 or config_dict["fading_function_width"] > 0.5: - pplogger.error( - "ERROR: fading_function_width out of bounds. Must be greater than zero and less than 0.5." - ) - sys.exit( - "ERROR: fading_function_width out of bounds. Must be greater than zero and less than 0.5." - ) - - if ( - config_dict["fading_function_peak_efficiency"] < 0.0 - or config_dict["fading_function_peak_efficiency"] > 1.0 - ): - pplogger.error("ERROR: fading_function_peak_efficiency out of bounds. Must be between 0 and 1.") - sys.exit("ERROR: fading_function_peak_efficiency out of bounds. Must be between 0 and 1.") - - elif config.has_option("FADINGFUNCTION", "fading_function_width"): - pplogger.error( - "ERROR: fading_function_width supplied in config file but fading_function_on is False." - ) - sys.exit("ERROR: fading_function_width supplied in config file but fading_function_on is False.") - elif config.has_option("FADINGFUNCTION", "fading_function_peak_efficiency"): - pplogger.error( - "ERROR: fading_function_peak_efficiency supplied in config file but fading_function_on is False." - ) - sys.exit( - "ERROR: fading_function_peak_efficiency supplied in config file but fading_function_on is False." - ) - - # LINKINGFILTER - - config_dict["SSP_separation_threshold"], _ = PPGetValueAndFlag( - config, "LINKINGFILTER", "SSP_separation_threshold", "float" - ) - config_dict["SSP_number_observations"], _ = PPGetValueAndFlag( - config, "LINKINGFILTER", "SSP_number_observations", "int" - ) - config_dict["SSP_number_tracklets"], _ = PPGetValueAndFlag( - config, "LINKINGFILTER", "SSP_number_tracklets", "int" - ) - config_dict["SSP_track_window"], _ = PPGetValueAndFlag(config, "LINKINGFILTER", "SSP_track_window", "int") - config_dict["SSP_detection_efficiency"], _ = PPGetValueAndFlag( - config, "LINKINGFILTER", "SSP_detection_efficiency", "float" - ) - config_dict["SSP_maximum_time"], _ = PPGetValueAndFlag( - config, "LINKINGFILTER", "SSP_maximum_time", "float" - ) - config_dict["SSP_night_start_utc"], _ = PPGetValueAndFlag( - config, "LINKINGFILTER", "SSP_night_start_utc", "float" - ) - - SSPvariables = [ - config_dict["SSP_separation_threshold"], - config_dict["SSP_number_observations"], - config_dict["SSP_number_tracklets"], - config_dict["SSP_track_window"], - config_dict["SSP_detection_efficiency"], - config_dict["SSP_maximum_time"], - config_dict["SSP_night_start_utc"], - ] - - # the below if-statement explicitly checks for None so a zero triggers the correct error - if all(v is not None for v in SSPvariables): - if config_dict["SSP_number_observations"] < 1: - pplogger.error("ERROR: SSP_number_observations is zero or negative.") - sys.exit("ERROR: SSP_number_observations is zero or negative.") - - if config_dict["SSP_number_tracklets"] < 1: - pplogger.error("ERROR: SSP_number_tracklets is zero or less.") - sys.exit("ERROR: SSP_number_tracklets is zero or less.") - - if config_dict["SSP_track_window"] <= 0.0: - pplogger.error("ERROR: SSP_track_window is negative.") - sys.exit("ERROR: SSP_track_window is negative.") - - if config_dict["SSP_detection_efficiency"] > 1.0 or config_dict["SSP_detection_efficiency"] > 1.0: - pplogger.error("ERROR: SSP_detection_efficiency out of bounds (should be between 0 and 1).") - sys.exit("ERROR: SSP_detection_efficiency out of bounds (should be between 0 and 1).") - - if config_dict["SSP_separation_threshold"] <= 0.0: - pplogger.error("ERROR: SSP_separation_threshold is zero or negative.") - sys.exit("ERROR: SSP_separation_threshold is zero or negative.") - - if config_dict["SSP_maximum_time"] < 0: - pplogger.error("ERROR: SSP_maximum_time is negative.") - sys.exit("ERROR: SSP_maximum_time is negative.") - - if config_dict["SSP_night_start_utc"] > 24.0 or config_dict["SSP_night_start_utc"] < 0.0: - pplogger.error("ERROR: SSP_night_start_utc must be a valid time between 0 and 24 hours.") - sys.exit("ERROR: SSP_night_start_utc must be a valid time between 0 and 24 hours.") - - config_dict["SSP_linking_on"] = True - - elif not any(SSPvariables): - config_dict["SSP_linking_on"] = False - - else: - pplogger.error( - "ERROR: only some SSP linking variables supplied. Supply all five required variables for SSP linking filter, or none to turn filter off." - ) - sys.exit( - "ERROR: only some SSP linking variables supplied. Supply all five required variables for SSP linking filter, or none to turn filter off." - ) - - try: - config_dict["drop_unlinked"] = config.getboolean("LINKINGFILTER", "drop_unlinked", fallback=True) - except ValueError: - pplogger.error( - "ERROR: could not parse value for drop_unlinked as a boolean. Check formatting and try again." - ) - sys.exit( - "ERROR: could not parse value for drop_unlinked as a boolean. Check formatting and try again." - ) - - # SIMULATION - - if config_dict["ephemerides_type"] == "ar": - config_dict["ar_ang_fov"] = PPGetFloatOrExit( - config, "SIMULATION", "ar_ang_fov", "ERROR: ar_ang_fov not specified." - ) - - config_dict["ar_fov_buffer"] = PPGetFloatOrExit( - config, "SIMULATION", "ar_fov_buffer", "ERROR: ar_fov_buffer not specified." - ) - - config_dict["ar_picket"] = PPGetIntOrExit( - config, "SIMULATION", "ar_picket", "ERROR: ar_picket not specified." - ) - - config_dict["ar_obs_code"] = PPGetOrExit( - config, "SIMULATION", "ar_obs_code", "ERROR: ar_picket not specified." - ) - - config_dict["ar_healpix_order"] = PPGetIntOrExit( - config, "SIMULATION", "ar_healpix_order", "ERROR: ar_healpix_order not specified." - ) - - # OUTPUT - - config_dict["output_format"] = PPGetOrExit( - config, "OUTPUT", "output_format", "ERROR: output format not specified." - ).lower() - if config_dict["output_format"] not in ["csv", "sqlite3", "hdf5", "h5"]: - pplogger.error("ERROR: output_format should be either csv, sqlite3 or hdf5.") - sys.exit("ERROR: output_format should be either csv, sqlite3 or hdf5.") - - config_dict["output_columns"] = PPGetOrExit( - config, "OUTPUT", "output_columns", "ERROR: output size not specified." - ) - - if (config_dict["output_columns"] not in ["basic", "all"]) and ("," not in config_dict["output_columns"]): - pplogger.error( - "ERROR: output_columns not recognised. Must be 'basic', 'all', or a comma-separated list of columns." - ) - sys.exit( - "ERROR: output_columns not recognised. Must be 'basic', 'all', or a comma-separated list of columns." - ) - - # note: if providing a comma-separated list of column names, this is NOT ERROR-HANDLED - # as we have no way of knowing ahead of time of columns that user-generated code or add-ons may add. - - if ( - "," in config_dict["output_columns"] - ): # assume list of column names: turn into a list and strip whitespace - config_dict["output_columns"] = [ - colname.strip(" ") for colname in config_dict["output_columns"].split(",") - ] - - config_dict["position_decimals"], _ = PPGetValueAndFlag(config, "OUTPUT", "position_decimals", "int") - config_dict["magnitude_decimals"], _ = PPGetValueAndFlag(config, "OUTPUT", "magnitude_decimals", "int") - - if config_dict["position_decimals"] and config_dict["position_decimals"] < 0: - pplogger.error("ERROR: decimal places config variables cannot be negative.") - sys.exit("ERROR: decimal places config variables cannot be negative.") - - if config_dict["magnitude_decimals"] and config_dict["magnitude_decimals"] < 0: - pplogger.error("ERROR: decimal places config variables cannot be negative.") - sys.exit("ERROR: decimal places config variables cannot be negative.") - - # EXPERT - - config_dict["SNR_limit"], config_dict["SNR_limit_on"] = PPGetValueAndFlag( - config, "EXPERT", "SNR_limit", "float" - ) - config_dict["mag_limit"], config_dict["mag_limit_on"] = PPGetValueAndFlag( - config, "EXPERT", "mag_limit", "float" - ) - - if config_dict["SNR_limit_on"] and config_dict["SNR_limit"] < 0: - pplogger.error("ERROR: SNR limit is negative.") - sys.exit("ERROR: SNR limit is negative.") - - if config_dict["mag_limit_on"] and config_dict["mag_limit"] < 0: - pplogger.error("ERROR: magnitude limit is negative.") - sys.exit("ERROR: magnitude limit is negative.") - - if config_dict["mag_limit_on"] and config_dict["SNR_limit_on"]: - pplogger.error( - "ERROR: SNR limit and magnitude limit are mutually exclusive. Please delete one or both from config file." - ) - sys.exit( - "ERROR: SNR limit and magnitude limit are mutually exclusive. Please delete one or both from config file." - ) - - try: - config_dict["trailing_losses_on"] = config.getboolean("EXPERT", "trailing_losses_on", fallback=True) - except ValueError: - pplogger.error( - "ERROR: could not parse value for trailing_losses_on as a boolean. Check formatting and try again." - ) - sys.exit( - "ERROR: could not parse value for trailing_losses_on as a boolean. Check formatting and try again." - ) - - try: - config_dict["default_SNR_cut"] = config.getboolean("EXPERT", "default_SNR_cut", fallback=True) - except ValueError: - pplogger.error( - "ERROR: could not parse value for default_SNR_cut as a boolean. Check formatting and try again." - ) - sys.exit( - "ERROR: could not parse value for default_SNR_cut as a boolean. Check formatting and try again." - ) - - try: - config_dict["randomization_on"] = config.getboolean("EXPERT", "randomization_on", fallback=True) - except ValueError: - pplogger.error( - "ERROR: could not parse value for randomization_on as a boolean. Check formatting and try again." - ) - sys.exit( - "ERROR: could not parse value for randomization_on as a boolean. Check formatting and try again." - ) - - try: - config_dict["vignetting_on"] = config.getboolean("EXPERT", "vignetting_on", fallback=True) - except ValueError: - pplogger.error( - "ERROR: could not parse value for vignetting_on as a boolean. Check formatting and try again." - ) - sys.exit( - "ERROR: could not parse value for vignetting_on as a boolean. Check formatting and try again." - ) - - # LIGHTCURVEß - - config_dict["lc_model"] = config.get("LIGHTCURVE", "lc_model", fallback=None) - config_dict["lc_model"] = None if config_dict["lc_model"] == "none" else config_dict["lc_model"] - - # If the user defined a lightcurve model, but the model has not been registered, exit the program. - if config_dict["lc_model"] and config_dict["lc_model"] not in LC_METHODS: - log_error_and_exit( - f"The requested light curve model, '{config_dict['lc_model']}', is not registered. Available lightcurve options are: {list(LC_METHODS.keys())}" - ) - - return config_dict - - -def PPPrintConfigsToLog(configs, cmd_args): - """ - Prints all the values from the config file and command line to the log. - - Parameters - ----------- - configs : dictionary - Dictionary of config file variables. - - cmd_args : dictionary - Dictionary of command line arguments. - - Returns - ---------- - None. - - """ - - pplogger = logging.getLogger(__name__) - - pplogger.info("The config file used is located at " + cmd_args.configfile) - pplogger.info("The physical parameters file used is located at " + cmd_args.paramsinput) - pplogger.info("The orbits file used is located at " + cmd_args.orbinfile) - if cmd_args.input_ephemeris_file: - pplogger.info("The ephemerides file used is located at " + cmd_args.input_ephemeris_file) - if cmd_args.output_ephemeris_file: - pplogger.info("The output ephemerides file is located " + cmd_args.output_ephemeris_file) - pplogger.info("The survey selected is: " + cmd_args.surveyname) - - if configs["comet_activity"] == "comet": - pplogger.info("Cometary activity set to: " + str(configs["comet_activity"])) - elif configs["comet_activity"] == "none": - pplogger.info("No cometary activity selected.") - - pplogger.info("Format of ephemerides file is: " + configs["eph_format"]) - pplogger.info("Format of auxiliary files is: " + configs["aux_format"]) - - pplogger.info("Pointing database path is: " + cmd_args.pointing_database) - pplogger.info("Pointing database required query is: " + configs["pointing_sql_query"]) - - pplogger.info( - "The number of objects processed in a single chunk is: " + str(configs["size_serial_chunk"]) - ) - pplogger.info("The main filter in which H is defined is " + configs["mainfilter"]) - rescs = " ".join(str(f) for f in configs["observing_filters"]) - pplogger.info("The filters included in the post-processing results are " + rescs) - - if configs["othercolours"]: - othcs = " ".join(str(e) for e in configs["othercolours"]) - pplogger.info("Thus, the colour indices included in the simulation are " + othcs) - - pplogger.info( - "The apparent brightness is calculated using the following phase function model: " - + configs["phase_function"] - ) - - if configs["trailing_losses_on"]: - pplogger.info("Computation of trailing losses is switched ON.") - else: - pplogger.info("Computation of trailing losses is switched OFF.") - - if configs["randomization_on"]: - pplogger.info("Randomization of position and magnitude around uncertainties is switched ON.") - else: - pplogger.info("Randomization of position and magnitude around uncertainties is switched OFF.") - - if configs["vignetting_on"]: - pplogger.info("Vignetting is switched ON.") - else: - pplogger.info("Vignetting is switched OFF.") - - if configs["camera_model"] == "footprint": - pplogger.info("Footprint is modelled after the actual camera footprint.") - if configs["footprint_path"]: - pplogger.info("Loading camera footprint from " + configs["footprint_path"]) - else: - pplogger.info("Loading default LSST footprint LSST_detector_corners_100123.csv") - elif configs["camera_model"] == "circle": - pplogger.info("Footprint is circular.") - if configs["fill_factor"]: - pplogger.info( - "The code will approximate chip gaps using filling factor: " + str(configs["fill_factor"]) - ) - elif configs["circle_radius"]: - pplogger.info( - "A circular footprint will be applied with radius: " + str(configs["circle_radius"]) - ) - else: - pplogger.info("Camera footprint is turned OFF.") - - if configs["bright_limit_on"]: - pplogger.info("The upper saturation limit(s) is/are: " + str(configs["bright_limit"])) - else: - pplogger.info("Saturation limit is turned OFF.") - - if configs["SNR_limit_on"]: - pplogger.info("The lower SNR limit is: " + str(configs["SNR_limit"])) - else: - pplogger.info("SNR limit is turned OFF.") - - if configs["default_SNR_cut"]: - pplogger.info("Default SNR cut is ON. All observations with SNR < 2.0 will be removed.") - - if configs["mag_limit_on"]: - pplogger.info("The magnitude limit is: " + str(configs["mag_limit"])) - else: - pplogger.info("Magnitude limit is turned OFF.") - - if configs["fading_function_on"]: - pplogger.info("The detection efficiency fading function is ON.") - pplogger.info( - "The width parameter of the fading function has been set to: " - + str(configs["fading_function_width"]) - ) - pplogger.info( - "The peak efficiency of the fading function has been set to: " - + str(configs["fading_function_peak_efficiency"]) - ) - else: - pplogger.info("The detection efficiency fading function is OFF.") - - if configs["SSP_linking_on"]: - pplogger.info("Solar System Processing linking filter is turned ON.") - pplogger.info("For SSP linking...") - pplogger.info( - "...the fractional detection efficiency is: " + str(configs["SSP_detection_efficiency"]) - ) - pplogger.info( - "...the minimum required number of observations in a tracklet is: " - + str(configs["SSP_number_observations"]) - ) - pplogger.info( - "...the minimum required number of tracklets to form a track is: " - + str(configs["SSP_number_tracklets"]) - ) - pplogger.info( - "...the maximum window of time in days of tracklets to be contained in to form a track is: " - + str(configs["SSP_track_window"]) - ) - pplogger.info( - "...the minimum angular separation between observations in arcseconds is: " - + str(configs["SSP_separation_threshold"]) - ) - pplogger.info( - "...the maximum temporal separation between subsequent observations in a tracklet in days is: " - + str(configs["SSP_maximum_time"]) - ) - if not configs["drop_unlinked"]: - pplogger.info("Unlinked objects will not be dropped.") - else: - pplogger.info("Solar System Processing linking filter is turned OFF.") - - if configs["ephemerides_type"] == "ar": - pplogger.info("ASSIST+REBOUND Simulation is turned ON.") - pplogger.info("For ASSIST+REBOUND...") - pplogger.info("...the field's angular FOV is: " + str(configs["ar_ang_fov"])) - pplogger.info("...the buffer around the FOV is: " + str(configs["ar_fov_buffer"])) - pplogger.info("...the picket interval is: " + str(configs["ar_picket"])) - pplogger.info("...the observatory code is: " + str(configs["ar_obs_code"])) - pplogger.info("...the healpix order is: " + str(configs["ar_healpix_order"])) - else: - pplogger.info("ASSIST+REBOUND Simulation is turned OFF.") - - if configs["lc_model"]: - pplogger.info("A lightcurve model is being applied.") - pplogger.info("The lightcurve model is: " + configs["lc_model"]) - else: - pplogger.info("No lightcurve model is being applied.") - - pplogger.info( - "Output files will be saved in path: " + cmd_args.outpath + " with filestem " + cmd_args.outfilestem - ) - pplogger.info("Output files will be saved as format: " + configs["output_format"]) - pplogger.info( - "In the output, positions will be rounded to " - + str(configs["position_decimals"]) - + " decimal places." - ) - pplogger.info( - "In the output, magnitudes will be rounded to " - + str(configs["magnitude_decimals"]) - + " decimal places." - ) - if isinstance(configs["output_columns"], list): - pplogger.info("The output columns are set to: " + " ".join(configs["output_columns"])) - else: - pplogger.info("The output columns are set to: " + configs["output_columns"]) diff --git a/src/sorcha/sorcha.py b/src/sorcha/sorcha.py index 9aad798f..aaa6adca 100755 --- a/src/sorcha/sorcha.py +++ b/src/sorcha/sorcha.py @@ -20,7 +20,7 @@ from sorcha.modules import PPAddUncertainties, PPRandomizeMeasurements from sorcha.modules import PPVignetting from sorcha.modules.PPFadingFunctionFilter import PPFadingFunctionFilter -from sorcha.modules.PPConfigParser import PPConfigFileParser, PPPrintConfigsToLog, PPFindFileOrExit +from sorcha.modules.PPConfigParser import PPFindFileOrExit from sorcha.modules.PPGetLogger import PPGetLogger from sorcha.modules.PPCommandLineParser import PPCommandLineParser diff --git a/src/sorcha/utilities/diffTestUtils.py b/src/sorcha/utilities/diffTestUtils.py index 1b3906c6..051164f3 100644 --- a/src/sorcha/utilities/diffTestUtils.py +++ b/src/sorcha/utilities/diffTestUtils.py @@ -7,7 +7,6 @@ from sorcha.sorcha import runLSSTSimulation from sorcha.utilities.dataUtilitiesForTests import get_demo_filepath, get_test_filepath from sorcha.utilities.sorchaArguments import sorchaArguments -from sorcha.modules.PPConfigParser import PPConfigFileParser from sorcha.utilities.sorchaConfigs import sorchaConfigs diff --git a/src/sorcha/utilities/sorchaConfigs.py b/src/sorcha/utilities/sorchaConfigs.py index d7ff37d3..c410f411 100644 --- a/src/sorcha/utilities/sorchaConfigs.py +++ b/src/sorcha/utilities/sorchaConfigs.py @@ -782,6 +782,8 @@ def _validate_expert_configs(self): @dataclass class auxiliaryConfigs: + """Data class for holding auxiliary section configuration file keys and validating them.""" + de440s: str = "de440s.bsp" """filename of de440s""" de440s_url: str = "https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de440s.bsp" @@ -1018,6 +1020,7 @@ class sorchaConfigs: auxiliary: auxiliaryConfigs = None """auxiliaryConfigs dataclass which stores the keywords from the AUXILIARY section of the config file.""" + # When adding a new config dataclass or new dataclass config parameters remember to add these to the function PrintConfigsToLog below. pplogger: None = None """The Python logger instance""" @@ -1405,6 +1408,14 @@ def PrintConfigsToLog(sconfigs, cmd_args): pplogger.info("Loading camera footprint from " + sconfigs.fov.footprint_path) else: pplogger.info("Loading default LSST footprint LSST_detector_corners_100123.csv") + if sconfigs.fov.footprint_edge_threshold: + pplogger.info( + "The footprint edge threshold is " + + str(sconfigs.fov.footprint_edge_threshold) + + " arcseconds" + ) + else: + pplogger.info("Default footprint edge threshold used (10px or 2 arcseconds).") elif sconfigs.fov.camera_model == "circle": pplogger.info("Footprint is circular.") if sconfigs.fov.fill_factor: @@ -1476,11 +1487,37 @@ def PrintConfigsToLog(sconfigs, cmd_args): "...the maximum temporal separation between subsequent observations in a tracklet in days is: " + str(sconfigs.linkingfilter.ssp_maximum_time) ) + pplogger.info( + "...the time in UTC at which it is noon at the observatory location (in standard time) is " + + str(sconfigs.linkingfilter.ssp_night_start_utc) + ) if not sconfigs.linkingfilter.drop_unlinked: pplogger.info("Unlinked objects will not be dropped.") else: pplogger.info("Solar System Processing linking filter is turned OFF.") - + pplogger.info("The auxiliary files used for emphemris generation...") + pplogger.info("...the leap second file is: " + str(sconfigs.auxiliary.leap_seconds)) + pplogger.info( + "...the historical Earth orientation specification file is: " + + str(sconfigs.auxiliary.earth_historical) + ) + pplogger.info( + "...the prediction of the Earth's future orientation file is: " + + str(sconfigs.auxiliary.earth_predict) + ) + pplogger.info( + "...the orientation information and physical constants for other bodies file is: " + + str(sconfigs.auxiliary.orientation_constants) + ) + pplogger.info("...the Earth's position for ephemerides file is: " + str(sconfigs.auxiliary.de440s)) + pplogger.info( + "...the regularly updated specification of the Earth's orientation file is: " + + str(sconfigs.auxiliary.earth_high_precision) + ) + pplogger.info( + "...the observatory position information and Minor Planet Center (MPC) observatory codes file is: " + + str(sconfigs.auxiliary.observatory_codes_compressed) + ) if sconfigs.input.ephemerides_type == "ar": pplogger.info("ASSIST+REBOUND Simulation is turned ON.") pplogger.info("For ASSIST+REBOUND...") diff --git a/src/sorcha_cmdline/run.py b/src/sorcha_cmdline/run.py index 67ca1a77..4dc248a4 100644 --- a/src/sorcha_cmdline/run.py +++ b/src/sorcha_cmdline/run.py @@ -12,8 +12,6 @@ def main(): description="Run a simulation.", ) - # parser = SorchaArgparse(parser) - required = parser.add_argument_group("Required arguments") required.add_argument( "-c", @@ -141,7 +139,6 @@ def execute(args): PPFindFileOrExit, PPGetLogger, PPCommandLineParser, - PPConfigFileParser, runLSSTSimulation, sorchaArguments, sorchaConfigs, @@ -163,7 +160,6 @@ def execute(args): # Extract and validate the remaining arguments. cmd_args = PPCommandLineParser(args) pplogger.info("Reading configuration file...") - # configs = PPConfigFileParser(cmd_args["configfile"], cmd_args["surveyname"]) sconfigs = sorchaConfigs(cmd_args["configfile"], cmd_args["surveyname"]) pplogger.info("Configuration file read.") diff --git a/tests/data/test_PPPrintConfigsToLog.txt b/tests/data/test_PPPrintConfigsToLog.txt index 8164a1b5..1589fd71 100644 --- a/tests/data/test_PPPrintConfigsToLog.txt +++ b/tests/data/test_PPPrintConfigsToLog.txt @@ -1,49 +1,229 @@ +# Sorcha Configuration File + + +[INPUT] + +# The simulation used for the ephemeris input. +# Options: "ar", "external" +ephemerides_type = ar + +# Format for ephemeris simulation output file. If reading from an existing temporary ephemeris +# database, this will be ignored. +# Options: csv, whitespace, hdf5 +eph_format = csv + +# SSPP chunks processing by object: how many objects should be processed at once? +size_serial_chunk = 10 + +# Format for orbit/colour/brightness/cometary data files. +# Options: comma, csv or whitespace +aux_format = whitespace + +# SQL query for extracting data from the pointing database. +pointing_sql_query = SELECT observationId, observationStartMJD as observationStartMJD_TAI, visitTime, visitExposureTime, filter, seeingFwhmGeom, seeingFwhmEff, fiveSigmaDepth, fieldRA, fieldDec, rotSkyPos FROM observations order by observationId + + +[FILTERS] + +# Filters of the observations you are interested in, comma-separated. +# Your physical parameters file must have H calculated in one of these filters +# and colour offset columns defined relative to that filter. +observing_filters = r,g,i,z + + +[SATURATION] + +# Upper magnitude limit on sources that will overfill the detector pixels/have +# counts above the non-linearity regime of the pixels where one can’t do +# photometry. Objects brighter than this limit (in magnitude) will be cut. +# Comment out for no saturation limit. +# Two formats are accepted: +# Single float: applies same saturation limit to observations in all filters. +# Comma-separated list of floats: applies saturation limit per filter, in order as +# given in observing_filters keyword. +bright_limit = 16.0 + + +[PHASECURVES] + +# The phase function used to calculate apparent magnitude. The physical parameters input +# file must contain the columns needed to calculate the phase function. +# Options: HG, HG1G2, HG12, linear, none. +phase_function = HG + + +[FOV] + +# Choose between circular or actual camera footprint, including chip gaps. +# Options: circle, footprint. +camera_model = footprint + +# The distance from the edge of a detector (in arcseconds on the focal plane) +# at which we will not correctly extract an object. By default this is 10px or 2 arcseconds. +# Comment out or do not include if not using footprint camera model. +footprint_edge_threshold = 2. + +# Path to camera footprint file. Uncomment to provide a path to the desired camera +# detector configuration file if not using the default built-in LSSTCam detector +# configuration for the actual camera footprint. +# footprint_path= ./data/detectors_corners.csv + + +[FADINGFUNCTION] + +# Detection efficiency fading function on or off. Uses the fading function as outlined in +# Chelsey and Vereš (2017) to remove observations. +fading_function_on = True + +# Width parameter for fading function. Should be greater than zero and less than 0.5. +# Suggested value is 0.1 after Chelsey and Vereš (2017). +fading_function_width = 0.1 + +# Peak efficiency for the fading function, called the 'fill factor' in Chelsey and Veres (2017). +# Suggested value is 1. Do not change this unless you are sure of what you are doing. +fading_function_peak_efficiency = 1. + + +[LINKINGFILTER] +# Remove this section if you do not wish to run the SSP linking filter. + +# SSP detection efficiency. Which fraction of the observations of an object will +# the automated solar system processing pipeline successfully link? Float. +SSP_detection_efficiency = 0.95 + +# Length of tracklets. How many observations of an object during one night are +# required to produce a valid tracklet? +SSP_number_observations = 2 + +# Minimum separation (in arcsec) between two observations of an object required for the linking software to distinguish them as separate and therefore as a valid tracklet. +SSP_separation_threshold = 0.5 + +# Maximum time separation (in days) between subsequent observations in a tracklet. Default is 0.0625 days (90mins). +SSP_maximum_time = 0.0625 + +# Number of tracklets for detection. How many tracklets are required to classify +# an object as detected? Must be an int. +SSP_number_tracklets = 3 + +# The number of tracklets defined above must occur in <= this number of days to +# constitute a complete track/detection. +SSP_track_window = 15 + +# The time in UTC at which it is noon at the observatory location (in standard time). +# For the LSST, 12pm Chile Standard Time is 4pm UTC. +SSP_night_start_utc = 16.0 + + +[SIMULATION] +# Configs for running the ASSIST+REBOUND ephemerides generator. + +# the field of view of our search field, in degrees +ar_ang_fov = 1.8 + +# the buffer zone around the field of view we want to include, in degrees +ar_fov_buffer = 0.2 + +# the "picket" is our imprecise discretization of time that allows us to move progress +# our simulations forward without getting too granular when we don't have to. +# the unit is number of days. +ar_picket = 1 + +# the obscode is the MPC observatory code for the provided telescope. +ar_obs_code = X05 + +# the order of healpix which we will use for the healpy portions of the code. +# the nside is equivalent to 2**ar_healpix_order +ar_healpix_order = 6 + + +[OUTPUT] + +# Output format. +# Options: csv, sqlite3, hdf5 +output_format = csv + +# Controls which columns are in the output files. +# Options are "basic" and "all", which returns all columns. +output_columns = basic + +# Decimal places to which RA and Dec should be rounded to in output. +position_decimals = 7 + +# Decimal places to which all magnitudes should be rounded to in output. +magnitude_decimals = 3 + + +[LIGHTCURVE] + +# The unique name of the lightcurve model to use. Defined in the ``name_id`` method +# of the subclasses of AbstractLightCurve. If not none, the complex physical parameters +# file must be specified at the command line.lc_model = none +lc_model = none + + +[ACTIVITY] + +# The unique name of the actvity model to use. Defined in the ``name_id`` method +# of the subclasses of AbstractCometaryActivity. If not none, a complex physical parameters +# file must be specified at the command line. +comet_activity = none + sorcha.utilities.sorchaArguments INFO the base rng seed is 24601 -sorcha.modules.PPConfigParser INFO The config file used is located at test_PPConfig.ini -sorcha.modules.PPConfigParser INFO The physical parameters file used is located at testcolour.txt -sorcha.modules.PPConfigParser INFO The orbits file used is located at testorb.des -sorcha.modules.PPConfigParser INFO The ephemerides file used is located at ephemtestoutput.txt -sorcha.modules.PPConfigParser INFO The survey selected is: rubin_sim -sorcha.modules.PPConfigParser INFO No cometary activity selected. -sorcha.modules.PPConfigParser INFO Format of ephemerides file is: csv -sorcha.modules.PPConfigParser INFO Format of auxiliary files is: whitespace -sorcha.modules.PPConfigParser INFO Pointing database path is: ./baseline_10klines_2.0.db -sorcha.modules.PPConfigParser INFO Pointing database required query is: SELECT observationId, observationStartMJD as observationStartMJD_TAI, visitTime, visitExposureTime, filter, seeingFwhmGeom, seeingFwhmEff, fiveSigmaDepth, fieldRA, fieldDec, rotSkyPos FROM observations order by observationId -sorcha.modules.PPConfigParser INFO The number of objects processed in a single chunk is: 10 -sorcha.modules.PPConfigParser INFO The main filter in which H is defined is r -sorcha.modules.PPConfigParser INFO The filters included in the post-processing results are r g i z -sorcha.modules.PPConfigParser INFO Thus, the colour indices included in the simulation are g-r i-r z-r -sorcha.modules.PPConfigParser INFO The apparent brightness is calculated using the following phase function model: HG -sorcha.modules.PPConfigParser INFO Computation of trailing losses is switched ON. -sorcha.modules.PPConfigParser INFO Randomization of position and magnitude around uncertainties is switched ON. -sorcha.modules.PPConfigParser INFO Vignetting is switched ON. -sorcha.modules.PPConfigParser INFO Footprint is modelled after the actual camera footprint. -sorcha.modules.PPConfigParser INFO Loading camera footprint from ./detectors_corners.csv -sorcha.modules.PPConfigParser INFO The upper saturation limit(s) is/are: 16.0 -sorcha.modules.PPConfigParser INFO SNR limit is turned OFF. -sorcha.modules.PPConfigParser INFO Default SNR cut is ON. All observations with SNR < 2.0 will be removed. -sorcha.modules.PPConfigParser INFO Magnitude limit is turned OFF. -sorcha.modules.PPConfigParser INFO The detection efficiency fading function is ON. -sorcha.modules.PPConfigParser INFO The width parameter of the fading function has been set to: 0.1 -sorcha.modules.PPConfigParser INFO The peak efficiency of the fading function has been set to: 1.0 -sorcha.modules.PPConfigParser INFO Solar System Processing linking filter is turned ON. -sorcha.modules.PPConfigParser INFO For SSP linking... -sorcha.modules.PPConfigParser INFO ...the fractional detection efficiency is: 0.95 -sorcha.modules.PPConfigParser INFO ...the minimum required number of observations in a tracklet is: 2 -sorcha.modules.PPConfigParser INFO ...the minimum required number of tracklets to form a track is: 3 -sorcha.modules.PPConfigParser INFO ...the maximum window of time in days of tracklets to be contained in to form a track is: 15.0 -sorcha.modules.PPConfigParser INFO ...the minimum angular separation between observations in arcseconds is: 0.5 -sorcha.modules.PPConfigParser INFO ...the maximum temporal separation between subsequent observations in a tracklet in days is: 0.0625 -sorcha.modules.PPConfigParser INFO ASSIST+REBOUND Simulation is turned ON. -sorcha.modules.PPConfigParser INFO For ASSIST+REBOUND... -sorcha.modules.PPConfigParser INFO ...the field's angular FOV is: 1.8 -sorcha.modules.PPConfigParser INFO ...the buffer around the FOV is: 0.2 -sorcha.modules.PPConfigParser INFO ...the picket interval is: 1 -sorcha.modules.PPConfigParser INFO ...the observatory code is: X05 -sorcha.modules.PPConfigParser INFO ...the healpix order is: 6 -sorcha.modules.PPConfigParser INFO No lightcurve model is being applied. -sorcha.modules.PPConfigParser INFO Output files will be saved in path: ./ with filestem testout -sorcha.modules.PPConfigParser INFO Output files will be saved as format: csv -sorcha.modules.PPConfigParser INFO In the output, positions will be rounded to 7 decimal places. -sorcha.modules.PPConfigParser INFO In the output, magnitudes will be rounded to 3 decimal places. -sorcha.modules.PPConfigParser INFO The output columns are set to: basic +sorcha.utilities.sorchaConfigs INFO The config file used is located at test_PPConfig.ini +sorcha.utilities.sorchaConfigs INFO The physical parameters file used is located at testcolour.txt +sorcha.utilities.sorchaConfigs INFO The orbits file used is located at testorb.des +sorcha.utilities.sorchaConfigs INFO The ephemerides file used is located at ephemtestoutput.txt +sorcha.utilities.sorchaConfigs INFO The survey selected is: rubin_sim +sorcha.utilities.sorchaConfigs INFO No cometary activity selected. +sorcha.utilities.sorchaConfigs INFO Format of ephemerides file is: csv +sorcha.utilities.sorchaConfigs INFO Format of auxiliary files is: whitespace +sorcha.utilities.sorchaConfigs INFO Pointing database path is: ./baseline_10klines_2.0.db +sorcha.utilities.sorchaConfigs INFO Pointing database required query is: SELECT observationId, observationStartMJD as observationStartMJD_TAI, visitTime, visitExposureTime, filter, seeingFwhmGeom, seeingFwhmEff, fiveSigmaDepth, fieldRA, fieldDec, rotSkyPos FROM observations order by observationId +sorcha.utilities.sorchaConfigs INFO The number of objects processed in a single chunk is: 10 +sorcha.utilities.sorchaConfigs INFO The main filter in which H is defined is r +sorcha.utilities.sorchaConfigs INFO The filters included in the post-processing results are r g i z +sorcha.utilities.sorchaConfigs INFO Thus, the colour indices included in the simulation are g-r i-r z-r +sorcha.utilities.sorchaConfigs INFO The apparent brightness is calculated using the following phase function model: HG +sorcha.utilities.sorchaConfigs INFO Computation of trailing losses is switched ON. +sorcha.utilities.sorchaConfigs INFO Randomization of position and magnitude around uncertainties is switched ON. +sorcha.utilities.sorchaConfigs INFO Vignetting is switched ON. +sorcha.utilities.sorchaConfigs INFO Footprint is modelled after the actual camera footprint. +sorcha.utilities.sorchaConfigs INFO Loading default LSST footprint LSST_detector_corners_100123.csv +sorcha.utilities.sorchaConfigs INFO The footprint edge threshold is 2.0 arcseconds +sorcha.utilities.sorchaConfigs INFO The upper saturation limit(s) is/are: 16.0 +sorcha.utilities.sorchaConfigs INFO SNR limit is turned OFF. +sorcha.utilities.sorchaConfigs INFO Default SNR cut is ON. All observations with SNR < 2.0 will be removed. +sorcha.utilities.sorchaConfigs INFO Magnitude limit is turned OFF. +sorcha.utilities.sorchaConfigs INFO The detection efficiency fading function is ON. +sorcha.utilities.sorchaConfigs INFO The width parameter of the fading function has been set to: 0.1 +sorcha.utilities.sorchaConfigs INFO The peak efficiency of the fading function has been set to: 1.0 +sorcha.utilities.sorchaConfigs INFO Solar System Processing linking filter is turned ON. +sorcha.utilities.sorchaConfigs INFO For SSP linking... +sorcha.utilities.sorchaConfigs INFO ...the fractional detection efficiency is: 0.95 +sorcha.utilities.sorchaConfigs INFO ...the minimum required number of observations in a tracklet is: 2 +sorcha.utilities.sorchaConfigs INFO ...the minimum required number of tracklets to form a track is: 3 +sorcha.utilities.sorchaConfigs INFO ...the maximum window of time in days of tracklets to be contained in to form a track is: 15 +sorcha.utilities.sorchaConfigs INFO ...the minimum angular separation between observations in arcseconds is: 0.5 +sorcha.utilities.sorchaConfigs INFO ...the maximum temporal separation between subsequent observations in a tracklet in days is: 0.0625 +sorcha.utilities.sorchaConfigs INFO ...the time in UTC at which it is noon at the observatory location (in standard time) is 16.0 +sorcha.utilities.sorchaConfigs INFO The auxiliary files used for emphemris generation... +sorcha.utilities.sorchaConfigs INFO ...the leap second file is: naif0012.tls +sorcha.utilities.sorchaConfigs INFO ...the historical Earth orientation specification file is: earth_620120_240827.bpc +sorcha.utilities.sorchaConfigs INFO ...the prediction of the Earth's future orientation file is: earth_200101_990827_predict.bpc +sorcha.utilities.sorchaConfigs INFO ...the orientation information and physical constants for other bodies file is: pck00010.pck +sorcha.utilities.sorchaConfigs INFO ...the Earth's position for ephemerides file is: de440s.bsp +sorcha.utilities.sorchaConfigs INFO ...the regularly updated specification of the Earth's orientation file is: earth_latest_high_prec.bpc +sorcha.utilities.sorchaConfigs INFO ...the observatory position information and Minor Planet Center (MPC) observatory codes file is: ObsCodes.json.gz +sorcha.utilities.sorchaConfigs INFO ASSIST+REBOUND Simulation is turned ON. +sorcha.utilities.sorchaConfigs INFO For ASSIST+REBOUND... +sorcha.utilities.sorchaConfigs INFO ...the field's angular FOV is: 1.8 +sorcha.utilities.sorchaConfigs INFO ...the buffer around the FOV is: 0.2 +sorcha.utilities.sorchaConfigs INFO ...the picket interval is: 1 +sorcha.utilities.sorchaConfigs INFO ...the observatory code is: X05 +sorcha.utilities.sorchaConfigs INFO ...the healpix order is: 6 +sorcha.utilities.sorchaConfigs INFO No lightcurve model is being applied. +sorcha.utilities.sorchaConfigs INFO Output files will be saved in path: ./ with filestem testout +sorcha.utilities.sorchaConfigs INFO Output files will be saved as format: csv +sorcha.utilities.sorchaConfigs INFO In the output, positions will be rounded to 7.0 decimal places. +sorcha.utilities.sorchaConfigs INFO In the output, magnitudes will be rounded to 3.0 decimal places. +sorcha.utilities.sorchaConfigs INFO The output columns are set to: basic diff --git a/tests/ephemeris/test_ephemeris_generation.py b/tests/ephemeris/test_ephemeris_generation.py index 87c0cae7..35bbc837 100644 --- a/tests/ephemeris/test_ephemeris_generation.py +++ b/tests/ephemeris/test_ephemeris_generation.py @@ -4,7 +4,6 @@ import re from sorcha.utilities.dataUtilitiesForTests import get_test_filepath, get_demo_filepath -from sorcha.modules.PPConfigParser import PPConfigFileParser from sorcha.modules.PPGetLogger import PPGetLogger from sorcha.utilities.sorchaArguments import sorchaArguments from sorcha.ephemeris.simulation_driver import create_ephemeris, write_out_ephemeris_file diff --git a/tests/ephemeris/test_pixdict.py b/tests/ephemeris/test_pixdict.py index ffe036f1..e709660d 100644 --- a/tests/ephemeris/test_pixdict.py +++ b/tests/ephemeris/test_pixdict.py @@ -1,7 +1,6 @@ import numpy as np import pandas as pd from sorcha.utilities.dataUtilitiesForTests import get_test_filepath, get_demo_filepath -from sorcha.modules.PPConfigParser import PPConfigFileParser from sorcha.modules.PPGetLogger import PPGetLogger from sorcha.utilities.sorchaArguments import sorchaArguments from sorcha.modules.PPReadPointingDatabase import PPReadPointingDatabase diff --git a/tests/sorcha/test_PPConfigParser.py b/tests/sorcha/test_PPConfigParser.py index 2b256225..a36bdc9e 100644 --- a/tests/sorcha/test_PPConfigParser.py +++ b/tests/sorcha/test_PPConfigParser.py @@ -8,188 +8,6 @@ from sorcha.utilities.dataUtilitiesForTests import get_test_filepath from sorcha.utilities.sorchaArguments import sorchaArguments - -@pytest.fixture -def setup_and_teardown_for_PPConfigFileParser(tmp_path): - # Record initial working directory - initial_wd = os.getcwd() - - # Copy files mentioned in config file into the temp directory - shutil.copy(get_test_filepath("baseline_10klines_2.0.db"), tmp_path) - shutil.copy(get_test_filepath("detectors_corners.csv"), tmp_path) - - # Move to the temp directory - os.chdir(tmp_path) - - # Yield to pytest to run the test - yield - - # After running the test, move back to initial working directory - os.chdir(initial_wd) - - -def test_PPConfigFileParser(setup_and_teardown_for_PPConfigFileParser): - from sorcha.modules.PPConfigParser import PPConfigFileParser - - configs = PPConfigFileParser(get_test_filepath("test_PPConfig.ini"), "rubin_sim") - - test_configs = { - "eph_format": "csv", - "aux_format": "whitespace", - "ephemerides_type": "ar", - "pointing_sql_query": "SELECT observationId, observationStartMJD as observationStartMJD_TAI, visitTime, visitExposureTime, filter, seeingFwhmGeom, seeingFwhmEff, fiveSigmaDepth, fieldRA, fieldDec, rotSkyPos FROM observations order by observationId", - "comet_activity": None, - "observing_filters": ["r", "g", "i", "z"], - "phase_function": "HG", - "trailing_losses_on": True, - "camera_model": "footprint", - "footprint_path": None, - "footprint_edge_threshold": 2.0, - "bright_limit": 16.0, - "bright_limit_on": True, - "SNR_limit": None, - "SNR_limit_on": False, - "default_SNR_cut": True, - "mag_limit": None, - "mag_limit_on": False, - "fading_function_on": True, - "fading_function_width": 0.1, - "fading_function_peak_efficiency": 1.0, - "SSP_separation_threshold": 0.5, - "SSP_number_observations": 2, - "SSP_number_tracklets": 3, - "SSP_track_window": 15.0, - "SSP_detection_efficiency": 0.95, - "SSP_maximum_time": 0.0625, - "SSP_night_start_utc": 16.0, - "SSP_linking_on": True, - "drop_unlinked": True, - "ar_ang_fov": 1.8, - "ar_fov_buffer": 0.2, - "ar_picket": 1, - "ar_obs_code": "X05", - "ar_healpix_order": 6, - "output_format": "csv", - "output_columns": "basic", - "position_decimals": 7, - "magnitude_decimals": 3, - "size_serial_chunk": 10, - "lc_model": None, - "randomization_on": True, - "vignetting_on": True, - } - - assert configs == test_configs - - return - - -def test_PPGetOrExit(): - from sorcha.modules.PPConfigParser import PPGetOrExit - - config = configparser.ConfigParser() - config.read(get_test_filepath("test_PPConfig.ini")) - - test_value = PPGetOrExit(config, "INPUT", "eph_format", "none") - - with pytest.raises(SystemExit) as e: - PPGetOrExit(config, "INPUT", "veryFakeKey", "this key does not exist!") - - assert test_value == "csv" - assert e.type == SystemExit - assert e.value.code == "this key does not exist!" - - return - - -def test_PPGetFloatOrExit(): - from sorcha.modules.PPConfigParser import PPGetFloatOrExit - - config = configparser.ConfigParser() - config.read(get_test_filepath("test_PPConfig.ini")) - - test_value = PPGetFloatOrExit(config, "FADINGFUNCTION", "fading_function_width", "none") - - with pytest.raises(SystemExit) as e: - PPGetFloatOrExit(config, "FADINGFUNCTION", "fading_function_on", "none") - - assert test_value == 0.1 - assert isinstance(test_value, float) - assert e.type == SystemExit - assert ( - e.value.code - == "ERROR: expected a float for config parameter fading_function_on. Check value in config file." - ) - - return - - -def test_PPGetIntOrExit(): - from sorcha.modules.PPConfigParser import PPGetIntOrExit - - config = configparser.ConfigParser() - config.read(get_test_filepath("test_PPConfig.ini")) - - test_value = PPGetIntOrExit(config, "OUTPUT", "position_decimals", "none") - - with pytest.raises(SystemExit) as e: - PPGetIntOrExit(config, "FADINGFUNCTION", "fading_function_on", "none") - - assert test_value == 7 - assert isinstance(test_value, int) - assert e.type == SystemExit - assert ( - e.value.code - == "ERROR: expected an int for config parameter fading_function_on. Check value in config file." - ) - - return - - -def test_PPGetBoolOrExit(): - from sorcha.modules.PPConfigParser import PPGetBoolOrExit - - config = configparser.ConfigParser() - config.read(get_test_filepath("test_PPConfig.ini")) - - test_value = PPGetBoolOrExit(config, "FADINGFUNCTION", "fading_function_on", "none") - - with pytest.raises(SystemExit) as e: - PPGetBoolOrExit(config, "OUTPUT", "position_decimals", "none") - - assert test_value is True - assert isinstance(test_value, bool) - assert e.type == SystemExit - assert e.value.code == "ERROR: position_decimals could not be converted to a Boolean." - - return - - -def test_PPGetValueAndFlag(): - from sorcha.modules.PPConfigParser import PPGetValueAndFlag - - config = configparser.ConfigParser() - config.read(get_test_filepath("test_PPConfig.ini")) - - test_value_1, test_flag_1 = PPGetValueAndFlag(config, "FOV", "fill_factor", "float") - test_value_2, test_flag_2 = PPGetValueAndFlag(config, "SATURATION", "bright_limit", "float") - - with pytest.raises(SystemExit) as e: - PPGetValueAndFlag(config, "SATURATION", "bright_limit", "int") - - assert test_value_1 is None - assert test_flag_1 is False - assert test_value_2 == 16.0 - assert test_flag_2 is True - assert e.type == SystemExit - assert ( - e.value.code - == "ERROR: expected an int for config parameter bright_limit. Check value in config file." - ) - - return - - def test_PPFindFileOrExit(): from sorcha.modules.PPConfigParser import PPFindFileOrExit @@ -204,7 +22,6 @@ def test_PPFindFileOrExit(): return - def test_PPFindDirectoryOrExit(): from sorcha.modules.PPConfigParser import PPFindDirectoryOrExit @@ -218,103 +35,3 @@ def test_PPFindDirectoryOrExit(): assert e.value.code == "ERROR: filepath ./fake_dir/ supplied for test argument does not exist." return - - -def test_PPCheckFiltersForSurvey(): - from sorcha.modules.PPConfigParser import PPCheckFiltersForSurvey - - PPCheckFiltersForSurvey("rubin_sim", ["u", "g", "r", "i", "z", "y"]) - - with pytest.raises(SystemExit) as e: - PPCheckFiltersForSurvey("rubin_sim", ["j"]) - - assert e.type == SystemExit - - -def test_PPPrintConfigsToLog(tmp_path): - from sorcha.modules.PPGetLogger import PPGetLogger - from sorcha.modules.PPConfigParser import PPPrintConfigsToLog - - test_path = os.path.dirname(get_test_filepath("test_input_fullobs.csv")) - - pplogger = PPGetLogger(tmp_path, "test_log", log_format="%(name)-12s %(levelname)-8s %(message)s ") - - cmd_args = { - "paramsinput": "testcolour.txt", - "orbinfile": "testorb.des", - "input_ephemeris_file": "ephemtestoutput.txt", - "configfile": "test_PPConfig.ini", - "pointing_database": "./baseline_10klines_2.0.db", - "outpath": "./", - "surveyname": "rubin_sim", - "outfilestem": "testout", - "loglevel": True, - "seed": 24601, - "stats": None, - } - - args = sorchaArguments(cmd_args) - - configs = { - "eph_format": "csv", - "aux_format": "whitespace", - "ephemerides_type": "ar", - "pointing_sql_query": "SELECT observationId, observationStartMJD as observationStartMJD_TAI, visitTime, visitExposureTime, filter, seeingFwhmGeom, seeingFwhmEff, fiveSigmaDepth, fieldRA, fieldDec, rotSkyPos FROM observations order by observationId", - "comet_activity": "none", - "observing_filters": ["r", "g", "i", "z"], - "phase_function": "HG", - "trailing_losses_on": True, - "camera_model": "footprint", - "footprint_path": "./detectors_corners.csv", - "footprint_edge_threshold": 2.0, - "bright_limit": 16.0, - "bright_limit_on": True, - "SNR_limit": None, - "SNR_limit_on": False, - "default_SNR_cut": True, - "mag_limit": None, - "mag_limit_on": False, - "fading_function_on": True, - "fading_function_width": 0.1, - "fading_function_peak_efficiency": 1.0, - "SSP_separation_threshold": 0.5, - "SSP_number_observations": 2, - "SSP_number_tracklets": 3, - "SSP_track_window": 15.0, - "SSP_detection_efficiency": 0.95, - "SSP_maximum_time": 0.0625, - "SSP_linking_on": True, - "drop_unlinked": True, - "ar_ang_fov": 1.8, - "ar_fov_buffer": 0.2, - "ar_picket": 1, - "ar_obs_code": "X05", - "ar_healpix_order": 6, - "output_format": "csv", - "output_columns": "basic", - "position_decimals": 7, - "magnitude_decimals": 3, - "size_serial_chunk": 10, - "mainfilter": "r", - "othercolours": ["g-r", "i-r", "z-r"], - "lc_model": None, - "randomization_on": True, - "vignetting_on": True, - } - - PPPrintConfigsToLog(configs, args) - - datalog = glob.glob(os.path.join(tmp_path, "*-sorcha.log")) - - testfile = open(os.path.join(test_path, "test_PPPrintConfigsToLog.txt"), mode="r") - newfile = open(datalog[0], mode="r") - - alltest = testfile.readlines() - allnew = newfile.readlines() - - assert alltest == allnew - - testfile.close() - newfile.close() - - return diff --git a/tests/sorcha/test_sorchaConfigs.py b/tests/sorcha/test_sorchaConfigs.py index 7007582d..a1711992 100644 --- a/tests/sorcha/test_sorchaConfigs.py +++ b/tests/sorcha/test_sorchaConfigs.py @@ -1,5 +1,5 @@ import pytest - +from sorcha.utilities.sorchaArguments import sorchaArguments from sorcha.utilities.dataUtilitiesForTests import get_demo_filepath from sorcha.lightcurves.lightcurve_registration import LC_METHODS from sorcha.activity.activity_registration import CA_METHODS @@ -1056,3 +1056,51 @@ def test_auxiliary_config_making_url_none(file): test_configs = auxiliaryConfigs(**aux_configs) assert getattr(test_configs, file + "_url") == None + + + + +def test_PrintConfigsToLog(tmp_path): + from sorcha.modules.PPGetLogger import PPGetLogger + from sorcha.utilities.sorchaConfigs import PrintConfigsToLog + from sorcha.utilities.dataUtilitiesForTests import get_test_filepath + import os + import glob + + test_path = os.path.dirname(get_test_filepath("test_input_fullobs.csv")) + config_file_location = get_test_filepath("test_PPConfig.ini") + pplogger = PPGetLogger(tmp_path, "test_log", log_format="%(name)-12s %(levelname)-8s %(message)s ") + + cmd_args = { + "paramsinput": "testcolour.txt", + "orbinfile": "testorb.des", + "input_ephemeris_file": "ephemtestoutput.txt", + "configfile": "test_PPConfig.ini", + "pointing_database": "./baseline_10klines_2.0.db", + "outpath": "./", + "surveyname": "rubin_sim", + "outfilestem": "testout", + "loglevel": True, + "seed": 24601, + "stats": None, + } + test_configs = sorchaConfigs(config_file_location, "rubin_sim") + test_configs.filters.mainfilter = "r" + test_configs.filters.othercolours = ["g-r", "i-r", "z-r"] + args = sorchaArguments(cmd_args) + + PrintConfigsToLog(test_configs, args) + + datalog = glob.glob(os.path.join(tmp_path, "*-sorcha.log")) + #when updating PrintConfigsToLog text file test_PPPrintConfigsToLog.txt needs to be updated too. + testfile = open(os.path.join(test_path, "test_PPPrintConfigsToLog.txt"), mode="r") + newfile = open(datalog[0], mode="r") + alltest = testfile.readlines() + allnew = newfile.readlines() + allnew_ = allnew[1:] #skipping first line as that line specifies user file location + assert alltest == allnew_ + + testfile.close() + newfile.close() + + return \ No newline at end of file From 9e9cb223420aeba7deb87525048e998df61edbf5 Mon Sep 17 00:00:00 2001 From: Ryan Lyttle Date: Tue, 7 Jan 2025 12:25:26 +0000 Subject: [PATCH 2/5] Added the rest of auxiliaryConfig parameters to log --- src/sorcha/utilities/sorchaConfigs.py | 11 +++++++++++ tests/data/test_PPPrintConfigsToLog.txt | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/sorcha/utilities/sorchaConfigs.py b/src/sorcha/utilities/sorchaConfigs.py index c410f411..c85c418e 100644 --- a/src/sorcha/utilities/sorchaConfigs.py +++ b/src/sorcha/utilities/sorchaConfigs.py @@ -1516,8 +1516,19 @@ def PrintConfigsToLog(sconfigs, cmd_args): ) pplogger.info( "...the observatory position information and Minor Planet Center (MPC) observatory codes file is: " + + str(sconfigs.auxiliary.observatory_codes) + + " and compressed file is: " + str(sconfigs.auxiliary.observatory_codes_compressed) ) + pplogger.info( + "...the ephemerides for solar-system planets from JPL's Horizon system file is: " + + str(sconfigs.auxiliary.jpl_planets) + ) + pplogger.info( + "...the ephemerides for solar-system small bodies from JPL's Horizon system file is: " + + str(sconfigs.auxiliary.jpl_small_bodies) + ) + pplogger.info("...the meta kernal file is : " + str(sconfigs.auxiliary.meta_kernel)) if sconfigs.input.ephemerides_type == "ar": pplogger.info("ASSIST+REBOUND Simulation is turned ON.") pplogger.info("For ASSIST+REBOUND...") diff --git a/tests/data/test_PPPrintConfigsToLog.txt b/tests/data/test_PPPrintConfigsToLog.txt index 1589fd71..94d3901b 100644 --- a/tests/data/test_PPPrintConfigsToLog.txt +++ b/tests/data/test_PPPrintConfigsToLog.txt @@ -213,7 +213,10 @@ sorcha.utilities.sorchaConfigs INFO ...the prediction of the Earth's future sorcha.utilities.sorchaConfigs INFO ...the orientation information and physical constants for other bodies file is: pck00010.pck sorcha.utilities.sorchaConfigs INFO ...the Earth's position for ephemerides file is: de440s.bsp sorcha.utilities.sorchaConfigs INFO ...the regularly updated specification of the Earth's orientation file is: earth_latest_high_prec.bpc -sorcha.utilities.sorchaConfigs INFO ...the observatory position information and Minor Planet Center (MPC) observatory codes file is: ObsCodes.json.gz +sorcha.utilities.sorchaConfigs INFO ...the observatory position information and Minor Planet Center (MPC) observatory codes file is: ObsCodes.json and compressed file is: ObsCodes.json.gz +sorcha.utilities.sorchaConfigs INFO ...the ephemerides for solar-system planets from JPL's Horizon system file is: linux_p1550p2650.440 +sorcha.utilities.sorchaConfigs INFO ...the ephemerides for solar-system small bodies from JPL's Horizon system file is: sb441-n16.bsp +sorcha.utilities.sorchaConfigs INFO ...the meta kernal file is : meta_kernel.txt sorcha.utilities.sorchaConfigs INFO ASSIST+REBOUND Simulation is turned ON. sorcha.utilities.sorchaConfigs INFO For ASSIST+REBOUND... sorcha.utilities.sorchaConfigs INFO ...the field's angular FOV is: 1.8 From 752103c6b246ebb05e905886edea0613446bace2 Mon Sep 17 00:00:00 2001 From: Ryan Lyttle Date: Tue, 7 Jan 2025 13:28:25 +0000 Subject: [PATCH 3/5] Added better statement for log --- src/sorcha/utilities/sorchaConfigs.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/sorcha/utilities/sorchaConfigs.py b/src/sorcha/utilities/sorchaConfigs.py index c85c418e..170fc26d 100644 --- a/src/sorcha/utilities/sorchaConfigs.py +++ b/src/sorcha/utilities/sorchaConfigs.py @@ -1550,16 +1550,22 @@ def PrintConfigsToLog(sconfigs, cmd_args): "Output files will be saved in path: " + cmd_args.outpath + " with filestem " + cmd_args.outfilestem ) pplogger.info("Output files will be saved as format: " + sconfigs.output.output_format) - pplogger.info( - "In the output, positions will be rounded to " - + str(sconfigs.output.position_decimals) - + " decimal places." - ) - pplogger.info( - "In the output, magnitudes will be rounded to " - + str(sconfigs.output.magnitude_decimals) - + " decimal places." - ) + if sconfigs.output.position_decimals: + pplogger.info( + "In the output, positions will be rounded to " + + str(sconfigs.output.position_decimals) + + " decimal places." + ) + else: + pplogger.info("In the output, positions will not be rounded") + if sconfigs.output.magnitude_decimals: + pplogger.info( + "In the output, magnitudes will be rounded to " + + str(sconfigs.output.magnitude_decimals) + + " decimal places." + ) + else: + pplogger.info("In the output, magnitudes will not be rounded") if isinstance(sconfigs.output.output_columns, list): pplogger.info("The output columns are set to: " + " ".join(sconfigs.output.output_columns)) else: From 81cfdd6bb1625cefaf13936e5116def1168187a9 Mon Sep 17 00:00:00 2001 From: Ryan Lyttle Date: Wed, 8 Jan 2025 17:32:55 +0000 Subject: [PATCH 4/5] removed PPConfigParser and moved remaining functions to new file Removed PPConfigParser and moved the two functions in it into a new file called Find_File_or_Directory.py in the utilities folder. Also moved the unit tests for these functions into test_Find_File_or_Directory.py --- src/sorcha/modules/PPCommandLineParser.py | 16 ++++++++-------- src/sorcha/modules/__init__.py | 1 - src/sorcha/sorcha.py | 2 +- .../Find_File_or_Directory.py} | 4 ++-- src/sorcha/utilities/sorchaConfigs.py | 4 ++-- src/sorcha/utilities/sorcha_copy_configs.py | 4 ++-- src/sorcha/utilities/sorcha_copy_demo_files.py | 4 ++-- src/sorcha_cmdline/outputs.py | 12 ++++++------ src/sorcha_cmdline/run.py | 4 ++-- ...gParser.py => test_Find_File_or_DIrectory.py} | 16 ++++++++-------- 10 files changed, 33 insertions(+), 34 deletions(-) rename src/sorcha/{modules/PPConfigParser.py => utilities/Find_File_or_Directory.py} (94%) rename tests/sorcha/{test_PPConfigParser.py => test_Find_File_or_DIrectory.py} (60%) diff --git a/src/sorcha/modules/PPCommandLineParser.py b/src/sorcha/modules/PPCommandLineParser.py index 9d34b09b..ff0f66a8 100644 --- a/src/sorcha/modules/PPCommandLineParser.py +++ b/src/sorcha/modules/PPCommandLineParser.py @@ -2,7 +2,7 @@ import sys import logging import glob -from .PPConfigParser import PPFindFileOrExit, PPFindDirectoryOrExit +from ..utilities.Find_File_or_Directory import FindFileOrExit, FindDirectoryOrExit def warn_or_remove_file(filepath, force_remove, pplogger): @@ -58,14 +58,14 @@ def PPCommandLineParser(args): cmd_args_dict = {} - cmd_args_dict["paramsinput"] = PPFindFileOrExit(args.p, "-p, --params") - cmd_args_dict["orbinfile"] = PPFindFileOrExit(args.ob, "-ob, --orbit") - cmd_args_dict["configfile"] = PPFindFileOrExit(args.c, "-c, --config") - cmd_args_dict["outpath"] = PPFindFileOrExit(args.o, "-o, --outfile") - cmd_args_dict["pointing_database"] = PPFindFileOrExit(args.pd, "-pd, --pointing_database") + cmd_args_dict["paramsinput"] = FindFileOrExit(args.p, "-p, --params") + cmd_args_dict["orbinfile"] = FindFileOrExit(args.ob, "-ob, --orbit") + cmd_args_dict["configfile"] = FindFileOrExit(args.c, "-c, --config") + cmd_args_dict["outpath"] = FindFileOrExit(args.o, "-o, --outfile") + cmd_args_dict["pointing_database"] = FindFileOrExit(args.pd, "-pd, --pointing_database") if args.cp: - cmd_args_dict["complex_physical_parameters"] = PPFindFileOrExit( + cmd_args_dict["complex_physical_parameters"] = FindFileOrExit( args.cp, "-cp, --complex_physical_parameters" ) @@ -95,7 +95,7 @@ def PPCommandLineParser(args): cmd_args_dict["ar_data_path"] = args.ar # default value for args.ar is `None`. if cmd_args_dict["ar_data_path"]: - PPFindDirectoryOrExit(cmd_args_dict["ar_data_path"], "-ar, --ar_data_path") + FindDirectoryOrExit(cmd_args_dict["ar_data_path"], "-ar, --ar_data_path") warn_or_remove_file( os.path.join(cmd_args_dict["outpath"], cmd_args_dict["outfilestem"] + ".*"), args.f, pplogger diff --git a/src/sorcha/modules/__init__.py b/src/sorcha/modules/__init__.py index 887d418f..c55ea3b0 100644 --- a/src/sorcha/modules/__init__.py +++ b/src/sorcha/modules/__init__.py @@ -14,7 +14,6 @@ from . import PPRandomizeMeasurements from . import PPTrailingLoss from . import PPVignetting -from . import PPConfigParser from . import PPCommandLineParser from . import PPGetLogger from . import PPApplyFOVFilter diff --git a/src/sorcha/sorcha.py b/src/sorcha/sorcha.py index aaa6adca..560b2f72 100755 --- a/src/sorcha/sorcha.py +++ b/src/sorcha/sorcha.py @@ -20,7 +20,7 @@ from sorcha.modules import PPAddUncertainties, PPRandomizeMeasurements from sorcha.modules import PPVignetting from sorcha.modules.PPFadingFunctionFilter import PPFadingFunctionFilter -from sorcha.modules.PPConfigParser import PPFindFileOrExit +from sorcha.utilities.Find_File_or_Directory import FindFileOrExit from sorcha.modules.PPGetLogger import PPGetLogger from sorcha.modules.PPCommandLineParser import PPCommandLineParser diff --git a/src/sorcha/modules/PPConfigParser.py b/src/sorcha/utilities/Find_File_or_Directory.py similarity index 94% rename from src/sorcha/modules/PPConfigParser.py rename to src/sorcha/utilities/Find_File_or_Directory.py index c586ed7e..8336791f 100755 --- a/src/sorcha/modules/PPConfigParser.py +++ b/src/sorcha/utilities/Find_File_or_Directory.py @@ -3,7 +3,7 @@ import sys -def PPFindFileOrExit(arg_fn, argname): +def FindFileOrExit(arg_fn, argname): """Checks to see if a file given by a filename exists. If it doesn't, this fails gracefully and exits to the command line. @@ -31,7 +31,7 @@ def PPFindFileOrExit(arg_fn, argname): sys.exit("ERROR: filename {} supplied for {} argument does not exist.".format(arg_fn, argname)) -def PPFindDirectoryOrExit(arg_fn, argname): +def FindDirectoryOrExit(arg_fn, argname): """Checks to see if a directory given by a filepath exists. If it doesn't, this fails gracefully and exits to the command line. diff --git a/src/sorcha/utilities/sorchaConfigs.py b/src/sorcha/utilities/sorchaConfigs.py index 170fc26d..f68ad1e2 100644 --- a/src/sorcha/utilities/sorchaConfigs.py +++ b/src/sorcha/utilities/sorchaConfigs.py @@ -331,7 +331,7 @@ def _camera_footprint(self): None """ if self.footprint_path is not None: - PPFindFileOrExit(self.footprint_path, "footprint_path") + FindFileOrExit(self.footprint_path, "footprint_path") elif self.survey_name.lower() not in ["lsst", "rubin_sim"]: logging.error( "ERROR: a default detector footprint is currently only provided for LSST; please provide your own footprint file." @@ -1267,7 +1267,7 @@ def check_value_in_list(value, valuelist, key): ) -def PPFindFileOrExit(arg_fn, argname): +def FindFileOrExit(arg_fn, argname): """Checks to see if a file given by a filename exists. If it doesn't, this fails gracefully and exits to the command line. diff --git a/src/sorcha/utilities/sorcha_copy_configs.py b/src/sorcha/utilities/sorcha_copy_configs.py index 38616959..402a6340 100644 --- a/src/sorcha/utilities/sorcha_copy_configs.py +++ b/src/sorcha/utilities/sorcha_copy_configs.py @@ -5,7 +5,7 @@ import sys from importlib.resources import files -from sorcha.modules.PPConfigParser import PPFindDirectoryOrExit +from sorcha.utilities.Find_File_or_Directory import FindDirectoryOrExit def copy_demo_configs(copy_location, which_configs, force_overwrite): @@ -29,7 +29,7 @@ def copy_demo_configs(copy_location, which_configs, force_overwrite): """ - _ = PPFindDirectoryOrExit(copy_location, "filepath") + _ = FindDirectoryOrExit(copy_location, "filepath") config_data_root = files("sorcha.data.survey_setups") diff --git a/src/sorcha/utilities/sorcha_copy_demo_files.py b/src/sorcha/utilities/sorcha_copy_demo_files.py index e2b8e897..3ed1f8a0 100644 --- a/src/sorcha/utilities/sorcha_copy_demo_files.py +++ b/src/sorcha/utilities/sorcha_copy_demo_files.py @@ -5,7 +5,7 @@ import sys from importlib.resources import files -from sorcha.modules.PPConfigParser import PPFindDirectoryOrExit +from sorcha.utilities.Find_File_or_Directory import FindDirectoryOrExit from sorcha.utilities.sorcha_demo_command import print_demo_command @@ -27,7 +27,7 @@ def copy_demo_files(copy_location, force_overwrite): """ - _ = PPFindDirectoryOrExit(copy_location, "filepath") + _ = FindDirectoryOrExit(copy_location, "filepath") demo_data_root = files("sorcha.data.demo") diff --git a/src/sorcha_cmdline/outputs.py b/src/sorcha_cmdline/outputs.py index 4483f089..efce9a57 100644 --- a/src/sorcha_cmdline/outputs.py +++ b/src/sorcha_cmdline/outputs.py @@ -16,15 +16,15 @@ def cmd_outputs_create_sqlite(args): # pragma: no cover # is poor user experience. # from sorcha.utilities.createResultsSQLDatabase import create_results_database - from sorcha.modules.PPConfigParser import PPFindDirectoryOrExit + from sorcha.utilities.Find_File_or_Directory import FindDirectoryOrExit import os args.output = os.path.abspath(args.output) args.inputs = os.path.abspath(args.inputs) args.results = os.path.abspath(args.results) - _ = PPFindDirectoryOrExit(args.inputs, "-i, --inputs") - _ = PPFindDirectoryOrExit(args.results, "-r, --results") + _ = FindDirectoryOrExit(args.inputs, "-i, --inputs") + _ = FindDirectoryOrExit(args.results, "-r, --results") return create_results_database(args) @@ -36,15 +36,15 @@ def cmd_outputs_create_sqlite(args): # pragma: no cover def cmd_outputs_check_logs(args): # pragma: no cover from sorcha.utilities.check_output_logs import check_output_logs - from sorcha.modules.PPConfigParser import PPFindDirectoryOrExit + from sorcha.utilities.Find_File_or_Directory import FindDirectoryOrExit import os args.filepath = os.path.abspath(args.filepath) - _ = PPFindDirectoryOrExit(args.filepath, "-f, --filepath") + _ = FindDirectoryOrExit(args.filepath, "-f, --filepath") if args.outpath: args.outpath = os.path.abspath(args.outpath) - _ = PPFindDirectoryOrExit(os.path.dirname(args.outpath), "-o, --outpath") + _ = FindDirectoryOrExit(os.path.dirname(args.outpath), "-o, --outpath") if os.path.exists(args.outpath) and not args.force: print( diff --git a/src/sorcha_cmdline/run.py b/src/sorcha_cmdline/run.py index 4dc248a4..9b3c4634 100644 --- a/src/sorcha_cmdline/run.py +++ b/src/sorcha_cmdline/run.py @@ -136,7 +136,7 @@ def execute(args): # is poor user experience. # from sorcha.sorcha import ( - PPFindFileOrExit, + FindFileOrExit, PPGetLogger, PPCommandLineParser, runLSSTSimulation, @@ -148,7 +148,7 @@ def execute(args): import sys, os # Extract the output file path now in order to set up logging. - outpath = PPFindFileOrExit(args.o, "-o, --outfile") + outpath = FindFileOrExit(args.o, "-o, --outfile") pplogger = PPGetLogger(outpath, args.t) pplogger.info("Sorcha Start (Main)") pplogger.info(f"Command line: {' '.join(sys.argv)}") diff --git a/tests/sorcha/test_PPConfigParser.py b/tests/sorcha/test_Find_File_or_DIrectory.py similarity index 60% rename from tests/sorcha/test_PPConfigParser.py rename to tests/sorcha/test_Find_File_or_DIrectory.py index a36bdc9e..bfa15090 100644 --- a/tests/sorcha/test_PPConfigParser.py +++ b/tests/sorcha/test_Find_File_or_DIrectory.py @@ -8,13 +8,13 @@ from sorcha.utilities.dataUtilitiesForTests import get_test_filepath from sorcha.utilities.sorchaArguments import sorchaArguments -def test_PPFindFileOrExit(): - from sorcha.modules.PPConfigParser import PPFindFileOrExit +def test_FindFileOrExit(): + from sorcha.utilities.Find_File_or_Directory import FindFileOrExit - test_file = PPFindFileOrExit(get_test_filepath("test_PPConfig.ini"), "config file") + test_file = FindFileOrExit(get_test_filepath("test_PPConfig.ini"), "config file") with pytest.raises(SystemExit) as e: - PPFindFileOrExit("totally_fake_file.txt", "test") + FindFileOrExit("totally_fake_file.txt", "test") assert test_file == get_test_filepath("test_PPConfig.ini") assert e.type == SystemExit @@ -22,13 +22,13 @@ def test_PPFindFileOrExit(): return -def test_PPFindDirectoryOrExit(): - from sorcha.modules.PPConfigParser import PPFindDirectoryOrExit +def test_FindDirectoryOrExit(): + from sorcha.utilities.Find_File_or_Directory import FindDirectoryOrExit - test_dir = PPFindDirectoryOrExit("./", "test") + test_dir = FindDirectoryOrExit("./", "test") with pytest.raises(SystemExit) as e: - PPFindDirectoryOrExit("./fake_dir/", "test") + FindDirectoryOrExit("./fake_dir/", "test") assert test_dir == "./" assert e.type == SystemExit From 39696e339bcc3239ab9bc0694f198990651be2f4 Mon Sep 17 00:00:00 2001 From: Ryan Lyttle Date: Thu, 9 Jan 2025 10:31:08 +0000 Subject: [PATCH 5/5] Changed name of Find_File_or_Directory to fileAccessUtils Changed name of Find_File_or_Directory to fileAccessUtils. Unit tests have been manually ran on my computer and they passed. --- src/sorcha/modules/PPCommandLineParser.py | 2 +- src/sorcha/sorcha.py | 3 +- ...ile_or_Directory.py => fileAccessUtils.py} | 0 src/sorcha/utilities/sorchaConfigs.py | 29 +------------------ src/sorcha/utilities/sorcha_copy_configs.py | 2 +- .../utilities/sorcha_copy_demo_files.py | 2 +- src/sorcha_cmdline/outputs.py | 4 +-- ...gsToLog.txt => test_PrintConfigsToLog.txt} | 0 ...r_DIrectory.py => test_fileAccessUtils.py} | 4 +-- tests/sorcha/test_sorchaConfigs.py | 2 +- 10 files changed, 11 insertions(+), 37 deletions(-) rename src/sorcha/utilities/{Find_File_or_Directory.py => fileAccessUtils.py} (100%) rename tests/data/{test_PPPrintConfigsToLog.txt => test_PrintConfigsToLog.txt} (100%) rename tests/sorcha/{test_Find_File_or_DIrectory.py => test_fileAccessUtils.py} (87%) diff --git a/src/sorcha/modules/PPCommandLineParser.py b/src/sorcha/modules/PPCommandLineParser.py index ff0f66a8..5334acb7 100644 --- a/src/sorcha/modules/PPCommandLineParser.py +++ b/src/sorcha/modules/PPCommandLineParser.py @@ -2,7 +2,7 @@ import sys import logging import glob -from ..utilities.Find_File_or_Directory import FindFileOrExit, FindDirectoryOrExit +from sorcha.utilities.fileAccessUtils import FindFileOrExit, FindDirectoryOrExit def warn_or_remove_file(filepath, force_remove, pplogger): diff --git a/src/sorcha/sorcha.py b/src/sorcha/sorcha.py index 560b2f72..3591047b 100755 --- a/src/sorcha/sorcha.py +++ b/src/sorcha/sorcha.py @@ -20,7 +20,7 @@ from sorcha.modules import PPAddUncertainties, PPRandomizeMeasurements from sorcha.modules import PPVignetting from sorcha.modules.PPFadingFunctionFilter import PPFadingFunctionFilter -from sorcha.utilities.Find_File_or_Directory import FindFileOrExit + from sorcha.modules.PPGetLogger import PPGetLogger from sorcha.modules.PPCommandLineParser import PPCommandLineParser @@ -41,6 +41,7 @@ from sorcha.utilities.sorchaArguments import sorchaArguments from sorcha.utilities.sorchaConfigs import sorchaConfigs, PrintConfigsToLog +from sorcha.utilities.fileAccessUtils import FindFileOrExit from sorcha.utilities.citation_text import cite_sorcha diff --git a/src/sorcha/utilities/Find_File_or_Directory.py b/src/sorcha/utilities/fileAccessUtils.py similarity index 100% rename from src/sorcha/utilities/Find_File_or_Directory.py rename to src/sorcha/utilities/fileAccessUtils.py diff --git a/src/sorcha/utilities/sorchaConfigs.py b/src/sorcha/utilities/sorchaConfigs.py index f68ad1e2..4eacb458 100644 --- a/src/sorcha/utilities/sorchaConfigs.py +++ b/src/sorcha/utilities/sorchaConfigs.py @@ -6,6 +6,7 @@ import numpy as np from sorcha.lightcurves.lightcurve_registration import LC_METHODS from sorcha.activity.activity_registration import CA_METHODS +from sorcha.utilities.fileAccessUtils import FindFileOrExit @dataclass @@ -1267,34 +1268,6 @@ def check_value_in_list(value, valuelist, key): ) -def FindFileOrExit(arg_fn, argname): - """Checks to see if a file given by a filename exists. If it doesn't, - this fails gracefully and exits to the command line. - - Parameters - ----------- - arg_fn : string - The filepath/name of the file to be checked. - - argname : string - The name of the argument being checked. Used for error message. - - Returns - ---------- - arg_fn : string - The filepath/name of the file to be checked. - - """ - - pplogger = logging.getLogger(__name__) - - if os.path.exists(arg_fn): - return arg_fn - else: - pplogger.error("ERROR: filename {} supplied for {} argument does not exist.".format(arg_fn, argname)) - sys.exit("ERROR: filename {} supplied for {} argument does not exist.".format(arg_fn, argname)) - - def cast_as_bool_or_set_default(value, key, default): # replaces PPGetBoolOrExit: checks to make sure the value can be cast as a bool. diff --git a/src/sorcha/utilities/sorcha_copy_configs.py b/src/sorcha/utilities/sorcha_copy_configs.py index 402a6340..45946b99 100644 --- a/src/sorcha/utilities/sorcha_copy_configs.py +++ b/src/sorcha/utilities/sorcha_copy_configs.py @@ -5,7 +5,7 @@ import sys from importlib.resources import files -from sorcha.utilities.Find_File_or_Directory import FindDirectoryOrExit +from sorcha.utilities.fileAccessUtils import FindDirectoryOrExit def copy_demo_configs(copy_location, which_configs, force_overwrite): diff --git a/src/sorcha/utilities/sorcha_copy_demo_files.py b/src/sorcha/utilities/sorcha_copy_demo_files.py index 3ed1f8a0..3504711b 100644 --- a/src/sorcha/utilities/sorcha_copy_demo_files.py +++ b/src/sorcha/utilities/sorcha_copy_demo_files.py @@ -5,7 +5,7 @@ import sys from importlib.resources import files -from sorcha.utilities.Find_File_or_Directory import FindDirectoryOrExit +from sorcha.utilities.fileAccessUtils import FindDirectoryOrExit from sorcha.utilities.sorcha_demo_command import print_demo_command diff --git a/src/sorcha_cmdline/outputs.py b/src/sorcha_cmdline/outputs.py index efce9a57..880e2fcc 100644 --- a/src/sorcha_cmdline/outputs.py +++ b/src/sorcha_cmdline/outputs.py @@ -16,7 +16,7 @@ def cmd_outputs_create_sqlite(args): # pragma: no cover # is poor user experience. # from sorcha.utilities.createResultsSQLDatabase import create_results_database - from sorcha.utilities.Find_File_or_Directory import FindDirectoryOrExit + from sorcha.utilities.fileAccessUtils import FindDirectoryOrExit import os args.output = os.path.abspath(args.output) @@ -36,7 +36,7 @@ def cmd_outputs_create_sqlite(args): # pragma: no cover def cmd_outputs_check_logs(args): # pragma: no cover from sorcha.utilities.check_output_logs import check_output_logs - from sorcha.utilities.Find_File_or_Directory import FindDirectoryOrExit + from sorcha.utilities.fileAccessUtils import FindDirectoryOrExit import os args.filepath = os.path.abspath(args.filepath) diff --git a/tests/data/test_PPPrintConfigsToLog.txt b/tests/data/test_PrintConfigsToLog.txt similarity index 100% rename from tests/data/test_PPPrintConfigsToLog.txt rename to tests/data/test_PrintConfigsToLog.txt diff --git a/tests/sorcha/test_Find_File_or_DIrectory.py b/tests/sorcha/test_fileAccessUtils.py similarity index 87% rename from tests/sorcha/test_Find_File_or_DIrectory.py rename to tests/sorcha/test_fileAccessUtils.py index bfa15090..4c9e91f0 100644 --- a/tests/sorcha/test_Find_File_or_DIrectory.py +++ b/tests/sorcha/test_fileAccessUtils.py @@ -9,7 +9,7 @@ from sorcha.utilities.sorchaArguments import sorchaArguments def test_FindFileOrExit(): - from sorcha.utilities.Find_File_or_Directory import FindFileOrExit + from sorcha.utilities.fileAccessUtils import FindFileOrExit test_file = FindFileOrExit(get_test_filepath("test_PPConfig.ini"), "config file") @@ -23,7 +23,7 @@ def test_FindFileOrExit(): return def test_FindDirectoryOrExit(): - from sorcha.utilities.Find_File_or_Directory import FindDirectoryOrExit + from sorcha.utilities.fileAccessUtils import FindDirectoryOrExit test_dir = FindDirectoryOrExit("./", "test") diff --git a/tests/sorcha/test_sorchaConfigs.py b/tests/sorcha/test_sorchaConfigs.py index a1711992..6f422a06 100644 --- a/tests/sorcha/test_sorchaConfigs.py +++ b/tests/sorcha/test_sorchaConfigs.py @@ -1093,7 +1093,7 @@ def test_PrintConfigsToLog(tmp_path): datalog = glob.glob(os.path.join(tmp_path, "*-sorcha.log")) #when updating PrintConfigsToLog text file test_PPPrintConfigsToLog.txt needs to be updated too. - testfile = open(os.path.join(test_path, "test_PPPrintConfigsToLog.txt"), mode="r") + testfile = open(os.path.join(test_path, "test_PrintConfigsToLog.txt"), mode="r") newfile = open(datalog[0], mode="r") alltest = testfile.readlines() allnew = newfile.readlines()