From a74c9662f23e7962abe2cd949b13fa2aea355ad3 Mon Sep 17 00:00:00 2001 From: morobking Date: Wed, 3 Apr 2024 12:03:05 +0100 Subject: [PATCH 01/25] cmorizer python and yml config file added --- .../cmorizers/data/cmor_config/ESACCI-LST.yml | 94 +++++++- .../data/formatters/datasets/esacci_lst.py | 227 +++++++----------- 2 files changed, 179 insertions(+), 142 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml index b15ce08be0..75ad37b6da 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml @@ -7,25 +7,105 @@ # Common global attributes for Cmorizer output attributes: dataset_id: ESACCI-LST - version: '1.00' + version: '3.00' tier: 2 project_id: OBS source: 'ESA CCI' modeling_realm: sat reference: 'esacci_lst' comment: '' - start_year: 2003 - end_year: 2018 + start_year: 2006 + end_year: 2007 # Variables to cmorize # These go into the vals dictionary in the python script variables: - ts: + tsDay: mip: Amon raw: land surface temperature raw_units: kelvin - file_day: 'ESACCI-LST-L3C-LST-MODISA-0.05deg_1MONTHLY_DAY-' - file_night: 'ESACCI-LST-L3C-LST-MODISA-0.05deg_1MONTHLY_NIGHT-' - # plan to make it possible to change MODISA here but only placeholders in the python there for now + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + start_year: 2006 + end_year: 2007 + short_name: XYZ_TESTING + + + # tsNight: + # mip: Amon + # raw: land surface temperature + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + # start_year: 2006 + # end_year: 2007 + + + # tsLocalAtmErrDay: + # mip: Amon + # raw: uncertainty from locally correlated errors on atmospheric scales + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + # start_year: 2006 + # end_year: 2007 + + # tsLocalAtmErrNight: + # mip: Amon + # raw: uncertainty from locally correlated errors on atmospheric scales + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + # start_year: 2006 + # end_year: 2007 + + # tsLocalSfcErrDay: + # mip: Amon + # raw: uncertainty from locally correlated errors on surface scales + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + # start_year: 2006 + # end_year: 2007 + + # tsLocalSfcErrNight: + # mip: Amon + # raw: uncertainty from locally correlated errors on surface scales + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + # start_year: 2006 + # end_year: 2007 + + # tsLSSysErrDay: + # mip: Amon + # raw: uncertainty from large-scale systematic errors + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + # start_year: 2006 + # end_year: 2007 + + # tsLSSysErrNight: + # mip: Amon + # raw: uncertainty from large-scale systematic errors + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + # start_year: 2006 + # end_year: 2007 + + # tsUnCorErrDay: + # mip: Amon + # raw: uncertainty from uncorrelated errors + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + # start_year: 2006 + # end_year: 2007 + + # tsUnCorErrNight: + # mip: Amon + # raw: uncertainty from uncorrelated errors + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + # start_year: 2006 + # end_year: 2007 + + + + +# plan to make it possible to change MODISA here but only placeholders in the python there for now # Period covered: 2003-2018 # 2003 Only has data from July onwards for Aqua MODIS diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index 70d8f1dc1e..44501e92b2 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -1,160 +1,117 @@ -"""ESMValTool CMORizer for ESACCI-LST data. - -Tier - Tier 2: other freely-available dataset. - -Source - On CEDA-JASMIN - /gws/nopw/j04/esacci_lst/public - For access to this JASMIN group workspace please register at - https://accounts.jasmin.ac.uk/services/group_workspaces/esacci_lst/ - -Download and processing instructions - Put all files under a single directory (no subdirectories with years) - in ${RAWOBS}/Tier2/ESACCI-LST - BOTH DAY and NIGHT files are needed for each month +"""ESMValTool CMORizer for ESACCI-LST-UNCERT data. +BOTH DAY and NIGHT files are needed for each month Currently set to work with only the MODIS AQUA L3 monthly data - -Modification history - 20201015 Started by Robert King - 20201029 Day/Night averaging added along with CMOR utils """ import datetime import logging -from calendar import monthrange - import iris +import cf_units as unit -from esmvaltool.cmorizers.data import utilities as utils +from ...utilities import fix_coords logger = logging.getLogger(__name__) def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): """Cmorization func call.""" - cmor_table = cfg['cmor_table'] glob_attrs = cfg['attributes'] - # run the cmorization + # variable_list contains the variable list + # variable_keys has the short 'code' as a key for the variables. + # both these lists are in made in the same order # vals has the info from the yml file # var is set up in the yml file + print('........................') + print(cfg['variables']) for var, vals in cfg['variables'].items(): - # leave this loop in as might be useful in - # the future for getting other info - # like uncertainty information from the original files - glob_attrs['mip'] = vals['mip'] - cmor_info = cmor_table.get_variable(vals['mip'], var) - var_name = cmor_info.short_name - - for key in vals.keys(): - logger.info("%s %s", key, vals[key]) - - variable = vals['raw'] - # not currently used, but referenced for future - # platform = 'MODISA' # loop over years and months # get years from start_year and end_year - # note 2003 doesn't start until July so not included at this stage - for year in range(glob_attrs['start_year'], - glob_attrs['end_year'] + 1): - this_years_cubes = iris.cube.CubeList() - for month0 in range(12): # Change this in final version - month = month0 + 1 + for year in range(vals['start_year'], vals['end_year'] + 1): + for month in range(1, 13): + logger.info(year) logger.info(month) - day_cube, night_cube = load_cubes(in_dir, vals['file_day'], - vals['file_night'], year, - month, variable) - - monthly_cube = make_monthly_average(day_cube, night_cube, year, - month) - - # use CMORizer utils - monthly_cube = utils.fix_coords(monthly_cube) - - this_years_cubes.append(monthly_cube) - - # Use utils save - # This seems to save files all with the same name! - # Fixed by making yearly files - this_years_cubes = this_years_cubes.merge_cube() - this_years_cubes.long_name = 'Surface Temperature' - this_years_cubes.standard_name = 'surface_temperature' - - # Fix variable metadata - utils.fix_var_metadata(this_years_cubes, cmor_info) - - # Fix global metadata - utils.set_global_atts(this_years_cubes, glob_attrs) - - utils.save_variable( - this_years_cubes, - var_name, - out_dir, - glob_attrs, - unlimited_dimensions=['time'], - ) - - -def load_cubes(in_dir, file_day, file_night, year, month, variable): - """Variable description. - - variable = land surface temperature - platform = AQUA not used for now - but in place for future expansion to all ESC CCI LST platforms - """ - logger.info('Loading %s/%s%s%s*.nc', in_dir, file_day, year, month) - day_cube = iris.load_cube( - '%s/%s%s%02d*.nc' % (in_dir, file_day, year, month), variable) - logger.info('Loading %s/%s%s%s*.nc', in_dir, file_night, year, month) - night_cube = iris.load_cube( - '%s/%s%s%02d*.nc' % (in_dir, file_night, year, month), variable) - - return day_cube, night_cube - - -def make_monthly_average(day_cube, night_cube, year, month): - """Make the average LST form the day time and night time files.""" - day_cube.attributes.clear() - night_cube.attributes.clear() - - co_time = night_cube.coord('time') - co_time.points = co_time.points + 100.0 - # maybe the arbitrary difference should go on day cubes to - # take the timestamp to 12Z? - # not really an issue when using monthly files - - result = iris.cube.CubeList([day_cube, night_cube]).concatenate_cube() - - # This corrects the lonitude coord name issue - # This should be fixed in the next version of the CCI data - logger.info("Longitude coordinate correction being applied") - result.coords()[2].var_name = 'longitude' - result.coords()[2].standard_name = 'longitude' - result.coords()[2].long_name = 'longitude' - - monthly_cube = result.collapsed('time', iris.analysis.MEAN) - - # fix time coordinate bounds - monthly_co_time = monthly_cube.coord('time') - - time_point = (datetime.datetime(year, month, 1, 0, 0) - - datetime.datetime(1981, 1, 1, 0, 0, 0)).total_seconds() - monthly_co_time.points = time_point - - num_days = monthrange(year, month)[1] - monthly_co_time.bounds = [ - time_point, time_point + ((num_days - 1) * 24 * 3600) - ] - # should this be num_days or num_days-1 ### question for Valeriu or Axel - # or 23:59:59 ??? - - monthly_cube.attributes = { - 'information': 'Mean of Day and Night Aqua MODIS monthly LST' - } - - return monthly_cube + cubes = load_cubes(in_dir, + vals['file'], + year, + month, + vals['raw'], + ) + + # make time coords + time_units = 'hours since 1970-01-01 00:00:00' + time_point = unit.date2num(datetime.datetime(year, month, 1), + time_units, + unit.CALENDAR_STANDARD + ) + + time_coord = iris.coords.DimCoord(time_point, + standard_name='time', + long_name='time', + var_name='time', + units=time_units, + bounds=None, + attributes=None, + coord_system=None, + circular=False + ) + + cubes.attributes = {} + cubes.attributes['var'] = var + + try: + cubes.remove_coord('time') + except: + logger.info('Coord fix issue %s' % cubes.long_name) + + cubes.add_dim_coord(time_coord, 0) + + if cubes.long_name == 'land surface temperature': + cubes.long_name = 'surface_temperature' + cubes.standard_name = 'surface_temperature' + + try: + cubes = fix_coords(cubes) + except: + logger.info('skip fixing') + logger.info(cubes.long_name) + + try: + cubes.coords()[2].standard_name = 'longitude' + except: + # No change needed + pass + + var_name = cubes.attributes['var'] + + if cubes.var_name == 'lst': + cubes.var_name = 'ts' + + if 'Day' in var_name: + cubes.long_name += ' Day' + cubes.var_name += '_day' + + + if 'Night' in var_name: + cubes.long_name += ' Night' + cubes.var_name += '_night' + + # make use - and _ consistent!!! + save_name = f'{out_dir}/OBS_ESACCI-LST_sat_3.00_Amon_{var_name}_{year}{month:02d}.nc' + iris.save(cubes, + save_name + ) + + +def load_cubes(in_dir, file, year, month, variable_list): + """Load files into cubes based on variables wanted in variable_list.""" + logger.info(f'Loading {in_dir}/{file}{year}{month:02d}.nc') + cube = iris.load_cube(f'{in_dir}/{file}{year}{month:02d}*.nc', + variable_list + ) + + return cube From 4985426cc7a3f36770cb06e0fbe85add9f1af2dd Mon Sep 17 00:00:00 2001 From: morobking Date: Wed, 3 Apr 2024 12:36:31 +0100 Subject: [PATCH 02/25] All variables now included --- .../cmorizers/data/cmor_config/ESACCI-LST.yml | 137 ++++++++---------- 1 file changed, 64 insertions(+), 73 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml index 75ad37b6da..8128f267f5 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml @@ -1,9 +1,7 @@ # CMORIZE ESA CCI LST -# Follwing CRU and ESACCI-OC as examples -# Only looking at AQUA MONTHLY data but python has placeholders for different platforms +# Only looking at AQUA MONTHLY --- - # Common global attributes for Cmorizer output attributes: dataset_id: ESACCI-LST @@ -30,82 +28,75 @@ variables: short_name: XYZ_TESTING - # tsNight: - # mip: Amon - # raw: land surface temperature - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2006 - # end_year: 2007 - - - # tsLocalAtmErrDay: - # mip: Amon - # raw: uncertainty from locally correlated errors on atmospheric scales - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2006 - # end_year: 2007 - - # tsLocalAtmErrNight: - # mip: Amon - # raw: uncertainty from locally correlated errors on atmospheric scales - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2006 - # end_year: 2007 - - # tsLocalSfcErrDay: - # mip: Amon - # raw: uncertainty from locally correlated errors on surface scales - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2006 - # end_year: 2007 + tsNight: + mip: Amon + raw: land surface temperature + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + start_year: 2006 + end_year: 2007 - # tsLocalSfcErrNight: - # mip: Amon - # raw: uncertainty from locally correlated errors on surface scales - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2006 - # end_year: 2007 - # tsLSSysErrDay: - # mip: Amon - # raw: uncertainty from large-scale systematic errors - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2006 - # end_year: 2007 + tsLocalAtmErrDay: + mip: Amon + raw: uncertainty from locally correlated errors on atmospheric scales + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + start_year: 2006 + end_year: 2007 - # tsLSSysErrNight: - # mip: Amon - # raw: uncertainty from large-scale systematic errors - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2006 - # end_year: 2007 + tsLocalAtmErrNight: + mip: Amon + raw: uncertainty from locally correlated errors on atmospheric scales + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2006 + end_year: 2007 - # tsUnCorErrDay: - # mip: Amon - # raw: uncertainty from uncorrelated errors - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2006 - # end_year: 2007 + tsLocalSfcErrDay: + mip: Amon + raw: uncertainty from locally correlated errors on surface scales + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + start_year: 2006 + end_year: 2007 - # tsUnCorErrNight: - # mip: Amon - # raw: uncertainty from uncorrelated errors - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2006 - # end_year: 2007 + tsLocalSfcErrNight: + mip: Amon + raw: uncertainty from locally correlated errors on surface scales + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2006 + end_year: 2007 + tsLSSysErrDay: + mip: Amon + raw: uncertainty from large-scale systematic errors + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + start_year: 2006 + end_year: 2007 + tsLSSysErrNight: + mip: Amon + raw: uncertainty from large-scale systematic errors + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2006 + end_year: 2007 + tsUnCorErrDay: + mip: Amon + raw: uncertainty from uncorrelated errors + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + start_year: 2006 + end_year: 2007 -# plan to make it possible to change MODISA here but only placeholders in the python there for now - # Period covered: 2003-2018 - # 2003 Only has data from July onwards for Aqua MODIS + tsUnCorErrNight: + mip: Amon + raw: uncertainty from uncorrelated errors + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2006 + end_year: 2007 From bdc92b58eb20f02397ae07863eef3ed4afecc18d Mon Sep 17 00:00:00 2001 From: morobking Date: Wed, 3 Apr 2024 16:39:37 +0100 Subject: [PATCH 03/25] Simple diagnostic to test CMORized V3 data. Not best practice but prints values, can be viewed in log file --- esmvaltool/diag_scripts/lst/lst_testing.py | 141 +++++++++++++++++++++ esmvaltool/recipes/recipe_esacci_lst.yml | 99 ++++++++++++--- 2 files changed, 222 insertions(+), 18 deletions(-) create mode 100644 esmvaltool/diag_scripts/lst/lst_testing.py diff --git a/esmvaltool/diag_scripts/lst/lst_testing.py b/esmvaltool/diag_scripts/lst/lst_testing.py new file mode 100644 index 0000000000..19b306c7e7 --- /dev/null +++ b/esmvaltool/diag_scripts/lst/lst_testing.py @@ -0,0 +1,141 @@ +""" +ESMValTool diagnostic for ESA CCI LST data. + +The code uses the all time average monthly data. +The ouptput is a timeseries plot of the mean differnce of +CCI LST to model ensemble average, with the ensemble spread +represented by a standard deviation either side of the mean. +""" + +import logging +import time +import iris +import matplotlib.pyplot as plt +import numpy as np + +from esmvaltool.diag_scripts.shared import ( + ProvenanceLogger, + get_plot_filename, + group_metadata, + run_diagnostic, +) + +logger = logging.getLogger(__name__) + + +def _get_input_cubes(metadata): + """Load the data files into cubes. + + Based on the hydrology diagnostic. + + Inputs: + metadata = List of dictionaries made from the preprocessor config + + Outputs: + inputs = Dictionary of cubes + ancestors = Dictionary of filename information + """ + inputs = {} + ancestors = {} + print('/\/\/\/\/\/\/\/\/\/\/') + for attributes in metadata: + print(attributes) + short_name = attributes['short_name'] + filename = attributes['filename'] + logger.info("Loading variable %s", short_name) + cube = iris.load_cube(filename) + cube.attributes.clear() + inputs[short_name] = cube + ancestors[short_name] = [filename] + print('@@@@@@@@@@@@@@@@@@@@@@') + return inputs, ancestors + + +def _get_provenance_record(attributes, ancestor_files): + """Create the provenance record dictionary. + + Inputs: + attributes = dictionary of ensembles/models used, the region bounds + and years of data used. + ancestor_files = list of data files used by the diagnostic. + + Outputs: + record = dictionary of provenance records. + """ + caption = "Timeseries of ESA CCI LST difference to mean of "\ + + "model ensembles calculated over region bounded by latitude "\ + + "{lat_south} to {lat_north}, longitude {lon_west} to {lon_east} "\ + + "and for model/ensembles {ensembles}. "\ + + "Shown for years {start_year} to {end_year}.".format(**attributes) + + record = { + 'caption': caption, + 'statistics': ['mean', 'stddev'], + 'domains': ['reg'], + 'plot_types': ['times'], + 'authors': ['king_robert'], + # 'references': [], + 'ancestors': ancestor_files + } + + return record + + +def _diagnostic(config): + """Perform the control for the ESA CCI LST diagnostic. + + Parameters + ---------- + config: dict + the preprocessor nested dictionary holding + all the needed information. + + Returns + ------- + figures made by make_plots. + """ + # this loading function is based on the hydrology diagnostic + input_metadata = config['input_data'].values() + print('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + print(input_metadata) + print('bbbbbbbbbbbbbbbbb') + loaded_data = {} + ancestor_list = [] + print('ccccccccccccccccccccccccccccccccccccccccccccccccc') + print(group_metadata(input_metadata, 'dataset')) + print('dddddddddddddddddddddddddddddddddddddddddddddddddddddddd') + for dataset, metadata in group_metadata(input_metadata, 'dataset').items(): + print(dataset, metadata) + cubes, ancestors = _get_input_cubes(metadata) + print(cubes, ancestors) + loaded_data[dataset] = cubes + # ancestor_list.append(ancestors['ts'][0]) + + print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$') + print(ancestor_list) + print('*************************************') + + # loaded data is a nested dictionary + # KEY1 model ESACCI-LST or something else + # KEY2 is ts, the surface temperature + # ie loaded_data['ESACCI-LST']['ts'] is the CCI cube + # loaded_data['MultiModelMean']['ts'] is CMIP6 data, emsemble means + # similarly dor Std, see preprocessor + + # The Diagnostic uses CCI - MODEL + print('###########################################') + print(loaded_data) + print('##########################################') + + START = time.time() + for KEY in loaded_data['ESACCI-LST'].keys(): + xyz = np.mean(loaded_data['ESACCI-LST'][KEY].data) + print(f'{KEY} {xyz}') + print(f'TIME {time.time()-START}') + + +if __name__ == '__main__': + # always use run_diagnostic() to get the config (the preprocessor + # nested dictionary holding all the needed information) + with run_diagnostic() as config: + _diagnostic(config) diff --git a/esmvaltool/recipes/recipe_esacci_lst.yml b/esmvaltool/recipes/recipe_esacci_lst.yml index 5f4aa5ca9f..e972353247 100644 --- a/esmvaltool/recipes/recipe_esacci_lst.yml +++ b/esmvaltool/recipes/recipe_esacci_lst.yml @@ -1,4 +1,4 @@ -# Recipe to call ESA CCI LST diagnostic. +# Recipe to call ESA CCI LST V3 testing diagnostic. --- documentation: title: ESA CCI LST diagnostic @@ -16,26 +16,25 @@ documentation: projects: - cmug -datasets: - - {dataset: CESM2, project: CMIP6, exp: historical, ensemble: r(2:3)i1p1f1, - start_year: 2004, end_year: 2005, grid: gn} - - {dataset: UKESM1-0-LL, project: CMIP6, exp: historical, - ensemble: r(1:2)i1p1f2, start_year: 2004, end_year: 2005, grid: gn} - - {dataset: ESACCI-LST, project: OBS, type: sat, tier: 2, - start_year: 2004, end_year: 2005, version: '1.00'} - preprocessors: - lst_preprocessor: + obs_preprocessor: + extract_region: + start_longitude: 35 + end_longitude: 45 + start_latitude: 55 + end_latitude: 60 + + cmip_preprocessor: regrid: target_grid: UKESM1-0-LL scheme: linear extract_region: start_longitude: 35 - end_longitude: 175 + end_longitude: 45 start_latitude: 55 - end_latitude: 70 + end_latitude: 60 mask_landsea: mask_out: sea @@ -43,10 +42,10 @@ preprocessors: area_statistics: operator: mean - multi_model_statistics: - span: overlap - statistics: [mean, std_dev] - exclude: [ESACCI-LST] + # ensemble_statistics: + # span: overlap + # statistics: [mean, std_dev] + # exclude: [ESACCI-LST] diagnostics: @@ -60,8 +59,72 @@ diagnostics: variables: ts: mip: Amon - preprocessor: lst_preprocessor + preprocessor: cmip_preprocessor + additional_datasets: + - {dataset: UKESM1-0-LL, project: CMIP6, exp: historical, ensemble: r1i1p1f2, start_year: 2006, end_year: 2007, grid: gn} + + tsDay: + mip: Amon + preprocessor: obs_preprocessor + additional_datasets: + - {dataset: ESACCI-LST, project: OBS, type: sat, tier: 2, start_year: 2006, end_year: 2007, version: '3.00'} + + + tsNight: + mip: Amon + preprocessor: obs_preprocessor + additional_datasets: + - {dataset: ESACCI-LST, project: OBS, type: sat, tier: 2, start_year: 2006, end_year: 2007, version: '3.00'} + + tsLocalSfcErrDay: + mip: Amon + preprocessor: obs_preprocessor + additional_datasets: + - {dataset: ESACCI-LST, project: OBS, type: sat, tier: 2, start_year: 2006, end_year: 2007, version: '3.00'} + + tsLocalSfcErrNight: + mip: Amon + preprocessor: obs_preprocessor + additional_datasets: + - {dataset: ESACCI-LST, project: OBS, type: sat, tier: 2, start_year: 2006, end_year: 2007, version: '3.00'} + + tsLocalAtmErrDay: + mip: Amon + preprocessor: obs_preprocessor + additional_datasets: + - {dataset: ESACCI-LST, project: OBS, type: sat, tier: 2, start_year: 2006, end_year: 2007, version: '3.00'} + + tsLocalAtmErrNight: + mip: Amon + preprocessor: obs_preprocessor + additional_datasets: + - {dataset: ESACCI-LST, project: OBS, type: sat, tier: 2, start_year: 2006, end_year: 2007, version: '3.00'} + + tsLSSysErrDay: + mip: Amon + # preprocessor: obs_preprocessor # Single value in files, no area cut out needed + additional_datasets: + - {dataset: ESACCI-LST, project: OBS, type: sat, tier: 2, start_year: 2006, end_year: 2007, version: '3.00'} + + tsLSSysErrNight: + mip: Amon + # preprocessor: obs_preprocessor # Single value in files, no area cut out needed + additional_datasets: + - {dataset: ESACCI-LST, project: OBS, type: sat, tier: 2, start_year: 2006, end_year: 2007, version: '3.00'} + + tsUnCorErrDay: + mip: Amon + preprocessor: obs_preprocessor + additional_datasets: + - {dataset: ESACCI-LST, project: OBS, type: sat, tier: 2, start_year: 2006, end_year: 2007, version: '3.00'} + + tsUnCorErrNight: + mip: Amon + preprocessor: obs_preprocessor + additional_datasets: + - {dataset: ESACCI-LST, project: OBS, type: sat, tier: 2, start_year: 2006, end_year: 2007, version: '3.00'} scripts: script1: - script: lst/lst.py + script: lst/lst_testing.py + From 45438fb6a984f14319821e3188cb710cd555e7b0 Mon Sep 17 00:00:00 2001 From: morobking Date: Thu, 11 Apr 2024 17:12:53 +0100 Subject: [PATCH 04/25] Correct DAY to NIGHT for tsNight variable in the filename --- esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml index 8128f267f5..2601bb63c5 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml @@ -32,7 +32,7 @@ variables: mip: Amon raw: land surface temperature raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' start_year: 2006 end_year: 2007 From d634622c7af295bf6322c13caf3b5b42ced3c4be Mon Sep 17 00:00:00 2001 From: morobking Date: Fri, 19 Apr 2024 12:36:32 +0100 Subject: [PATCH 05/25] Flake8 and yamllint tidy of code --- .../cmorizers/data/cmor_config/ESACCI-LST.yml | 23 ++++++++----------- .../data/formatters/datasets/esacci_lst.py | 10 +++----- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml index 2601bb63c5..98e6d47ed3 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml @@ -1,5 +1,5 @@ -# CMORIZE ESA CCI LST -# Only looking at AQUA MONTHLY +# CMORIZE ESA CCI LST - V3 +# Only looking at AQUA MODIS MONTHLY --- # Common global attributes for Cmorizer output @@ -25,8 +25,6 @@ variables: file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' start_year: 2006 end_year: 2007 - short_name: XYZ_TESTING - tsNight: mip: Amon @@ -36,8 +34,7 @@ variables: start_year: 2006 end_year: 2007 - - tsLocalAtmErrDay: + tsLocalAtmErrDay: mip: Amon raw: uncertainty from locally correlated errors on atmospheric scales raw_units: kelvin @@ -45,7 +42,7 @@ variables: start_year: 2006 end_year: 2007 - tsLocalAtmErrNight: + tsLocalAtmErrNight: mip: Amon raw: uncertainty from locally correlated errors on atmospheric scales raw_units: kelvin @@ -53,7 +50,7 @@ variables: start_year: 2006 end_year: 2007 - tsLocalSfcErrDay: + tsLocalSfcErrDay: mip: Amon raw: uncertainty from locally correlated errors on surface scales raw_units: kelvin @@ -61,7 +58,7 @@ variables: start_year: 2006 end_year: 2007 - tsLocalSfcErrNight: + tsLocalSfcErrNight: mip: Amon raw: uncertainty from locally correlated errors on surface scales raw_units: kelvin @@ -69,7 +66,7 @@ variables: start_year: 2006 end_year: 2007 - tsLSSysErrDay: + tsLSSysErrDay: mip: Amon raw: uncertainty from large-scale systematic errors raw_units: kelvin @@ -77,7 +74,7 @@ variables: start_year: 2006 end_year: 2007 - tsLSSysErrNight: + tsLSSysErrNight: mip: Amon raw: uncertainty from large-scale systematic errors raw_units: kelvin @@ -85,7 +82,7 @@ variables: start_year: 2006 end_year: 2007 - tsUnCorErrDay: + tsUnCorErrDay: mip: Amon raw: uncertainty from uncorrelated errors raw_units: kelvin @@ -93,7 +90,7 @@ variables: start_year: 2006 end_year: 2007 - tsUnCorErrNight: + tsUnCorErrNight: mip: Amon raw: uncertainty from uncorrelated errors raw_units: kelvin diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index 44501e92b2..779d2fee51 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -1,7 +1,7 @@ -"""ESMValTool CMORizer for ESACCI-LST-UNCERT data. +"""ESMValTool CMORizer for ESACCI-LST. -BOTH DAY and NIGHT files are needed for each month -Currently set to work with only the MODIS AQUA L3 monthly data +Tested for the V3 of the data +Works for both day and night files """ import datetime @@ -24,8 +24,6 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): # vals has the info from the yml file # var is set up in the yml file - print('........................') - print(cfg['variables']) for var, vals in cfg['variables'].items(): glob_attrs['mip'] = vals['mip'] @@ -94,13 +92,11 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): if 'Day' in var_name: cubes.long_name += ' Day' cubes.var_name += '_day' - if 'Night' in var_name: cubes.long_name += ' Night' cubes.var_name += '_night' - # make use - and _ consistent!!! save_name = f'{out_dir}/OBS_ESACCI-LST_sat_3.00_Amon_{var_name}_{year}{month:02d}.nc' iris.save(cubes, save_name From c85cce06d2b131708cfe481d957da7a579be85a7 Mon Sep 17 00:00:00 2001 From: morobking Date: Fri, 19 Apr 2024 15:09:57 +0100 Subject: [PATCH 06/25] minor edits --- .../cmorizers/data/cmor_config/ESACCI-LST.yml | 22 +++++++++---------- .../data/formatters/datasets/esacci_lst.py | 1 + 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml index 98e6d47ed3..79a64119d6 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml @@ -13,7 +13,7 @@ attributes: reference: 'esacci_lst' comment: '' start_year: 2006 - end_year: 2007 + end_year: 2006 # Variables to cmorize # These go into the vals dictionary in the python script @@ -24,7 +24,7 @@ variables: raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' start_year: 2006 - end_year: 2007 + end_year: 2006 tsNight: mip: Amon @@ -32,7 +32,7 @@ variables: raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' start_year: 2006 - end_year: 2007 + end_year: 2006 tsLocalAtmErrDay: mip: Amon @@ -40,7 +40,7 @@ variables: raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' start_year: 2006 - end_year: 2007 + end_year: 2006 tsLocalAtmErrNight: mip: Amon @@ -48,7 +48,7 @@ variables: raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' start_year: 2006 - end_year: 2007 + end_year: 2006 tsLocalSfcErrDay: mip: Amon @@ -56,7 +56,7 @@ variables: raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' start_year: 2006 - end_year: 2007 + end_year: 2006 tsLocalSfcErrNight: mip: Amon @@ -64,7 +64,7 @@ variables: raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' start_year: 2006 - end_year: 2007 + end_year: 2006 tsLSSysErrDay: mip: Amon @@ -72,7 +72,7 @@ variables: raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' start_year: 2006 - end_year: 2007 + end_year: 2006 tsLSSysErrNight: mip: Amon @@ -80,7 +80,7 @@ variables: raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' start_year: 2006 - end_year: 2007 + end_year: 2006 tsUnCorErrDay: mip: Amon @@ -88,7 +88,7 @@ variables: raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' start_year: 2006 - end_year: 2007 + end_year: 2006 tsUnCorErrNight: mip: Amon @@ -96,4 +96,4 @@ variables: raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' start_year: 2006 - end_year: 2007 + end_year: 2006 diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index 779d2fee51..66961b4d2a 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -78,6 +78,7 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): logger.info('skip fixing') logger.info(cubes.long_name) + # this is needed for V1 data, V3 data is ok try: cubes.coords()[2].standard_name = 'longitude' except: From 6ba03d3bc0b252e3d2fa545275c7e90c0aef76a7 Mon Sep 17 00:00:00 2001 From: morobking Date: Wed, 24 Apr 2024 15:03:42 +0100 Subject: [PATCH 07/25] try/except added around load_cubes to prevent missing or bad files crashing the whole run --- .../cmorizers/data/cmor_config/ESACCI-LST.yml | 44 +++++++++---------- .../data/formatters/datasets/esacci_lst.py | 14 +++--- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml index 79a64119d6..4bb0d04a6c 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml @@ -12,8 +12,8 @@ attributes: modeling_realm: sat reference: 'esacci_lst' comment: '' - start_year: 2006 - end_year: 2006 + start_year: 2014 + end_year: 2014 # Variables to cmorize # These go into the vals dictionary in the python script @@ -23,77 +23,77 @@ variables: raw: land surface temperature raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - start_year: 2006 - end_year: 2006 + start_year: 2014 + end_year: 2014 tsNight: mip: Amon raw: land surface temperature raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2006 - end_year: 2006 + start_year: 2014 + end_year: 2014 tsLocalAtmErrDay: mip: Amon raw: uncertainty from locally correlated errors on atmospheric scales raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - start_year: 2006 - end_year: 2006 + start_year: 2014 + end_year: 2014 tsLocalAtmErrNight: mip: Amon raw: uncertainty from locally correlated errors on atmospheric scales raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2006 - end_year: 2006 + start_year: 2014 + end_year: 2014 tsLocalSfcErrDay: mip: Amon raw: uncertainty from locally correlated errors on surface scales raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - start_year: 2006 - end_year: 2006 + start_year: 2014 + end_year: 2014 tsLocalSfcErrNight: mip: Amon raw: uncertainty from locally correlated errors on surface scales raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2006 - end_year: 2006 + start_year: 2014 + end_year: 2014 tsLSSysErrDay: mip: Amon raw: uncertainty from large-scale systematic errors raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - start_year: 2006 - end_year: 2006 + start_year: 2014 + end_year: 2014 tsLSSysErrNight: mip: Amon raw: uncertainty from large-scale systematic errors raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2006 - end_year: 2006 + start_year: 2014 + end_year: 2014 tsUnCorErrDay: mip: Amon raw: uncertainty from uncorrelated errors raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - start_year: 2006 - end_year: 2006 + start_year: 2014 + end_year: 2014 tsUnCorErrNight: mip: Amon raw: uncertainty from uncorrelated errors raw_units: kelvin file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2006 - end_year: 2006 + start_year: 2014 + end_year: 2014 diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index 66961b4d2a..d39e8d0cdf 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -33,12 +33,16 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): for month in range(1, 13): logger.info(year) logger.info(month) - cubes = load_cubes(in_dir, - vals['file'], - year, - month, - vals['raw'], + try: + cubes = load_cubes(in_dir, + vals['file'], + year, + month, + vals['raw'], ) + except: + logger.info(f'Problem with Month {month} in {year}') + continue # make time coords time_units = 'hours since 1970-01-01 00:00:00' From 224d89079166913dbded8802088896300a14bfa3 Mon Sep 17 00:00:00 2001 From: morobking Date: Thu, 16 May 2024 12:05:36 +0100 Subject: [PATCH 08/25] Add Total uncertainty and LST Variance --- .../cmorizers/data/cmor_config/ESACCI-LST.yml | 97 +++++++++++++------ 1 file changed, 65 insertions(+), 32 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml index 4bb0d04a6c..2eeb28df10 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml @@ -12,88 +12,121 @@ attributes: modeling_realm: sat reference: 'esacci_lst' comment: '' - start_year: 2014 - end_year: 2014 + start_year: 2003 + end_year: 2018 # Variables to cmorize # These go into the vals dictionary in the python script variables: + tsTotalDay: + mip: Amon + raw: land surface temperature total uncertainty + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 + + tsTotalNight: + mip: Amon + raw: land surface temperature total uncertainty + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 + + tsVarDay: + mip: Amon + raw: land surface temperature variance + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 + + tsVarNight: + mip: Amon + raw: land surface temperature variance + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 + tsDay: mip: Amon raw: land surface temperature raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - start_year: 2014 - end_year: 2014 + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 tsNight: mip: Amon raw: land surface temperature raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2014 - end_year: 2014 + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 tsLocalAtmErrDay: mip: Amon raw: uncertainty from locally correlated errors on atmospheric scales raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - start_year: 2014 - end_year: 2014 + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 tsLocalAtmErrNight: mip: Amon raw: uncertainty from locally correlated errors on atmospheric scales raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2014 - end_year: 2014 + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 tsLocalSfcErrDay: mip: Amon raw: uncertainty from locally correlated errors on surface scales raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - start_year: 2014 - end_year: 2014 + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 tsLocalSfcErrNight: mip: Amon raw: uncertainty from locally correlated errors on surface scales raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2014 - end_year: 2014 + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 tsLSSysErrDay: mip: Amon raw: uncertainty from large-scale systematic errors raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - start_year: 2014 - end_year: 2014 + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 tsLSSysErrNight: mip: Amon raw: uncertainty from large-scale systematic errors raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2014 - end_year: 2014 + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 tsUnCorErrDay: mip: Amon raw: uncertainty from uncorrelated errors raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - start_year: 2014 - end_year: 2014 + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 tsUnCorErrNight: mip: Amon raw: uncertainty from uncorrelated errors raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2014 - end_year: 2014 + file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 + From ed18b08f69e94a4510c42c49f8810eda83b9c69d Mon Sep 17 00:00:00 2001 From: morobking Date: Thu, 16 May 2024 14:31:59 +0100 Subject: [PATCH 09/25] update to run 2003-2018 land cover for MODISA --- .../cmorizers/data/cmor_config/ESACCI-LST.yml | 223 ++++++++++-------- 1 file changed, 119 insertions(+), 104 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml index 2eeb28df10..d0a1b8555b 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml @@ -18,115 +18,130 @@ attributes: # Variables to cmorize # These go into the vals dictionary in the python script variables: - tsTotalDay: + tsLCDay: mip: Amon - raw: land surface temperature total uncertainty - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' + raw: land cover class + raw_units: + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' start_year: 2003 end_year: 2018 - tsTotalNight: + tsLCNight: mip: Amon - raw: land surface temperature total uncertainty - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2003 - end_year: 2018 - - tsVarDay: - mip: Amon - raw: land surface temperature variance - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' - start_year: 2003 - end_year: 2018 - - tsVarNight: - mip: Amon - raw: land surface temperature variance - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2003 - end_year: 2018 - - tsDay: - mip: Amon - raw: land surface temperature - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' - start_year: 2003 - end_year: 2018 - - tsNight: - mip: Amon - raw: land surface temperature - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2003 - end_year: 2018 - - tsLocalAtmErrDay: - mip: Amon - raw: uncertainty from locally correlated errors on atmospheric scales - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' - start_year: 2003 - end_year: 2018 - - tsLocalAtmErrNight: - mip: Amon - raw: uncertainty from locally correlated errors on atmospheric scales - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2003 - end_year: 2018 - - tsLocalSfcErrDay: - mip: Amon - raw: uncertainty from locally correlated errors on surface scales - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' - start_year: 2003 - end_year: 2018 - - tsLocalSfcErrNight: - mip: Amon - raw: uncertainty from locally correlated errors on surface scales - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2003 - end_year: 2018 - - tsLSSysErrDay: - mip: Amon - raw: uncertainty from large-scale systematic errors - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' - start_year: 2003 - end_year: 2018 - - tsLSSysErrNight: - mip: Amon - raw: uncertainty from large-scale systematic errors - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2003 - end_year: 2018 - - tsUnCorErrDay: - mip: Amon - raw: uncertainty from uncorrelated errors - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_DAY-' - start_year: 2003 - end_year: 2018 - - tsUnCorErrNight: - mip: Amon - raw: uncertainty from uncorrelated errors - raw_units: kelvin - file: 'ESACCI-LST-L3C-LST-MOSISA-0.01deg_1MONTHLY_NIGHT-' + raw: land cover class + raw_units: + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' start_year: 2003 end_year: 2018 + # tsTotalDay: + # mip: Amon + # raw: land surface temperature total uncertainty + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + # start_year: 2003 + # end_year: 2018 + + # tsTotalNight: + # mip: Amon + # raw: land surface temperature total uncertainty + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + # start_year: 2003 + # end_year: 2018 + + # tsVarDay: + # mip: Amon + # raw: land surface temperature variance + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + # start_year: 2003 + # end_year: 2018 + + # tsVarNight: + # mip: Amon + # raw: land surface temperature variance + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + # start_year: 2003 + # end_year: 2018 + + # tsDay: + # mip: Amon + # raw: land surface temperature + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + # start_year: 2003 + # end_year: 2018 + + # tsNight: + # mip: Amon + # raw: land surface temperature + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + # start_year: 2003 + # end_year: 2018 + + # tsLocalAtmErrDay: + # mip: Amon + # raw: uncertainty from locally correlated errors on atmospheric scales + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + # start_year: 2003 + # end_year: 2018 + + # tsLocalAtmErrNight: + # mip: Amon + # raw: uncertainty from locally correlated errors on atmospheric scales + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + # start_year: 2003 + # end_year: 2018 + + # tsLocalSfcErrDay: + # mip: Amon + # raw: uncertainty from locally correlated errors on surface scales + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + # start_year: 2003 + # end_year: 2018 + + # tsLocalSfcErrNight: + # mip: Amon + # raw: uncertainty from locally correlated errors on surface scales + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + # start_year: 2003 + # end_year: 2018 + + # tsLSSysErrDay: + # mip: Amon + # raw: uncertainty from large-scale systematic errors + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + # start_year: 2003 + # end_year: 2018 + + # tsLSSysErrNight: + # mip: Amon + # raw: uncertainty from large-scale systematic errors + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + # start_year: 2003 + # end_year: 2018 + + # tsUnCorErrDay: + # mip: Amon + # raw: uncertainty from uncorrelated errors + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + # start_year: 2003 + # end_year: 2018 + + # tsUnCorErrNight: + # mip: Amon + # raw: uncertainty from uncorrelated errors + # raw_units: kelvin + # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + # start_year: 2003 + # end_year: 2018 \ No newline at end of file From e239ec5520f5f6564ad96591551b633ae76193cb Mon Sep 17 00:00:00 2001 From: morobking Date: Fri, 17 May 2024 13:21:16 +0100 Subject: [PATCH 10/25] Attempt to fix long int error in LC variable --- .../cmorizers/data/cmor_config/ESACCI-LST.yml | 68 +++++++++---------- .../data/formatters/datasets/esacci_lst.py | 9 +++ 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml index d0a1b8555b..2173fc92fa 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml @@ -12,8 +12,8 @@ attributes: modeling_realm: sat reference: 'esacci_lst' comment: '' - start_year: 2003 - end_year: 2018 + start_year: 2004 + end_year: 2004 # Variables to cmorize # These go into the vals dictionary in the python script @@ -23,125 +23,125 @@ variables: raw: land cover class raw_units: file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - start_year: 2003 - end_year: 2018 + start_year: 2004 + end_year: 2004 tsLCNight: mip: Amon raw: land cover class raw_units: file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2003 - end_year: 2018 + start_year: 2004 + end_year: 2004 # tsTotalDay: # mip: Amon # raw: land surface temperature total uncertainty # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2018 + # start_year: 2004 + # end_year: 2004 # tsTotalNight: # mip: Amon # raw: land surface temperature total uncertainty # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2018 + # start_year: 2004 + # end_year: 2004 # tsVarDay: # mip: Amon # raw: land surface temperature variance # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2018 + # start_year: 2004 + # end_year: 2004 # tsVarNight: # mip: Amon # raw: land surface temperature variance # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2018 + # start_year: 2004 + # end_year: 2004 # tsDay: # mip: Amon # raw: land surface temperature # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2018 + # start_year: 2004 + # end_year: 2004 # tsNight: # mip: Amon # raw: land surface temperature # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2018 + # start_year: 2004 + # end_year: 2004 # tsLocalAtmErrDay: # mip: Amon # raw: uncertainty from locally correlated errors on atmospheric scales # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2018 + # start_year: 2004 + # end_year: 2004 # tsLocalAtmErrNight: # mip: Amon # raw: uncertainty from locally correlated errors on atmospheric scales # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2018 + # start_year: 2004 + # end_year: 2004 # tsLocalSfcErrDay: # mip: Amon # raw: uncertainty from locally correlated errors on surface scales # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2018 + # start_year: 2004 + # end_year: 2004 # tsLocalSfcErrNight: # mip: Amon # raw: uncertainty from locally correlated errors on surface scales # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2018 + # start_year: 2004 + # end_year: 2004 # tsLSSysErrDay: # mip: Amon # raw: uncertainty from large-scale systematic errors # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2018 + # start_year: 2004 + # end_year: 2004 # tsLSSysErrNight: # mip: Amon # raw: uncertainty from large-scale systematic errors # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2018 + # start_year: 2004 + # end_year: 2004 # tsUnCorErrDay: # mip: Amon # raw: uncertainty from uncorrelated errors # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2018 + # start_year: 2004 + # end_year: 2004 # tsUnCorErrNight: # mip: Amon # raw: uncertainty from uncorrelated errors # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2018 \ No newline at end of file + # start_year: 2004 + # end_year: 2004 \ No newline at end of file diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index d39e8d0cdf..cbce301dd1 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -8,6 +8,7 @@ import logging import iris import cf_units as unit +import numpy as np from ...utilities import fix_coords @@ -101,6 +102,14 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): if 'Night' in var_name: cubes.long_name += ' Night' cubes.var_name += '_night' + + # land cover class gives this error when loading in CMORised fiels + # OverflowError: Python int too large to convert to C long error + # Attempt to fix it + if 'land cover' in cubes.long_name: + cubes.data.fill_value = 0 + cubes.data = cubes.data.filled() + cubes.data = np.ma.masked_equal(cubes.data, 0) save_name = f'{out_dir}/OBS_ESACCI-LST_sat_3.00_Amon_{var_name}_{year}{month:02d}.nc' iris.save(cubes, From 09484bd2aa7e97726e2e36689d88ce02e174caf3 Mon Sep 17 00:00:00 2001 From: morobking Date: Fri, 17 May 2024 15:17:00 +0100 Subject: [PATCH 11/25] try normal array for LC instead of masked arry --- esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index cbce301dd1..b5933558c3 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -109,7 +109,7 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): if 'land cover' in cubes.long_name: cubes.data.fill_value = 0 cubes.data = cubes.data.filled() - cubes.data = np.ma.masked_equal(cubes.data, 0) + #cubes.data = np.ma.masked_equal(cubes.data, 0) save_name = f'{out_dir}/OBS_ESACCI-LST_sat_3.00_Amon_{var_name}_{year}{month:02d}.nc' iris.save(cubes, From 409ec2864ec77b52d18b15ebf5933cf9648fdb7a Mon Sep 17 00:00:00 2001 From: morobking Date: Fri, 17 May 2024 16:22:28 +0100 Subject: [PATCH 12/25] using np.float32 to try and solve issue --- .../cmorizers/data/formatters/datasets/esacci_lst.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index b5933558c3..c8483823dc 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -107,8 +107,16 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): # OverflowError: Python int too large to convert to C long error # Attempt to fix it if 'land cover' in cubes.long_name: + logger.info("Got a Land Cover Class cube") cubes.data.fill_value = 0 + logger.info("fill value = 0") + cubes.data = cubes.data.filled() + logger.info("cube filled") + + #cubes.data.dtype = np.float32 + cubes.data = np.array(cubes.data, dtype=np.float32) + logger.info("dtype np float32") #cubes.data = np.ma.masked_equal(cubes.data, 0) save_name = f'{out_dir}/OBS_ESACCI-LST_sat_3.00_Amon_{var_name}_{year}{month:02d}.nc' From 3babc17f7b5a761f353c89cf561127e7063e3c49 Mon Sep 17 00:00:00 2001 From: morobking Date: Fri, 17 May 2024 18:23:42 +0100 Subject: [PATCH 13/25] trying to remove any values in LC outside 0-255 --- esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index c8483823dc..00de6a213b 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -111,6 +111,10 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): cubes.data.fill_value = 0 logger.info("fill value = 0") + # try to get rid of anything outide 0-255 + cubes.data[np.where(cubes.data<0)] = 0 + cubes.data[np.where(cubes.data>255)] = 0 + cubes.data = cubes.data.filled() logger.info("cube filled") From f4191a097a91b0cec44f8cba2d91b7287b24400a Mon Sep 17 00:00:00 2001 From: morobking Date: Wed, 22 May 2024 12:19:00 +0100 Subject: [PATCH 14/25] Another thing to try solve the int issue --- .../cmorizers/data/cmor_config/ESACCI-LST.yml | 34 +++++++++---------- .../data/formatters/datasets/esacci_lst.py | 5 ++- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml index 2173fc92fa..28579d109c 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml @@ -12,7 +12,7 @@ attributes: modeling_realm: sat reference: 'esacci_lst' comment: '' - start_year: 2004 + start_year: 2003 end_year: 2004 # Variables to cmorize @@ -23,7 +23,7 @@ variables: raw: land cover class raw_units: file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - start_year: 2004 + start_year: 2003 end_year: 2004 tsLCNight: @@ -31,7 +31,7 @@ variables: raw: land cover class raw_units: file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - start_year: 2004 + start_year: 2003 end_year: 2004 # tsTotalDay: @@ -39,7 +39,7 @@ variables: # raw: land surface temperature total uncertainty # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 # tsTotalNight: @@ -47,7 +47,7 @@ variables: # raw: land surface temperature total uncertainty # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 # tsVarDay: @@ -55,7 +55,7 @@ variables: # raw: land surface temperature variance # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 # tsVarNight: @@ -63,7 +63,7 @@ variables: # raw: land surface temperature variance # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 # tsDay: @@ -71,7 +71,7 @@ variables: # raw: land surface temperature # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 # tsNight: @@ -79,7 +79,7 @@ variables: # raw: land surface temperature # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 # tsLocalAtmErrDay: @@ -87,7 +87,7 @@ variables: # raw: uncertainty from locally correlated errors on atmospheric scales # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 # tsLocalAtmErrNight: @@ -95,7 +95,7 @@ variables: # raw: uncertainty from locally correlated errors on atmospheric scales # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 # tsLocalSfcErrDay: @@ -103,7 +103,7 @@ variables: # raw: uncertainty from locally correlated errors on surface scales # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 # tsLocalSfcErrNight: @@ -111,7 +111,7 @@ variables: # raw: uncertainty from locally correlated errors on surface scales # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 # tsLSSysErrDay: @@ -119,7 +119,7 @@ variables: # raw: uncertainty from large-scale systematic errors # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 # tsLSSysErrNight: @@ -127,7 +127,7 @@ variables: # raw: uncertainty from large-scale systematic errors # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 # tsUnCorErrDay: @@ -135,7 +135,7 @@ variables: # raw: uncertainty from uncorrelated errors # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 # tsUnCorErrNight: @@ -143,5 +143,5 @@ variables: # raw: uncertainty from uncorrelated errors # raw_units: kelvin # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2004 + # start_year: 2003 # end_year: 2004 \ No newline at end of file diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index 00de6a213b..d8328c5d44 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -120,9 +120,12 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): #cubes.data.dtype = np.float32 cubes.data = np.array(cubes.data, dtype=np.float32) - logger.info("dtype np float32") + logger.info("dtype np float32 applied") + cubes.data *= 1.0 #cubes.data = np.ma.masked_equal(cubes.data, 0) + logger.info(cubes.data.dtype) + save_name = f'{out_dir}/OBS_ESACCI-LST_sat_3.00_Amon_{var_name}_{year}{month:02d}.nc' iris.save(cubes, save_name From c279353d5d51101999e331e97db0fd6ca5a5cf13 Mon Sep 17 00:00:00 2001 From: morobking Date: Wed, 22 May 2024 15:28:19 +0100 Subject: [PATCH 15/25] try plus zero to keep as int --- esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index d8328c5d44..fec873488f 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -119,9 +119,9 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): logger.info("cube filled") #cubes.data.dtype = np.float32 - cubes.data = np.array(cubes.data, dtype=np.float32) - logger.info("dtype np float32 applied") - cubes.data *= 1.0 + #cubes.data = np.array(cubes.data, dtype=np.float32) + #logger.info("dtype np float32 applied") + cubes.data += 0 #cubes.data = np.ma.masked_equal(cubes.data, 0) logger.info(cubes.data.dtype) From 75011836afd26da45a22fd8713e057165a5a73db Mon Sep 17 00:00:00 2001 From: morobking Date: Wed, 22 May 2024 15:46:58 +0100 Subject: [PATCH 16/25] Revert to *1.0 and a tidy --- .../data/formatters/datasets/esacci_lst.py | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index fec873488f..e7b7028168 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -103,28 +103,22 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): cubes.long_name += ' Night' cubes.var_name += '_night' - # land cover class gives this error when loading in CMORised fiels + # land cover class gives this error when loading in CMORised files # OverflowError: Python int too large to convert to C long error - # Attempt to fix it + # This fixes it if 'land cover' in cubes.long_name: logger.info("Got a Land Cover Class cube") cubes.data.fill_value = 0 - logger.info("fill value = 0") - # try to get rid of anything outide 0-255 + # Get rid of anything outide 0-255 cubes.data[np.where(cubes.data<0)] = 0 cubes.data[np.where(cubes.data>255)] = 0 cubes.data = cubes.data.filled() - logger.info("cube filled") - - #cubes.data.dtype = np.float32 - #cubes.data = np.array(cubes.data, dtype=np.float32) - #logger.info("dtype np float32 applied") - cubes.data += 0 - #cubes.data = np.ma.masked_equal(cubes.data, 0) - - logger.info(cubes.data.dtype) + + # This is the line the ultimately solves things. + # Will leave the other checks above in because they + cubes.data *= 1.0 save_name = f'{out_dir}/OBS_ESACCI-LST_sat_3.00_Amon_{var_name}_{year}{month:02d}.nc' iris.save(cubes, From 8a6de39b4437e95d69588192f87970742bd0ac27 Mon Sep 17 00:00:00 2001 From: morobking Date: Thu, 23 May 2024 16:04:11 +0100 Subject: [PATCH 17/25] Testing, probably wont need --- .../cmorizers/data/formatters/datasets/esacci_lst.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index e7b7028168..492a929cea 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -34,6 +34,9 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): for month in range(1, 13): logger.info(year) logger.info(month) + # test breaks to find all masked data issue + if year != 2003: continue + if month != 1: continue try: cubes = load_cubes(in_dir, vals['file'], @@ -107,7 +110,7 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): # OverflowError: Python int too large to convert to C long error # This fixes it if 'land cover' in cubes.long_name: - logger.info("Got a Land Cover Class cube") + logger.info(f"Got a Land Cover Class cube {cubes.long_name}") cubes.data.fill_value = 0 # Get rid of anything outide 0-255 @@ -118,7 +121,9 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): # This is the line the ultimately solves things. # Will leave the other checks above in because they - cubes.data *= 1.0 + cubes.data = cubes.data * 1.0 + else: + logger.info(f"not a land cover cube {cubes.log_name}") save_name = f'{out_dir}/OBS_ESACCI-LST_sat_3.00_Amon_{var_name}_{year}{month:02d}.nc' iris.save(cubes, From df031554119fc57d62b02901bf75e0f038adc85b Mon Sep 17 00:00:00 2001 From: morobking Date: Tue, 18 Jun 2024 11:59:22 +0100 Subject: [PATCH 18/25] Tidy ready for PR --- .../cmorizers/data/cmor_config/ESACCI-LST.yml | 230 +++++++++--------- .../data/formatters/datasets/esacci_lst.py | 5 +- 2 files changed, 116 insertions(+), 119 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml index 28579d109c..b1b3c65e9c 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml @@ -13,7 +13,7 @@ attributes: reference: 'esacci_lst' comment: '' start_year: 2003 - end_year: 2004 + end_year: 2018 # Variables to cmorize # These go into the vals dictionary in the python script @@ -24,7 +24,7 @@ variables: raw_units: file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' start_year: 2003 - end_year: 2004 + end_year: 2018 tsLCNight: mip: Amon @@ -32,116 +32,116 @@ variables: raw_units: file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' start_year: 2003 - end_year: 2004 - - # tsTotalDay: - # mip: Amon - # raw: land surface temperature total uncertainty - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2004 - - # tsTotalNight: - # mip: Amon - # raw: land surface temperature total uncertainty - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2004 - - # tsVarDay: - # mip: Amon - # raw: land surface temperature variance - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2004 - - # tsVarNight: - # mip: Amon - # raw: land surface temperature variance - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2004 - - # tsDay: - # mip: Amon - # raw: land surface temperature - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2004 - - # tsNight: - # mip: Amon - # raw: land surface temperature - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2004 - - # tsLocalAtmErrDay: - # mip: Amon - # raw: uncertainty from locally correlated errors on atmospheric scales - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2004 - - # tsLocalAtmErrNight: - # mip: Amon - # raw: uncertainty from locally correlated errors on atmospheric scales - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2004 - - # tsLocalSfcErrDay: - # mip: Amon - # raw: uncertainty from locally correlated errors on surface scales - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2004 - - # tsLocalSfcErrNight: - # mip: Amon - # raw: uncertainty from locally correlated errors on surface scales - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2004 - - # tsLSSysErrDay: - # mip: Amon - # raw: uncertainty from large-scale systematic errors - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2004 - - # tsLSSysErrNight: - # mip: Amon - # raw: uncertainty from large-scale systematic errors - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2004 - - # tsUnCorErrDay: - # mip: Amon - # raw: uncertainty from uncorrelated errors - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' - # start_year: 2003 - # end_year: 2004 - - # tsUnCorErrNight: - # mip: Amon - # raw: uncertainty from uncorrelated errors - # raw_units: kelvin - # file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' - # start_year: 2003 - # end_year: 2004 \ No newline at end of file + end_year: 2018 + + tsTotalDay: + mip: Amon + raw: land surface temperature total uncertainty + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 + + tsTotalNight: + mip: Amon + raw: land surface temperature total uncertainty + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 + + tsVarDay: + mip: Amon + raw: land surface temperature variance + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 + + tsVarNight: + mip: Amon + raw: land surface temperature variance + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 + + tsDay: + mip: Amon + raw: land surface temperature + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 + + tsNight: + mip: Amon + raw: land surface temperature + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 + + tsLocalAtmErrDay: + mip: Amon + raw: uncertainty from locally correlated errors on atmospheric scales + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 + + tsLocalAtmErrNight: + mip: Amon + raw: uncertainty from locally correlated errors on atmospheric scales + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 + + tsLocalSfcErrDay: + mip: Amon + raw: uncertainty from locally correlated errors on surface scales + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 + + tsLocalSfcErrNight: + mip: Amon + raw: uncertainty from locally correlated errors on surface scales + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 + + tsLSSysErrDay: + mip: Amon + raw: uncertainty from large-scale systematic errors + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 + + tsLSSysErrNight: + mip: Amon + raw: uncertainty from large-scale systematic errors + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 + + tsUnCorErrDay: + mip: Amon + raw: uncertainty from uncorrelated errors + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-' + start_year: 2003 + end_year: 2018 + + tsUnCorErrNight: + mip: Amon + raw: uncertainty from uncorrelated errors + raw_units: kelvin + file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-' + start_year: 2003 + end_year: 2018 diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index 492a929cea..4fb842dde8 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -34,9 +34,6 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): for month in range(1, 13): logger.info(year) logger.info(month) - # test breaks to find all masked data issue - if year != 2003: continue - if month != 1: continue try: cubes = load_cubes(in_dir, vals['file'], @@ -138,4 +135,4 @@ def load_cubes(in_dir, file, year, month, variable_list): variable_list ) - return cube + return cube \ No newline at end of file From b70455c4e13d9b30bd3493e057c8412631fa6e12 Mon Sep 17 00:00:00 2001 From: morobking Date: Tue, 18 Jun 2024 12:46:41 +0100 Subject: [PATCH 19/25] Update datasets.yml to reflect LST V3 and CEDA as source --- esmvaltool/cmorizers/data/datasets.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml index 757fce7d67..52b042c381 100644 --- a/esmvaltool/cmorizers/data/datasets.yml +++ b/esmvaltool/cmorizers/data/datasets.yml @@ -469,18 +469,18 @@ datasets: ESACCI-LST: tier: 2 - source: On CEDA-JASMIN, /gws/nopw/j04/esacci_lst/public - last_access: 2022-01-26 + source: CEDA + last_access: 2024-06-18 info: | - For access to this JASMIN group workspace please register at: - https://accounts.jasmin.ac.uk/services/group_workspaces/esacci_lst/ + Download the netCDF files from: + https://data.ceda.ac.uk/neodc/esacci/land_surface_temperature/data/AQUA_MODIS/L3C/0.01/v3.00/monthly Download and processing instructions: Put all files under a single directory (no subdirectories with years) in ${RAWOBS}/Tier2/ESACCI-LST BOTH DAY and NIGHT files are needed for each month Caveats - Currently set to work with only the MODIS AQUA L3 monthly data + Currently set to work with only the MODIS AQUA L3 monthly data (MODISA) ESACCI-OC: tier: 2 From 6a6bfd532f59eb1c28c179a232bb4eb2b4b1051a Mon Sep 17 00:00:00 2001 From: morobking Date: Tue, 18 Jun 2024 16:07:53 +0100 Subject: [PATCH 20/25] Reference and check_obs updated --- esmvaltool/recipes/examples/recipe_check_obs.yml | 3 +-- esmvaltool/references/esacci_lst.bibtex | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml index 70368c87dc..117a9eb4d3 100644 --- a/esmvaltool/recipes/examples/recipe_check_obs.yml +++ b/esmvaltool/recipes/examples/recipe_check_obs.yml @@ -219,13 +219,12 @@ diagnostics: scripts: null ESACCI-LST: - # TODO change years to full data coverage description: ESACCI-LST check variables: ts: additional_datasets: - {dataset: ESACCI-LST, project: OBS, mip: Amon, tier: 2, - type: sat, version: "1.00", + type: sat, version: "3.00", start_year: 2003, end_year: 2018} scripts: null diff --git a/esmvaltool/references/esacci_lst.bibtex b/esmvaltool/references/esacci_lst.bibtex index f81ab82f46..1af52baf66 100644 --- a/esmvaltool/references/esacci_lst.bibtex +++ b/esmvaltool/references/esacci_lst.bibtex @@ -1,12 +1,12 @@ @misc{esacci-lst, url = {https://climate.esa.int/en/projects/land-surface-temperature/}, title = {{ESA CCI} {L}and {S}urface {T}emperature}, - year = {2020}, + year = {2022}, note = {The {ESA} {LST}\_cci ({E}uropean {S}pace {A}gency {L}and {S}urface {T}emperature {C}limate {C}hange {I}nitiative) project, is led by the {U}niversity of {L}eicester in the {UK}. The {LST}\_cci project aims to deliver a significant improvement on the capability of current satellite {LST} data records to meet the {G}lobal {C}limate {O}bserving {S}ystem requirements for climate applications and realise the full potential of long-term {LST} data for climate science. - The {LST}\_cci products used here is the {AQUA\_MODIS\_L3C} v1.00 0.05 degree global lon-lat grid monthly product.} + The {LST}\_cci product used here is the {AQUA\_MODIS\_L3C} v3.00 0.01 degree global lon-lat grid monthly product.} } From d58aefabb062dea1d4525008faf58359852c4a7f Mon Sep 17 00:00:00 2001 From: morobking Date: Wed, 19 Jun 2024 12:18:28 +0100 Subject: [PATCH 21/25] Update to address some Codacy issues --- .../cmorizers/data/formatters/datasets/esacci_lst.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index 4fb842dde8..0402e5333b 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -42,7 +42,7 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): vals['raw'], ) except: - logger.info(f'Problem with Month {month} in {year}') + logger.info(f'Problem: Month %s in %s' % (month, year}) continue # make time coords @@ -102,12 +102,12 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): if 'Night' in var_name: cubes.long_name += ' Night' cubes.var_name += '_night' - - # land cover class gives this error when loading in CMORised files + + # Land cover class gives this error when + # loading in CMORised files # OverflowError: Python int too large to convert to C long error # This fixes it if 'land cover' in cubes.long_name: - logger.info(f"Got a Land Cover Class cube {cubes.long_name}") cubes.data.fill_value = 0 # Get rid of anything outide 0-255 @@ -119,8 +119,6 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): # This is the line the ultimately solves things. # Will leave the other checks above in because they cubes.data = cubes.data * 1.0 - else: - logger.info(f"not a land cover cube {cubes.log_name}") save_name = f'{out_dir}/OBS_ESACCI-LST_sat_3.00_Amon_{var_name}_{year}{month:02d}.nc' iris.save(cubes, @@ -135,4 +133,4 @@ def load_cubes(in_dir, file, year, month, variable_list): variable_list ) - return cube \ No newline at end of file + return cube From 616ac0973bea5b2906942636b5710a02a8bde4a7 Mon Sep 17 00:00:00 2001 From: morobking Date: Wed, 19 Jun 2024 12:41:47 +0100 Subject: [PATCH 22/25] Removed unused lst_testing.py --- esmvaltool/diag_scripts/lst/lst_testing.py | 141 --------------------- 1 file changed, 141 deletions(-) delete mode 100644 esmvaltool/diag_scripts/lst/lst_testing.py diff --git a/esmvaltool/diag_scripts/lst/lst_testing.py b/esmvaltool/diag_scripts/lst/lst_testing.py deleted file mode 100644 index 19b306c7e7..0000000000 --- a/esmvaltool/diag_scripts/lst/lst_testing.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -ESMValTool diagnostic for ESA CCI LST data. - -The code uses the all time average monthly data. -The ouptput is a timeseries plot of the mean differnce of -CCI LST to model ensemble average, with the ensemble spread -represented by a standard deviation either side of the mean. -""" - -import logging -import time -import iris -import matplotlib.pyplot as plt -import numpy as np - -from esmvaltool.diag_scripts.shared import ( - ProvenanceLogger, - get_plot_filename, - group_metadata, - run_diagnostic, -) - -logger = logging.getLogger(__name__) - - -def _get_input_cubes(metadata): - """Load the data files into cubes. - - Based on the hydrology diagnostic. - - Inputs: - metadata = List of dictionaries made from the preprocessor config - - Outputs: - inputs = Dictionary of cubes - ancestors = Dictionary of filename information - """ - inputs = {} - ancestors = {} - print('/\/\/\/\/\/\/\/\/\/\/') - for attributes in metadata: - print(attributes) - short_name = attributes['short_name'] - filename = attributes['filename'] - logger.info("Loading variable %s", short_name) - cube = iris.load_cube(filename) - cube.attributes.clear() - inputs[short_name] = cube - ancestors[short_name] = [filename] - print('@@@@@@@@@@@@@@@@@@@@@@') - return inputs, ancestors - - -def _get_provenance_record(attributes, ancestor_files): - """Create the provenance record dictionary. - - Inputs: - attributes = dictionary of ensembles/models used, the region bounds - and years of data used. - ancestor_files = list of data files used by the diagnostic. - - Outputs: - record = dictionary of provenance records. - """ - caption = "Timeseries of ESA CCI LST difference to mean of "\ - + "model ensembles calculated over region bounded by latitude "\ - + "{lat_south} to {lat_north}, longitude {lon_west} to {lon_east} "\ - + "and for model/ensembles {ensembles}. "\ - + "Shown for years {start_year} to {end_year}.".format(**attributes) - - record = { - 'caption': caption, - 'statistics': ['mean', 'stddev'], - 'domains': ['reg'], - 'plot_types': ['times'], - 'authors': ['king_robert'], - # 'references': [], - 'ancestors': ancestor_files - } - - return record - - -def _diagnostic(config): - """Perform the control for the ESA CCI LST diagnostic. - - Parameters - ---------- - config: dict - the preprocessor nested dictionary holding - all the needed information. - - Returns - ------- - figures made by make_plots. - """ - # this loading function is based on the hydrology diagnostic - input_metadata = config['input_data'].values() - print('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') - print(input_metadata) - print('bbbbbbbbbbbbbbbbb') - loaded_data = {} - ancestor_list = [] - print('ccccccccccccccccccccccccccccccccccccccccccccccccc') - print(group_metadata(input_metadata, 'dataset')) - print('dddddddddddddddddddddddddddddddddddddddddddddddddddddddd') - for dataset, metadata in group_metadata(input_metadata, 'dataset').items(): - print(dataset, metadata) - cubes, ancestors = _get_input_cubes(metadata) - print(cubes, ancestors) - loaded_data[dataset] = cubes - # ancestor_list.append(ancestors['ts'][0]) - - print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$') - print(ancestor_list) - print('*************************************') - - # loaded data is a nested dictionary - # KEY1 model ESACCI-LST or something else - # KEY2 is ts, the surface temperature - # ie loaded_data['ESACCI-LST']['ts'] is the CCI cube - # loaded_data['MultiModelMean']['ts'] is CMIP6 data, emsemble means - # similarly dor Std, see preprocessor - - # The Diagnostic uses CCI - MODEL - print('###########################################') - print(loaded_data) - print('##########################################') - - START = time.time() - for KEY in loaded_data['ESACCI-LST'].keys(): - xyz = np.mean(loaded_data['ESACCI-LST'][KEY].data) - print(f'{KEY} {xyz}') - print(f'TIME {time.time()-START}') - - -if __name__ == '__main__': - # always use run_diagnostic() to get the config (the preprocessor - # nested dictionary holding all the needed information) - with run_diagnostic() as config: - _diagnostic(config) From 94726a4e9a21f05390d125a5a43887b0434c2c93 Mon Sep 17 00:00:00 2001 From: morobking Date: Wed, 19 Jun 2024 12:55:43 +0100 Subject: [PATCH 23/25] Codacy fixes --- .../data/formatters/datasets/esacci_lst.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index 0402e5333b..813d5afa6d 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -40,9 +40,9 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): year, month, vals['raw'], - ) + ) except: - logger.info(f'Problem: Month %s in %s' % (month, year}) + logger.info(f'Problem: Month %s in %s' % (month, year)) continue # make time coords @@ -104,23 +104,25 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): cubes.var_name += '_night' # Land cover class gives this error when - # loading in CMORised files - # OverflowError: Python int too large to convert to C long error - # This fixes it + # loading in CMORised files + # OverflowError: + # Python int too large to convert to C long error + # This fixes it: if 'land cover' in cubes.long_name: cubes.data.fill_value = 0 # Get rid of anything outide 0-255 - cubes.data[np.where(cubes.data<0)] = 0 - cubes.data[np.where(cubes.data>255)] = 0 + cubes.data[np.where(cubes.data < 0)] = 0 + cubes.data[np.where(cubes.data > 255)] = 0 cubes.data = cubes.data.filled() - + # This is the line the ultimately solves things. # Will leave the other checks above in because they cubes.data = cubes.data * 1.0 - save_name = f'{out_dir}/OBS_ESACCI-LST_sat_3.00_Amon_{var_name}_{year}{month:02d}.nc' + save_name = f'{out_dir}/OBS_ESACCI-LST_sat_3.00_Amon_' + \ + '{var_name}_{year}{month:02d}.nc' iris.save(cubes, save_name ) From 1043ab215a2e48125cc2ddc8aae597a54f4ef898 Mon Sep 17 00:00:00 2001 From: morobking Date: Fri, 21 Jun 2024 18:52:12 +0100 Subject: [PATCH 24/25] f string error fixed --- esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py index 813d5afa6d..e61d818f6e 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py @@ -122,7 +122,7 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): cubes.data = cubes.data * 1.0 save_name = f'{out_dir}/OBS_ESACCI-LST_sat_3.00_Amon_' + \ - '{var_name}_{year}{month:02d}.nc' + f'{var_name}_{year}{month:02d}.nc' iris.save(cubes, save_name ) From 234a21b498f59556b8384ca7c9d7e8df60124870 Mon Sep 17 00:00:00 2001 From: Axel Lauer Date: Tue, 25 Jun 2024 13:53:06 +0200 Subject: [PATCH 25/25] added downloader for ESACCI-LST --- .../data/downloaders/datasets/esacci_lst.py | 90 +++++++++++++++++++ esmvaltool/cmorizers/data/downloaders/ftp.py | 11 ++- 2 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 esmvaltool/cmorizers/data/downloaders/datasets/esacci_lst.py diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_lst.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_lst.py new file mode 100644 index 0000000000..a3db01fd65 --- /dev/null +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_lst.py @@ -0,0 +1,90 @@ +"""Script to download ESACCI-LST from CCI CEDA ftp.""" + +import logging + +from datetime import datetime + +from dateutil import relativedelta + +from esmvaltool.cmorizers.data.downloaders.ftp import CCIDownloader + +logger = logging.getLogger(__name__) + + +def download_dataset(config, dataset, dataset_info, start_date, end_date, + overwrite): + """Download dataset. + + Parameters + ---------- + config : dict + ESMValTool's user configuration + dataset : str + Name of the dataset + dataset_info : dict + Dataset information from the datasets.yml file + start_date : datetime + Start of the interval to download + end_date : datetime + End of the interval to download + overwrite : bool + Overwrite already downloaded files + """ + if start_date is None: + start_date = datetime(2003, 1, 1) + if end_date is None: + end_date = datetime(2018, 12, 31) + loop_date = start_date + + downloader = CCIDownloader( + config=config, + dataset=dataset, + dataset_info=dataset_info, + overwrite=overwrite, + dataset_name='land_surface_temperature', + ) + downloader.connect() + + # download daily data + +# loop_date = start_date +# rel_base_dir = 'AQUA_MODIS/L3C/0.01/v3.00/daily' +# while loop_date <= end_date: +# downloader.set_cwd(rel_base_dir) +# if downloader.exists(f'{loop_date.year}'): +# downloader.set_cwd(f'{rel_base_dir}/{loop_date.year}') +# if downloader.exists(f'{loop_date.month:02}'): +# downloader.set_cwd(f'{rel_base_dir}/{loop_date.year}/' +# f'{loop_date.month:02}') +# if downloader.exists(f'{loop_date.day:02}'): +# downloader.download_folder(f'{loop_date.day:02}', +# sub_folder=f'{loop_date.year}_daily') +# else: +# logger.info('%d/%d/%d: no data found', loop_date.year, +# loop_date.month, loop_date.day) +# loop_date += relativedelta.relativedelta(days=1) +# else: +# logger.info('%d/%d: no data found', loop_date.year, +# loop_date.month) +# loop_date += relativedelta.relativedelta(months=1) +# else: +# logger.info('%d: no data found', loop_date.year) +# loop_date += relativedelta.relativedelta(years=1) + + # download monthly data + + loop_date = start_date + rel_base_dir = 'AQUA_MODIS/L3C/0.01/v3.00/monthly' + while loop_date <= end_date: + downloader.set_cwd(rel_base_dir) + if downloader.exists(f'{loop_date.year}'): + downloader.set_cwd(f'{rel_base_dir}/{loop_date.year}') + if downloader.exists(f'{loop_date.month:02}'): + downloader.download_folder(f'{loop_date.month:02}') + else: + logger.info('%d/%d: no data found', loop_date.year, + loop_date.month) + loop_date += relativedelta.relativedelta(months=1) + else: + logger.info('%d: no data found', loop_date.year) + loop_date += relativedelta.relativedelta(years=1) diff --git a/esmvaltool/cmorizers/data/downloaders/ftp.py b/esmvaltool/cmorizers/data/downloaders/ftp.py index 9f0cd5e8f9..533a5dca63 100644 --- a/esmvaltool/cmorizers/data/downloaders/ftp.py +++ b/esmvaltool/cmorizers/data/downloaders/ftp.py @@ -176,11 +176,18 @@ class CCIDownloader(FTPDownloader): Dataset information from the datasets.yml file overwrite : bool Overwrite already downloaded files + dataset_name : str + optional override for directory name on CEDA within + 'esacci' folder """ - def __init__(self, config, dataset, dataset_info, overwrite): + def __init__(self, config, dataset, dataset_info, overwrite, + dataset_name=None): super().__init__(config, 'anon-ftp.ceda.ac.uk', dataset, dataset_info, overwrite) - self.ftp_name = self.dataset_name[7:] + if dataset_name: + self.ftp_name = dataset_name + else: + self.ftp_name = self.dataset_name[7:] def set_cwd(self, path): """Set current work directory.