From 7dd307290d8ebcff4717f8ddf5f49a228086629d Mon Sep 17 00:00:00 2001 From: rizac Date: Sat, 23 Nov 2024 09:18:43 +0100 Subject: [PATCH] return residuals plots also for non-supported residual type AND imt --- egsim/app/forms.py | 80 +++++++++++++++++++++------------- tests/django/test_app_views.py | 37 +++++++++++++--- 2 files changed, 82 insertions(+), 35 deletions(-) diff --git a/egsim/app/forms.py b/egsim/app/forms.py index d6cdaf31..cb88def2 100644 --- a/egsim/app/forms.py +++ b/egsim/app/forms.py @@ -1,4 +1,6 @@ """Forms handling data (flatfiles)""" +from itertools import product + import numpy as np import pandas as pd from scipy.stats import linregress @@ -267,21 +269,26 @@ def output(self) -> dict: inter_res: 'Inter event' } - for col in dataframe.columns: - if col[1] not in res_columns: - continue - imt, res_type, model = col - - if not col_x: # histogram (residuals or LH): - x = dataframe[col] - y = None - else: - x = self.cleaned_data['flatfile'][col_x] - y = dataframe[col] - - color = colors.setdefault(model, next(c_cycle)) - data, layout = self.get_plot_traces_and_layout(model, imt, x, y, - likelihood, col_x, color) + for imt, res_type, model in product( + self.cleaned_data['imt'], + res_columns, + self.cleaned_data['gsim'] + ): + col = (imt, res_type, model) + data = [{}] + layout = None + + if col in dataframe.columns: + if not col_x: # histogram (residuals or LH): + x = dataframe[col] + y = None + else: + x = self.cleaned_data['flatfile'][col_x] + y = dataframe[col] + + color = colors.setdefault(model, next(c_cycle)) + data, layout = self.get_plot_traces_and_layout(model, imt, x, y, + likelihood, col_x, color) # provide a key that is comparable for sorting the plots. Note that imt # is separated into name and period (so that "SA(9)" < "SA(10)") and that @@ -302,21 +309,34 @@ def output(self) -> dict: 'layout': layout } - # if we are processing total residuals, also set intra and inter - # defaults as empty plot. If intra and inter were already (or will be ) - # processed, the skip this - if res_type == total_res: - for r_type in (intra_res, inter_res): - key[-1] = r_type - plots.setdefault(tuple(key), { - 'data': [{}], - 'params': { - 'model': model, - 'imt': imt, - 'residual type': residual_label[r_type] - }, - 'layout': dict(layout) - }) + for key, plot in plots.items(): + if plot['layout'] is not None: + continue + layout_to_copy = {} + for key_, plot_ in plots.items(): + if plot_['layout'] is None: + continue + if plot['params']['imt'] == plot_['params']['imt']: + layout_to_copy = dict(plot_['layout']) + if plot['params']['model'] == plot_['params']['model']: + break + plot['layout'] = layout_to_copy + + # # if we are processing total residuals, also set intra and inter + # # defaults as empty plot. If intra and inter were already (or will be ) + # # processed, the skip this + # if res_type == total_res: + # for r_type in (intra_res, inter_res): + # key[-1] = r_type + # plots.setdefault(tuple(key), { + # 'data': [{}], + # 'params': { + # 'model': model, + # 'imt': imt, + # 'residual type': residual_label[r_type] + # }, + # 'layout': dict(layout) + # }) # return keys sorted so that the frontend displays them accordingly: return {'plots': [plots[key] for key in sorted(plots.keys())]} diff --git a/tests/django/test_app_views.py b/tests/django/test_app_views.py index 4551214f..608e1300 100644 --- a/tests/django/test_app_views.py +++ b/tests/django/test_app_views.py @@ -29,6 +29,9 @@ from egsim.app.views import URLS, img_ext, form2dict from django.test.client import Client +from egsim.smtk import get_sa_limits +from egsim.smtk.registry import Clabel + GSIM, IMT = 'gsim', 'imt' @@ -317,20 +320,44 @@ def test_residuals_visualize_no_missing_plots(self): """ client = Client() # SgobbaEtAl not defined for the given SA, AbrahmsonSilva only for total + models = ['SgobbaEtAl2020', 'AbrahamsonSilva1997'] + imts = ['SA(0.1)', 'PGA'] data = { - 'model': ['SgobbaEtAl2020', 'AbrahamsonSilva1997'], - 'imt': ['SA(0.1)'], + 'model': models, + 'imt': imts, 'flatfile': 'esm2018', 'flatfile-query': 'mag > 7' } - response1 = client.post(f"{URLS.RESIDUALS}", - json.dumps(data), + saLim1 = get_sa_limits('SgobbaEtAl2020') + salim2 = get_sa_limits('AbrahamsonSilva1997') + response1 = client.post(f"/{URLS.RESIDUALS}.csv", + json.dumps(data | {'format': 'csv'}), content_type="application/json") - content = pd.read_csv(BytesIO(response1.content())) + content = b''.join(response1.streaming_content) + dframe = pd.read_csv(BytesIO(content), index_col=0, header=0) + assert f'SA(0.1) {Clabel.total_res} SgobbaEtAl2020' not in dframe.columns + assert f'SA(0.1) {Clabel.intra_ev_res} AbrahamsonSilva1997' not in dframe.columns + assert f'PGA {Clabel.total_res} SgobbaEtAl2020' in dframe.columns + assert f'SA(0.1) {Clabel.total_res} AbrahamsonSilva1997' in dframe.columns + # 3 imts for SgobbaEtAl, 2 for Abrahamson et al: + len_c = len([c for c in dframe.columns if not c.startswith(f'{Clabel.input} ')]) + assert len_c == 5 response = client.post(f"/{URLS.RESIDUALS_VISUALIZE}", json.dumps(data), content_type="application/json") + assert response.status == 200 + expected_params = set( + product(imts, models, ('Total', 'Inter event', 'Intra event')) + ) + json_c = response.json() + plots = json_c['plots'] + actual_params = { + (c['params']['imt'], c['params']['model'], c['params']['residual type']) + for c in plots + } + assert len(actual_params) == 12 + assert expected_params == actual_params def test_download_response_img_formats(self): for url, ext in product((URLS.PREDICTIONS_PLOT_IMG, URLS.RESIDUALS_PLOT_IMG),