return residuals plots also for non-supported residual type AND imt

egsim/app/forms.py

@@ -1,4 +1,6 @@
 """Forms handling data (flatfiles)"""
+from itertools import product
+
 import numpy as np
 import pandas as pd
 from scipy.stats import linregress
@@ -267,21 +269,26 @@ def output(self) -> dict:
             inter_res: 'Inter event'
         }
 
-        for col in dataframe.columns:
-            if col[1] not in res_columns:
-                continue
-            imt, res_type, model = col
-
-            if not col_x:  # histogram (residuals or LH):
-                x = dataframe[col]
-                y = None
-            else:
-                x = self.cleaned_data['flatfile'][col_x]
-                y = dataframe[col]
-
-            color = colors.setdefault(model, next(c_cycle))
-            data, layout = self.get_plot_traces_and_layout(model, imt, x, y,
-                                                           likelihood, col_x, color)
+        for imt, res_type, model in product(
+            self.cleaned_data['imt'],
+            res_columns,
+            self.cleaned_data['gsim']
+        ):
+            col = (imt, res_type, model)
+            data = [{}]
+            layout = None
+
+            if col in dataframe.columns:
+                if not col_x:  # histogram (residuals or LH):
+                    x = dataframe[col]
+                    y = None
+                else:
+                    x = self.cleaned_data['flatfile'][col_x]
+                    y = dataframe[col]
+
+                color = colors.setdefault(model, next(c_cycle))
+                data, layout = self.get_plot_traces_and_layout(model, imt, x, y,
+                                                               likelihood, col_x, color)
 
             # provide a key that is comparable for sorting the plots. Note that imt
             # is separated into name and period (so that "SA(9)" < "SA(10)") and that
@@ -302,21 +309,34 @@ def output(self) -> dict:
                 'layout': layout
             }
 
-            # if we are processing total residuals, also set intra and inter
-            # defaults as empty plot. If intra and inter were already (or will be )
-            # processed, the  skip this
-            if res_type == total_res:
-                for r_type in (intra_res, inter_res):
-                    key[-1] = r_type
-                    plots.setdefault(tuple(key), {
-                        'data': [{}],
-                        'params': {
-                            'model': model,
-                            'imt': imt,
-                            'residual type': residual_label[r_type]
-                        },
-                        'layout': dict(layout)
-                    })
+        for key, plot in plots.items():
+            if plot['layout'] is not None:
+                continue
+            layout_to_copy = {}
+            for key_, plot_ in plots.items():
+                if plot_['layout'] is None:
+                    continue
+                if plot['params']['imt'] == plot_['params']['imt']:
+                    layout_to_copy = dict(plot_['layout'])
+                    if plot['params']['model'] == plot_['params']['model']:
+                        break
+            plot['layout'] = layout_to_copy
+
+            # # if we are processing total residuals, also set intra and inter
+            # # defaults as empty plot. If intra and inter were already (or will be )
+            # # processed, the  skip this
+            # if res_type == total_res:
+            #     for r_type in (intra_res, inter_res):
+            #         key[-1] = r_type
+            #         plots.setdefault(tuple(key), {
+            #             'data': [{}],
+            #             'params': {
+            #                 'model': model,
+            #                 'imt': imt,
+            #                 'residual type': residual_label[r_type]
+            #             },
+            #             'layout': dict(layout)
+            #         })
 
         # return keys sorted so that the frontend displays them accordingly:
         return {'plots': [plots[key] for key in sorted(plots.keys())]}

tests/django/test_app_views.py

@@ -29,6 +29,9 @@
 from egsim.app.views import URLS, img_ext, form2dict
 from django.test.client import Client
 
+from egsim.smtk import get_sa_limits
+from egsim.smtk.registry import Clabel
+
 GSIM, IMT = 'gsim', 'imt'
 
 
@@ -317,20 +320,44 @@ def test_residuals_visualize_no_missing_plots(self):
         """
         client = Client()
         # SgobbaEtAl not defined for the given SA, AbrahmsonSilva only for total
+        models = ['SgobbaEtAl2020', 'AbrahamsonSilva1997']
+        imts = ['SA(0.1)', 'PGA']
         data = {
-            'model': ['SgobbaEtAl2020', 'AbrahamsonSilva1997'],
-            'imt': ['SA(0.1)'],
+            'model': models,
+            'imt': imts,
             'flatfile': 'esm2018',
             'flatfile-query': 'mag > 7'
         }
-        response1 = client.post(f"{URLS.RESIDUALS}",
-                                json.dumps(data),
+        saLim1 = get_sa_limits('SgobbaEtAl2020')
+        salim2 = get_sa_limits('AbrahamsonSilva1997')
+        response1 = client.post(f"/{URLS.RESIDUALS}.csv",
+                                json.dumps(data | {'format': 'csv'}),
                                 content_type="application/json")
-        content = pd.read_csv(BytesIO(response1.content()))
+        content = b''.join(response1.streaming_content)
+        dframe = pd.read_csv(BytesIO(content), index_col=0, header=0)
+        assert f'SA(0.1) {Clabel.total_res} SgobbaEtAl2020' not in dframe.columns
+        assert f'SA(0.1) {Clabel.intra_ev_res} AbrahamsonSilva1997' not in dframe.columns
+        assert f'PGA {Clabel.total_res} SgobbaEtAl2020' in dframe.columns
+        assert f'SA(0.1) {Clabel.total_res} AbrahamsonSilva1997' in dframe.columns
+        # 3 imts for SgobbaEtAl, 2 for Abrahamson et al:
+        len_c = len([c for c in dframe.columns if not c.startswith(f'{Clabel.input} ')])
+        assert len_c == 5
 
         response = client.post(f"/{URLS.RESIDUALS_VISUALIZE}",
                                json.dumps(data),
                                content_type="application/json")
+        assert response.status == 200
+        expected_params = set(
+            product(imts, models, ('Total', 'Inter event', 'Intra event'))
+        )
+        json_c = response.json()
+        plots = json_c['plots']
+        actual_params = {
+            (c['params']['imt'], c['params']['model'], c['params']['residual type'])
+            for c in plots
+        }
+        assert len(actual_params) == 12
+        assert expected_params == actual_params
 
     def test_download_response_img_formats(self):
         for url, ext in product((URLS.PREDICTIONS_PLOT_IMG, URLS.RESIDUALS_PLOT_IMG),