Dashboard version 1.0, datatable and murakami plot (#61)

Co-authored-by: Niko Sirmpilatze <[email protected]>
Co-authored-by: Joe Ziminski <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

MANIFEST.in

@@ -5,6 +5,7 @@ include *.py
 recursive-include docs *.md
 recursive-include docs *.png
 recursive-include docs *.svg
+recursive-include rsp_vision *.css
 
 exclude *.yml
 exclude *.cruft.json

README.md

@@ -1,6 +1,14 @@
 # RSP vision
 Tools for analyzing responses of visual neurons to drifting gratings, based on the original work of Lee Cossell.
 
+## Get Started Quickly
+In your environment...
+Install: `pip install .`
+Make config file: `python3 setup_for_demo.py`
+Edit config file and insert your path: `nano rsp_vision/config/config.yml`
+Run analysis: `python3 demo_cli.py`
+Run dashboard: `python3 demo_dash.py`
+
 ## Installation
 
 Clone the repository and install the dependencies in your environment with:
@@ -23,6 +31,11 @@ Please edit the `allen_dff` path to point to the folder where you stored the pre
 
 Finally, run `python3 demo_cli.py` to run the analysis. The script will create a file containing the analysis output which will then be used by the dashboard.
 
+### Run the analysis
+The script `demo_cli.py` runs the analysis and stores the output. You can start it with:
+```bash
+python3 demo_cli.py
+```
 ### Data processing
 
 The original data is stored as a nested dictionary, usually referred to as the data_raw attribute. It contains the following keys: `day`, `imaging`, `f`, `is_cell`, `r_neu`, `stim`, `trig`. For our analysis, we focus mainly on `f` and `stim`.
@@ -97,4 +110,32 @@ After the fit, the Gaussian is sampled to generate a 6x6 and a 100x100 matrix, w
 
 ### Schema of the current analysis pipeline
 A schematic representation of the process is shown below:
-![Responsiveness analysis diagram](https://github.com/neuroinformatics-unit/rsp-vision/assets/29216006/e4af3107-2f3b-431f-98f7-8ead20e3de04)
+<img src="./docs/Responsiveness analysis diagram.svg">
+
+### Data saving
+The output of the analysis is saved in the folder specified in the config file as `path:output`. Folder names are generated automatically according to [SWC_Blueprint specifications](https://swc-blueprint.neuroinformatics.dev/). Filenames do not follow the convention right now.
+
+Schematic representation of the output folder structure:
+```
+rsp_vision
+├── logs/
+│   ├── log_file_1.log
+├── devirvatives/
+│   ├── analysis_log.csv
+│   ├── subject_level_folder/
+│   │   ├── session_level_folder/
+│   │   │   ├── metadata.yml
+│   │   │   ├── roi_1.pkl
+```
+The files are currently saved as pickle files. Each session folder, in addition to the pickle files, contains a `metadata.yml` file, with the parameters that were used for that analysis.
+
+In the same project folder, you will find in addition to the `deirvatives` folder, a `logs` folder, where the logs of the analysis are saved. Inside the derivatives folder, you will find the `analysis_log.csv` file, a table containing the records of which datasets have already been analyised. This file is updated at the end of each analysis.
+
+Check `rsp_vision/save/save_data.py` and `rsp_vision/objects/SWC_Blueprint.py` for more details.
+
+## Dashboard
+You can browse the analysed data via a Dash app. The app is currently in development, but you can already run it.
+Run the following command in the terminal:
+```
+python3 demo_dash.py
+```

demo_dash.py

@@ -0,0 +1,5 @@
+from rsp_vision.dashboard.app import app
+
+if __name__ == "__main__":
+    # app = get_app()
+    app.run_server(debug=True)

pyproject.toml

@@ -31,6 +31,9 @@ dependencies = [
     "numba",
     "tqdm",
     "GitPython",
+    "dash",
+    "dash_mantine_components",
+    "dash_bootstrap_components",
 ]
 
 [project.urls]
@@ -52,6 +55,8 @@ dev = [
   "ruff",
   "setuptools_scm",
   "check-manifest",
+  "pytest-mock",
+  "GitPython",
 ]
 
 [build-system]

rsp_vision/analysis/spatial_freq_temporal_freq.py

@@ -634,7 +634,7 @@ def manage_fitting(
 
         if best_result is None:
             logging.warning(
-                f"ROI {roi_id} and direction {dir} failed to fit."
+                f"ROI {roi_id} and direction {direction} failed to fit."
                 + "Skipping..."
             )
             best_result = OptimizeResult()

rsp_vision/dashboard/app.py

@@ -0,0 +1,48 @@
+# This file is used to create the Dash app and the scaffold layout.
+import dash_mantine_components as dmc
+from dash import Dash, dcc, html, page_container
+
+# I am using Open Sans font from Google Fonts for the whole app.
+# The design of the app is controlled by the CSS file in
+# rsp_vision/dashboard/assets/style.css.
+google_fonts_link = (
+    "https://fonts.googleapis.com/css2?family=Open+Sans&display=swap"
+)
+external_stylesheets = [
+    {
+        "href": google_fonts_link,
+        "rel": "stylesheet",
+    }
+]
+# fmt: off
+app = Dash(
+    __name__,
+    use_pages=True,
+    external_stylesheets=external_stylesheets,
+    suppress_callback_exceptions=True,
+)
+# fmt: on
+
+header = dmc.Header(
+    height=70,
+    children=[
+        dmc.Title(
+            "RSP vision 👁️",
+            order=1,
+            className="main-title",
+        )
+    ],
+    className="header",
+)
+
+# Here I define the layout of the app. The layout is a composition of
+# Mantine components and Dash components.
+# Importantly, here I define the `Store` component, which is fundamental
+# to share information between pages.
+app.layout = html.Div(
+    [
+        dcc.Store(id="store", data={}),
+        header,
+        page_container,
+    ]
+)

rsp_vision/dashboard/assets/styles.css

@@ -0,0 +1,82 @@
+.header {
+    background-color: #0c0223;
+    margin: auto;
+    width: 100%;
+    height: 50px;
+}
+
+.main-title {
+  color: #ffffff;
+  text-align: center;
+}
+
+.page-title{
+    color: #0c0223;
+    text-align: center;
+}
+
+.page{
+    padding: 1%;
+}
+
+.table{
+    padding: 2%;
+}
+
+.selected-data-container{
+    width: 40%;
+    padding: 1%;
+    margin: auto;
+}
+
+.load-data-button{
+    margin: 1%;
+
+    border-radius: 8px;
+    border-width: 0;
+    text-align: center;
+}
+
+a:-webkit-any-link {
+    text-align: center;
+    text-decoration: none !important;
+    text-rendering: auto;
+    text-transform: none;
+    text-indent: 0px;
+    text-shadow: none;
+
+    letter-spacing: normal;
+    word-spacing: normal;
+    line-height: normal;
+    font: -webkit-small-control;
+
+    color: rgba(0, 0, 0);
+    border-color: rgba(0, 0, 0, 0.3);
+
+    appearance: auto;
+    display: inline-block;
+    align-items: flex-start;
+    cursor: default;
+    margin: 1%;
+    padding: 1px 6px;
+
+    box-sizing: border-box;
+    border-radius: 8px;
+    border-width: 0;
+    border-width: 2px;
+    border-style: outset;
+    border-color: buttonborder;
+    border-image: initial;
+}
+
+.responsive-switch{
+    margin: 5%;
+}
+
+.responsive-switch-text{
+    margin: 5%;
+}
+
+.show-hide {
+    display: none;
+}

rsp_vision/dashboard/pages/data_table.py

@@ -0,0 +1,154 @@
+from pathlib import Path
+
+import dash_bootstrap_components as dbc
+import dash_mantine_components as dmc
+import pandas as pd
+from dash import Input, Output, callback, dash_table, html, register_page
+from decouple import config
+
+from rsp_vision.load.load_data import read_config_file
+from rsp_vision.objects.SWC_Blueprint import (
+    SessionFolder,
+    SubjectFolder,
+    SWC_Blueprint_Spec,
+)
+
+# The following code lives outside of the callback
+# because it is executed only once, when the app starts
+# It reads the config file and creates the SWC_Blueprint_Spec object
+# that is used to load the data.
+CONFIG_PATH = config("CONFIG_PATH")
+config_path = Path(__file__).parents[2] / CONFIG_PATH
+config = read_config_file(config_path)
+swc_blueprint_spec = SWC_Blueprint_Spec(
+    project_name="rsp_vision",
+    raw_data=False,
+    derivatives=True,
+    local_path=Path(config["paths"]["output"]),
+)
+with open(swc_blueprint_spec.path / "analysis_log.csv", "r") as file:
+    col = file.readline().split(",")
+    col[0] = "index"
+    columns = [{"name": idx, "id": idx} for idx in col]
+    dataframe = pd.read_csv(file, names=col, index_col=0)
+    data = dataframe.to_dict(orient="records")
+
+
+register_page(__name__, path="/")
+
+
+layout = html.Div(
+    [
+        dmc.Title(
+            "Select dataset to be loaded 👇",
+            order=2,
+            className="page-title",
+        ),
+        dmc.Container(
+            children=[
+                dmc.Text(
+                    id="selected_data_str",
+                ),
+                dbc.Button(
+                    "Load selected dataset ✨",
+                    id="button",
+                    className="load-data-button",
+                    href="/murakami_plot",
+                    n_clicks=0,
+                ),
+                html.Br(),
+                html.Br(),
+                dash_table.DataTable(
+                    id="table",
+                    columns=columns,
+                    data=data,
+                    editable=False,
+                    filter_action="native",
+                    sort_action="native",
+                    sort_mode="multi",
+                    row_selectable="single",
+                    row_deletable=False,
+                    selected_columns=[],
+                    selected_rows=[],
+                    page_action="native",
+                    page_current=0,
+                    page_size=20,
+                    hidden_columns=[
+                        "index",
+                        "sub",
+                        "ses",
+                        "mouse_line",
+                        "mouse_id",
+                        "hemisphere",
+                        "brain_region",
+                        "monitor_position",
+                        "fov",
+                        "cre",
+                        "analysed",
+                        "commit_hash",
+                        "microscope",
+                    ],
+                ),
+            ],
+            className="table",
+        ),
+    ],
+    className="page",
+)
+
+
+@callback(
+    [
+        Output("selected_data_str", "children"),
+        Output("store", "data"),
+        Output("button", "disabled"),
+    ],
+    Input("table", "selected_rows"),
+)
+def update_storage(selected_rows: list) -> tuple:
+    """This callback is triggered when the user selects a row in the table.
+    It updates the `Store` component with the information about the selected
+    row, i.e. the dataset that the user wants to load.
+
+    Parameters
+    ----------
+    selected_rows : list
+        List of selected rows in the Dash table.
+
+    Returns
+    -------
+    tuple
+        A tuple containing the following elements:
+        - A string with the name of the selected dataset.
+        - A dictionary containing the information about the selected dataset.
+        - A boolean indicating whether the button for loading the data should
+        be disabled or not.
+    """
+    if selected_rows is None or len(selected_rows) == 0:
+        return "No row selected 🤷🏻‍♀️", {}, True
+
+    else:
+        sub_folder = SubjectFolder(
+            swc_blueprint_spec,
+            dataframe.iloc[selected_rows[0]].to_dict(),
+        )
+        session_folder = SessionFolder(
+            sub_folder,
+            dataframe.iloc[selected_rows[0]].to_dict(),
+        )
+
+        store = {
+            "data": dataframe.iloc[selected_rows[0]],
+            "path": str(swc_blueprint_spec.path),
+            "config": config,
+            "subject_folder_path": str(sub_folder.sub_folder_path),
+            "session_folder_path": str(session_folder.ses_folder_path),
+        }
+
+        folder_name = dataframe.iloc[selected_rows[0]]["folder_name"]
+
+        return (
+            f"Dataset selected: {folder_name}",
+            store,
+            False,
+        )