From 66128c57d4f8e1369525d5675d2e76aca0b0e57d Mon Sep 17 00:00:00 2001 From: aranega Date: Mon, 15 Jul 2024 07:51:01 -0600 Subject: [PATCH 01/49] CELE-57 Add first format ingestion documentation --- format-ingestion.md | 121 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 format-ingestion.md diff --git a/format-ingestion.md b/format-ingestion.md new file mode 100644 index 00000000..f3ae3613 --- /dev/null +++ b/format-ingestion.md @@ -0,0 +1,121 @@ +# Format of data ingested in the database + +The management script is able to ingest data represented in a JSON format. +Different files are necessary: + +* `neurons.json` that encodes the information about the neurons in general +* `datasets.json` that encodes the information about the different datasets +* `connections/xxx.json` that encodes the different connections for dedicated datasets +* `annotations/xxx.json` that encodes annotatinos for different zones of the anatomy + +Those files are automatically exported from third-party tool and shouldn't be edited manually. + +## Format of `neurons.json` + +This file defines a list of JSON object as root structure: + +```json +[ + { + ... // definition of neuron 1 + }, + ..., + { + ... // definition of neuron N + } +] +``` + +Each JSON object represents a neuron with this schema: + +```json +{ + "inhead": int, // int used as bool, is the neuron part of the head or not + "name": string, // name of the neuron, can be same as classes, or L or R of classes + "emb": int, // int used as bool + "nt": string, // neurotransmitter type + "intail": int, // int used as bool + "classes": string, // general name of the neuron + "typ": string // type of the neuron: "i" => , TODO fillme +} +``` + + +## Format of `datasets.json` + +This file defines a list of JSON object as root structure. + +```json +[ + { + ... // definition of dataset 1 + }, + ..., + { + ... // definition of dataset N + } +] +``` + +Each JSON object represents a specific dataset with this schema: + +```json +{ + "id": string // unique ID for the dataset + "name": string // display name of the dataset + "type": string // type of dataset: "complete" or "head" + "time": int // time of the dataset + "visualTime": int // visualTime of the dataset + "description": string // description of the dataset + "axes": [ // OPTIONAL: different axes and their representation, not used but can appear in the file + ... + ] +} +``` + +## Format of `connections/xxx.json` + +The `connections` directory encodes the information about the different connections by dataset. +Each file in this directory is named after the `id` of a dataset present in the `datasets.json` file, e.g.: a dataset defined using the `id` `white_1986_jsh` will defines each of the connections of the dataset in the file `connections/white_1986_jsh.json`. + +Each of those files is a list of JSON object where each of the JSON objects encodes different connections between different neurons. +The schema is the following: + +```json +{ + "ids": [ ... ], // a list of int, where each int represents the ID of the neurons involved in this connection + "post": string, // the name of a neuron as defined in "neurons.json" + "post_tid": [ ... ], // a list of int where each int represents the ID of a post synapse for a dedicated post neuron + "pre": string, // the name of a neuron as defined in "neurons.json" + "pre_tid": [ ... ], // a list of int where each int represents the ID of a pre synapse for a dedicated pre neuron + "syn": [ ... ], // a list of int where each int represents the weight of a post or pre synapses (indice matches the neuron in pre/post_tid) + "typ": int // the type of connection ("electrical" or "chemical") +} +``` + +For each of those objects: `ids`, `post_tid`, `pre_tid` and `syn` need to have the same number of elements. + +## Format of `annotations/xxx.json` + +The `annotations` directory encodes annotations about the different part (`head` or `complete`) following the naming convention `part.annotations.json`, e.g.: the annotations for the `head` are located in `annotations/head.annotations.json`. + +Each of those files is a JSON object that defines categories as keys and a list of neurons couples as values. +Here is the schema for the `head.annotations.json` file (the `complete.annotations.json` file, while existing, is an empty JSON object). + +```json +{ + "increase": [ // the type of annotation + [ + string, // pre, the ID/name of a neuron from "neurons.json" + string // post, the ID/name of the other neuron from "neurons.json" that is part of the couple + ] + ] +} +``` + +The types of annotations can be `increase`, `variable`, `postembryonic`, `decrease` or `stable` + +### Note: + +The existing repository contains a `trajectories` folder with a set of JSON files. +Those files are not ingested anymore, they are part of a legacy system. From cd001901b8dd3d6c1ade8b5258c28edf38ab5019 Mon Sep 17 00:00:00 2001 From: afonso Date: Tue, 16 Jul 2024 16:43:36 +0100 Subject: [PATCH 02/49] CELE-32 feat: Connect datasets sidebar --- .../ViewerContainer/CustomListItem.tsx | 85 ++++------ .../ViewerContainer/CustomSwitch.tsx | 96 ++++++----- .../components/ViewerContainer/DataSets.tsx | 153 +++++++++--------- 3 files changed, 156 insertions(+), 178 deletions(-) diff --git a/applications/visualizer/frontend/src/components/ViewerContainer/CustomListItem.tsx b/applications/visualizer/frontend/src/components/ViewerContainer/CustomListItem.tsx index 75cac039..d86c2c21 100644 --- a/applications/visualizer/frontend/src/components/ViewerContainer/CustomListItem.tsx +++ b/applications/visualizer/frontend/src/components/ViewerContainer/CustomListItem.tsx @@ -1,58 +1,56 @@ -import { useState } from "react"; -import Stack from '@mui/material/Stack'; -import Typography from "@mui/material/Typography"; -import Tooltip from '@mui/material/Tooltip'; -import FormControlLabel from '@mui/material/FormControlLabel'; -import HelpOutlineIcon from '@mui/icons-material/HelpOutline'; -import {Box, IconButton} from "@mui/material"; +import {useState} from "react"; +import { Box, Stack, Typography, IconButton, Tooltip, FormControlLabel } from "@mui/material"; +import { HelpOutline as HelpOutlineIcon, DeleteOutlined as DeleteOutlinedIcon, Add as AddIcon } from '@mui/icons-material'; +import CustomSwitch from "./CustomSwitch"; +import PickerWrapper from "./PickerWrapper"; import { vars } from "../../theme/variables.ts"; -import CustomSwitch from "./CustomSwitch.tsx"; -import PickerWrapper from "./PickerWrapper.tsx"; -import DeleteOutlinedIcon from '@mui/icons-material/DeleteOutlined'; -import AddIcon from '@mui/icons-material/Add'; - const { gray600, gray400B, gray500, gray50, error700 } = vars; -const CustomListItem = ({ data, showTooltip = true, listType, showExtraActions = false }) => { - const [checked, setChecked] = useState(false); +const CustomListItem = ({ data, showTooltip = true, listType, showExtraActions = false, onSwitchChange }) => { const [anchorEl, setAnchorEl] = useState(null); const [open, setOpen] = useState(false); const [selectedColor, setSelectedColor] = useState('#9FEE9A'); const [itemHovered, setItemHovered] = useState(false); - + const isNeurons = listType === 'neurons'; - const onSwitchChange = (e) => { - setChecked(e.target.checked); + + const handleSwitchChange = (event: React.ChangeEvent, checked: boolean) => { + if (onSwitchChange) { + onSwitchChange(data.id, checked); + } }; - + const handleClick = (event) => { event.stopPropagation(); event.preventDefault(); setAnchorEl(event.currentTarget); setOpen(true); }; - + const handleClose = () => { setAnchorEl(null); setOpen(false); }; + const handleColorChange = (color) => { setSelectedColor(color.hex); }; - - const handleOnMouseEnter = () => { + + const handleOnMouseEnter = () => { setItemHovered(true); }; - - const handleOnMouseLeave = () => { + + const handleOnMouseLeave = () => { setItemHovered(false); }; - + return ( <> } - onChange={onSwitchChange} + control={ + + + } onMouseEnter={handleOnMouseEnter} onMouseLeave={handleOnMouseLeave} sx={{ @@ -66,13 +64,11 @@ const CustomListItem = ({ data, showTooltip = true, listType, showExtraActions = "& .MuiFormControlLabel-label": { width: "100%", }, - "& .MuiIconButton-root": { padding: '.25rem', borderRadius: '.25rem', } }} - checked={checked} label={ - {showTooltip && ( - + )} - { - showExtraActions && itemHovered && + {showExtraActions && itemHovered && - + - - - } + + } {data.description && ( @@ -146,13 +129,7 @@ const CustomListItem = ({ data, showTooltip = true, listType, showExtraActions = } value={undefined} /> - + ); }; diff --git a/applications/visualizer/frontend/src/components/ViewerContainer/CustomSwitch.tsx b/applications/visualizer/frontend/src/components/ViewerContainer/CustomSwitch.tsx index 6bb385c8..e252685e 100644 --- a/applications/visualizer/frontend/src/components/ViewerContainer/CustomSwitch.tsx +++ b/applications/visualizer/frontend/src/components/ViewerContainer/CustomSwitch.tsx @@ -1,7 +1,7 @@ import Switch from '@mui/material/Switch'; -import { vars } from "../../theme/variables.ts"; import Tooltip from "@mui/material/Tooltip"; -import React,{ useState } from "react"; +import React from "react"; +import { vars } from "../../theme/variables.ts"; const { white, brand600, gray100 } = vars; @@ -21,57 +21,67 @@ const CustomSwitch: React.FC = ({ height, thumbDimension, checkedPosition, + checked, + onChange, + showTooltip, + disabled }) => { - const [checked, setChecked] = useState(false); - - const handleChange = (event) => { - setChecked(event.target.checked); + const handleChange = (event: React.ChangeEvent) => { + if (onChange) { + onChange(event, event.target.checked); + } }; - + return ( - ({ - marginRight: '.5rem', - width: width ?? 23, - height: height ?? 13, - padding: 0, - '& .MuiSwitch-switchBase': { + ({ + marginRight: '.5rem', + width: width ?? 23, + height: height ?? 13, padding: 0, - margin: '0.0938rem', - transitionDuration: '300ms', - '&.Mui-checked': { - transform: checkedPosition ?? 'translateX(0.5775rem)', - color: white, - '& + .MuiSwitch-track': { - backgroundColor: brand600, - opacity: 1, - border: 0, + '& .MuiSwitch-switchBase': { + padding: 0, + margin: '0.0938rem', + transitionDuration: '300ms', + '&.Mui-checked': { + transform: checkedPosition ?? 'translateX(0.5775rem)', + color: white, + '& + .MuiSwitch-track': { + backgroundColor: brand600, + opacity: 1, + border: 0, + }, + '&.Mui-disabled + .MuiSwitch-track': { + opacity: 0.5, + }, }, - '&.Mui-disabled + .MuiSwitch-track': { - opacity: 0.5, + '&.Mui-disabled .MuiSwitch-thumb': { + color: gray100, }, }, - '&.Mui-disabled .MuiSwitch-thumb': { - color: gray100, + '& .MuiSwitch-thumb': { + boxSizing: 'border-box', + width: thumbDimension ?? 10.24, + height: thumbDimension ?? 10.24, + boxShadow: 'none' + }, + '& .MuiSwitch-track': { + borderRadius: 26 / 2, + backgroundColor: gray100, + opacity: 1, + transition: theme.transitions.create(['background-color'], { + duration: 500, + }), }, - }, - '& .MuiSwitch-thumb': { - boxSizing: 'border-box', - width: thumbDimension ?? 10.24, - height: thumbDimension ?? 10.24, - boxShadow: 'none' - }, - '& .MuiSwitch-track': { - borderRadius: 26 / 2, - backgroundColor: gray100, - opacity: 1, - transition: theme.transitions.create(['background-color'], { - duration: 500, - }), - }, - })} /> + })} + /> ); }; diff --git a/applications/visualizer/frontend/src/components/ViewerContainer/DataSets.tsx b/applications/visualizer/frontend/src/components/ViewerContainer/DataSets.tsx index 67675a60..5bd10293 100644 --- a/applications/visualizer/frontend/src/components/ViewerContainer/DataSets.tsx +++ b/applications/visualizer/frontend/src/components/ViewerContainer/DataSets.tsx @@ -1,75 +1,79 @@ -import {Box, Stack, Typography, MenuItem, FormControl, IconButton} from "@mui/material"; +import { Box, Stack, Typography, MenuItem, FormControl, IconButton, Select } from "@mui/material"; import { vars } from "../../theme/variables.ts"; import CustomEntitiesDropdown from "./CustomEntitiesDropdown.tsx"; import CustomListItem from "./CustomListItem.tsx"; -import Select from '@mui/material/Select'; import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown'; import FilterListIcon from '@mui/icons-material/FilterList'; +import { useGlobalContext } from "../../contexts/GlobalContext.tsx"; +import { Dataset } from "../../rest"; + const { gray900, gray500, gray400 } = vars; -const data = [ - { - title: "Development stage 1", - dataSets: [ - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: '23 hours after birth', helpText: 'helpText' }, - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: '27 hours after birth', helpText: 'helpText' }, - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: 'L4 legacy dataset', helpText: 'helpText' }, - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: 'L4 legacy dataset', helpText: 'helpText' }, - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: 'L4 legacy dataset', helpText: 'helpText' }, - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: 'L4 legacy dataset', helpText: 'helpText' }, - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: 'L4 legacy dataset', helpText: 'helpText' }, - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: 'L4 legacy dataset', helpText: 'helpText' }, - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: 'L4 legacy dataset', helpText: 'helpText' }, - ] - }, - { - title: "Development stage 2", - dataSets: [ - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: 'L4 legacy dataset', helpText: 'helpText' }, - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: 'L4 legacy dataset', helpText: 'helpText' }, - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: 'L4 legacy dataset', helpText: 'helpText' }, - ] - }, - { - title: "Development stage 3", - dataSets: [ - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: 'L4 legacy dataset', helpText: 'helpText' }, - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: 'L4 legacy dataset', helpText: 'helpText' }, - ] - }, - { - title: "Adult", - dataSets: [ - { label: 'Witvliet et al., 2020, Dataset 1 (L1) 0, Dataset 1 (L1)', checked: true, description: '50 hours after birth', helpText: 'helpText' }, - ] - } -]; +// Categorize datasets based on their visualTime +const categorizeDatasets = (datasets: Dataset[]) => { + const categories = { + "Development stage 1": [], + "Development stage 2": [], + "Development stage 3": [], + "Adult": [] + }; + + datasets.forEach(dataset => { + if (dataset.visualTime >= 0 && dataset.visualTime < 10) { + categories["Development stage 1"].push(dataset); + } else if (dataset.visualTime >= 10 && dataset.visualTime < 20) { + categories["Development stage 2"].push(dataset); + } else if (dataset.visualTime >= 20 && dataset.visualTime < 30) { + categories["Development stage 3"].push(dataset); + } else if (dataset.visualTime >= 30) { + categories["Adult"].push(dataset); + } + }); + + return categories; +}; + +// Map Dataset to ListItem format +const mapDatasetToListItem = (dataset: Dataset, isActive: boolean) => ({ + id: dataset.id, // This is mandatory so that we can get the real Dataset back + label: dataset.name, + checked: isActive, + description: dataset.description, + helpText: dataset.collection, +}); const DataSets = () => { + const { datasets, workspaces, currentWorkspaceId } = useGlobalContext(); + const currentWorkspace = workspaces[currentWorkspaceId]; + const activeDatasets = currentWorkspace.activeDatasets; + + // Categorize the datasets + const categorizedDatasets = categorizeDatasets(datasets); + + // Handle activation and deactivation of datasets + const handleSwitchChange = async (datasetId: string, checked: boolean) => { + const dataset = datasets.find(ds => ds.id === datasetId); + if (!dataset) return; + + if (checked) { + await currentWorkspace.activateDataset(dataset); + } else { + await currentWorkspace.deactivateDataset(dataset.id); + } + }; + return ( - + Datasets - - Toggle on and off to view datasets on the workspace. This will affect - all viewers. + Toggle on and off to view datasets on the workspace. This will affect all viewers. - + - - + + { overflow: "auto", }} > - {data.map((section, index) => ( - - - {section.title} + {Object.entries(categorizedDatasets).map(([category, datasets], index) => ( + + + {category} - - {section.dataSets.map((item, i) => ( - + + {datasets.map((dataset) => ( + ))} From f26dd596f2fc6b00bcb789a0c59c51eb34a015a8 Mon Sep 17 00:00:00 2001 From: afonso Date: Tue, 16 Jul 2024 17:08:08 +0100 Subject: [PATCH 03/49] CELE-32 feat: Connect neurons sidebar --- .../components/ViewerContainer/Neurons.tsx | 69 ++++++++++++------- .../frontend/src/models/workspace.ts | 8 +-- 2 files changed, 48 insertions(+), 29 deletions(-) diff --git a/applications/visualizer/frontend/src/components/ViewerContainer/Neurons.tsx b/applications/visualizer/frontend/src/components/ViewerContainer/Neurons.tsx index 363b1b2f..3a61eaf9 100644 --- a/applications/visualizer/frontend/src/components/ViewerContainer/Neurons.tsx +++ b/applications/visualizer/frontend/src/components/ViewerContainer/Neurons.tsx @@ -1,24 +1,44 @@ -import {Box, IconButton, Stack, Typography} from "@mui/material"; +import { Box, IconButton, Stack, Typography, Tooltip } from "@mui/material"; import { vars } from "../../theme/variables.ts"; import CustomEntitiesDropdown from "./CustomEntitiesDropdown.tsx"; import CustomListItem from "./CustomListItem.tsx"; import AddIcon from '@mui/icons-material/Add'; -import Tooltip from "@mui/material/Tooltip"; +import { useGlobalContext } from "../../contexts/GlobalContext.tsx"; +import { Neuron } from "../../rest"; + const { gray900, gray500 } = vars; const Neurons = () => { - const activeNeurons = { - title: "Active neurons", - neurons: [ - { label: 'ADAR', checked: true, helpText: 'helpText' }, - { label: 'ADAL', checked: true, helpText: 'helpText' }, - { label: 'RIDD', checked: true, helpText: 'helpText' }, - ] - } + const { workspaces, currentWorkspaceId } = useGlobalContext(); + const currentWorkspace = workspaces[currentWorkspaceId]; + const activeNeurons = currentWorkspace.activeNeurons; + + // Transform active neurons data to the format expected by CustomListItem + const neuronList = Array.from(activeNeurons).map(neuronName => { + const neuron = currentWorkspace.availableNeurons[neuronName]; + return { + id: neuron.name, + label: neuron.name, + checked: true, + }; + }); + + // Handle activation and deactivation of neurons + const handleSwitchChange = (neuronName: string, checked: boolean) => { + const neuron = currentWorkspace.availableNeurons[neuronName]; + if (!neuron) return; + + if (checked) { + currentWorkspace.activateNeuron(neuron); + } else { + currentWorkspace.deactivateNeuron(neuronName); + } + }; + return ( - { > Neurons - - Search for the neurons and add it to your workspace. This will affect - all viewers. + Search for the neurons and add it to your workspace. This will affect all viewers. - + - {activeNeurons.title} + Active neurons - + - {activeNeurons.neurons.map((item, i) => ( - + {neuronList.map((item) => ( + ))} diff --git a/applications/visualizer/frontend/src/models/workspace.ts b/applications/visualizer/frontend/src/models/workspace.ts index 1dea15a1..c1bf39f4 100644 --- a/applications/visualizer/frontend/src/models/workspace.ts +++ b/applications/visualizer/frontend/src/models/workspace.ts @@ -4,7 +4,7 @@ import {NeuronGroup, ViewerSynchronizationPair, ViewerType} from "./models.ts"; import getLayoutManagerAndStore from "../layout-manager/layoutManagerFactory.ts"; import {Dataset, DatasetsService, Neuron} from "../rest"; import {fetchDatasets} from "../helpers/workspaceHelper.ts"; -import { LayoutManager } from '@metacell/geppetto-meta-client/common/layout/LayoutManager'; +import {LayoutManager} from '@metacell/geppetto-meta-client/common/layout/LayoutManager'; export class Workspace { [immerable] = true @@ -59,14 +59,14 @@ export class Workspace { activateNeuron(neuron: Neuron): void { const updated = produce(this, (draft: Workspace) => { - draft.activeNeurons[neuron.name] = neuron; + draft.activeNeurons.add(neuron.name); }); this.updateContext(updated); } deactivateNeuron(neuronId: string): void { const updated = produce(this, (draft: Workspace) => { - delete draft.activeNeurons[neuronId]; + draft.activeNeurons.delete(neuronId); }); this.updateContext(updated); } @@ -155,7 +155,7 @@ export class Workspace { neuronArrays.flat().forEach(neuron => { uniqueNeurons.add(neuron); // Add class neuron as well - const classNeuron = { ...neuron, name: neuron.nclass }; + const classNeuron = {...neuron, name: neuron.nclass}; uniqueNeurons.add(classNeuron); }); From 5ff0f01b26acbc204499c131ebf54e8eb24e9122 Mon Sep 17 00:00:00 2001 From: afonso Date: Tue, 16 Jul 2024 17:54:30 +0100 Subject: [PATCH 04/49] CELE-32 feat: Connect neurons search --- .../CustomEntitiesDropdown.tsx | 333 +++++++++--------- .../components/ViewerContainer/DataSets.tsx | 2 +- .../components/ViewerContainer/Neurons.tsx | 27 +- 3 files changed, 196 insertions(+), 166 deletions(-) diff --git a/applications/visualizer/frontend/src/components/ViewerContainer/CustomEntitiesDropdown.tsx b/applications/visualizer/frontend/src/components/ViewerContainer/CustomEntitiesDropdown.tsx index 506ba647..685e9242 100644 --- a/applications/visualizer/frontend/src/components/ViewerContainer/CustomEntitiesDropdown.tsx +++ b/applications/visualizer/frontend/src/components/ViewerContainer/CustomEntitiesDropdown.tsx @@ -1,10 +1,10 @@ -import React, { useState, useEffect } from 'react'; -import { Box, Typography, Popper, TextField, InputAdornment } from '@mui/material'; +import React, { useState } from 'react'; +import { Box, Typography, Popper, TextField, InputAdornment, ClickAwayListener } from '@mui/material'; import SearchIcon from '@mui/icons-material/Search'; import ArrowForwardIcon from '@mui/icons-material/ArrowForward'; -import {vars} from "../../theme/variables.ts"; +import { vars } from "../../theme/variables.ts"; -const {gray50, brand600} = vars +const { gray50, brand600 } = vars; type OptionDetail = { title: string; @@ -17,178 +17,189 @@ type Option = { content: OptionDetail[]; }; -const options = [ - { - id: '1', - label: 'Option', - content: [] - } -]; -export default function CustomEntitiesDropdown() { +interface CustomEntitiesDropdownProps { + options: Option[]; + onSelect: (option: Option) => void; +} + +const CustomEntitiesDropdown: React.FC = ({ options, onSelect }) => { const [anchorEl, setAnchorEl] = useState(null); const [hoveredOption, setHoveredOption] = useState - + {}}/>