diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index bf154a9f..238d911a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,6 +6,8 @@ For all schema changes: - [ ] Add release notes for the PR to `docs/format/source/format_release_notes.rst`. +- [ ] Have you included the relevant issue number using "Fix #XXX" notation where XXX is the issue number? By including "Fix #XXX" you allow GitHub to close issue #XXX when the PR is merged. +- [ ] Make sure that `hdmf-common-schema` points to the latest release and not the latest commit on the `main` branch. If this is the first schema change after a schema release (i.e., the version string in `core/nwb.namespace.yaml` does not end in "-alpha"), then: diff --git a/.github/workflows/check_sphinx_links.yml b/.github/workflows/check_sphinx_links.yml new file mode 100644 index 00000000..f2595eb9 --- /dev/null +++ b/.github/workflows/check_sphinx_links.yml @@ -0,0 +1,34 @@ +name: Check Sphinx external links and references +on: + pull_request: + schedule: + - cron: '0 5 * * *' # once per day at midnight ET + workflow_dispatch: + +jobs: + check-external-links: + runs-on: ubuntu-latest + steps: + - name: Cancel non-latest runs + uses: styfle/cancel-workflow-action@0.11.0 + with: + all_but_latest: true + access_token: ${{ github.token }} + + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + fetch-depth: 0 # tags are required for versioneer to determine the version + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install Sphinx dependencies and package + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements-doc.txt + + - name: Check Sphinx links and references + run: sphinx-build -n -b linkcheck ./docs/format/source ./test_build diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 7a1e8dc0..156afa03 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -14,6 +14,6 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Codespell - uses: codespell-project/actions-codespell@v1 + uses: codespell-project/actions-codespell@v2 diff --git a/.github/workflows/validate_schema.yml b/.github/workflows/validate_schema.yml index a0a31eb2..914586ca 100644 --- a/.github/workflows/validate_schema.yml +++ b/.github/workflows/validate_schema.yml @@ -8,11 +8,11 @@ jobs: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - uses: actions/checkout@v4 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.12" - name: Install dev branch of HDMF run: | pip install git+https://github.com/hdmf-dev/hdmf.git diff --git a/.gitignore b/.gitignore index bd328f1b..d9133939 100644 --- a/.gitignore +++ b/.gitignore @@ -5,9 +5,7 @@ __pycache__/ # Sphinx documentation docs/format/_build/ -docs/language/_build/ docs/storage/_build/ -docs/general/_build # Autogenerated Sphinx sources docs/format/source/_format_auto_docs/ @@ -17,3 +15,5 @@ docs/format/source/_format_auto_docs/ #PyCharm .idea/ + +.DS_Store diff --git a/Legal.txt b/Legal.txt index 1139ba74..7dda8dec 100644 --- a/Legal.txt +++ b/Legal.txt @@ -1,4 +1,4 @@ -“nwb-schema” Copyright (c) 2017-2023, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. +“nwb-schema” Copyright (c) 2017-2024, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. If you have questions about your rights to use or distribute this software, please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. diff --git a/README.rst b/README.rst index c7cc942e..0edc8dfc 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,9 @@ NWB Schema Format -======================== +================= A format specification schema for the Neurodata Without Borders (NWB) data format. -To get started using NWB, please go to the [NWB overview website](https://nwb-overview.readthedocs.io/en/latest/). +To get started using NWB, please go to the `NWB overview website `_. This repo contains: @@ -27,7 +27,7 @@ The NWB 1.0 format and API are archived in the [NeurodataWithoutBorders/api-pyth License ======================== -“nwb-schema” Copyright (c) 2017-2023, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. +“nwb-schema” Copyright (c) 2017-2024, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -45,7 +45,7 @@ You are under no obligation whatsoever to provide any bug fixes, patches, or upg Copyright ======================== -“nwb-schema” Copyright (c) 2017-2023, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. +“nwb-schema” Copyright (c) 2017-2024, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. If you have questions about your rights to use or distribute this software, please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. diff --git a/core/nwb.behavior.yaml b/core/nwb.behavior.yaml index 1a951b2f..4636e759 100644 --- a/core/nwb.behavior.yaml +++ b/core/nwb.behavior.yaml @@ -38,6 +38,24 @@ groups: is 'meters'. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'. required: false + - name: bounds + dtype: numeric + doc: The boundary range (min, max) for each dimension of `data`. The units are the same as the units for the data. + shape: + - - 1 + - 2 + - - 2 + - 2 + - - 3 + - 2 + dims: + - - x + - min,max + - - x,y + - min,max + - - x,y,z + - min,max + required: false - name: reference_frame dtype: text doc: Description defining what exactly 'straight-ahead' means. diff --git a/core/nwb.file.yaml b/core/nwb.file.yaml index 16ae7b3e..416c74aa 100644 --- a/core/nwb.file.yaml +++ b/core/nwb.file.yaml @@ -7,7 +7,7 @@ groups: attributes: - name: nwb_version dtype: text - value: "2.6.0" + value: "2.7.0" doc: File version string. Use semantic versioning, e.g. 1.2.1. This will be the name of the format with trailing major, minor and patch numbers. datasets: @@ -131,6 +131,16 @@ groups: - neurodata_type_inc: TimeSeries doc: TimeSeries objects containing data of presented stimuli. quantity: '*' + - neurodata_type_inc: NWBDataInterface + doc: 'Generic NWB data interfaces, usually from an extension, + containing data of presented stimuli.' + quantity: '*' + - neurodata_type_inc: DynamicTable + doc: DynamicTable objects containing data of presented stimuli. + quantity: '*' + # even though TimeSeries is a child type of NWBDataInterface, we do not remove TimeSeries + # in order to maintain backwards compatibility in the APIs that + # use the neurodata_type_inc from the schema to set the variable name - name: templates doc: 'Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are diff --git a/core/nwb.icephys.yaml b/core/nwb.icephys.yaml index 7f717315..62c1e46b 100644 --- a/core/nwb.icephys.yaml +++ b/core/nwb.icephys.yaml @@ -286,6 +286,10 @@ groups: - name: stimulus neurodata_type_inc: TimeSeriesReferenceVectorData doc: Column storing the reference to the recorded stimulus for the recording (rows). + - name: stimulus_template + neurodata_type_inc: TimeSeriesReferenceVectorData + doc: Column storing the reference to the stimulus template for the recording (rows). + quantity: '?' - neurodata_type_def: IntracellularResponsesTable neurodata_type_inc: DynamicTable diff --git a/core/nwb.namespace.yaml b/core/nwb.namespace.yaml index 5d146ed1..aabf4534 100644 --- a/core/nwb.namespace.yaml +++ b/core/nwb.namespace.yaml @@ -57,4 +57,4 @@ namespaces: - doc: This source module contains neurodata_type for retinotopy data. source: nwb.retinotopy.yaml title: Retinotopy - version: "2.6.0-alpha" + version: "2.8.0-alpha" diff --git a/core/nwb.ogen.yaml b/core/nwb.ogen.yaml index eb0bcdac..419d6190 100644 --- a/core/nwb.ogen.yaml +++ b/core/nwb.ogen.yaml @@ -6,10 +6,16 @@ groups: - name: data dtype: numeric dims: - - num_times + - - num_times + - - num_times + - num_rois shape: - - null - doc: Applied power for optogenetic stimulus, in watts. + - - null + - - null + - null + doc: Applied power for optogenetic stimulus, in watts. Shape can be 1D or 2D. + 2D data is meant to be used in an extension of OptogeneticSeries that + defines what the second dimension represents. attributes: - name: unit dtype: text diff --git a/core/nwb.ophys.yaml b/core/nwb.ophys.yaml index e5a21bd0..7fd4b217 100644 --- a/core/nwb.ophys.yaml +++ b/core/nwb.ophys.yaml @@ -340,12 +340,12 @@ groups: each point in time is assumed to be 2-D (has only x & y dimensions).' groups: - neurodata_type_inc: CorrectedImageStack - doc: Reuslts from motion correction of an image stack. + doc: Results from motion correction of an image stack. quantity: '+' - neurodata_type_def: CorrectedImageStack neurodata_type_inc: NWBDataInterface - doc: Reuslts from motion correction of an image stack. + doc: Results from motion correction of an image stack. groups: - name: corrected neurodata_type_inc: ImageSeries diff --git a/core/nwb.retinotopy.yaml b/core/nwb.retinotopy.yaml index e8972395..1cf98457 100644 --- a/core/nwb.retinotopy.yaml +++ b/core/nwb.retinotopy.yaml @@ -2,9 +2,9 @@ groups: - neurodata_type_def: ImagingRetinotopy neurodata_type_inc: NWBDataInterface default_name: ImagingRetinotopy - doc: 'Intrinsic signal optical imaging or widefield imaging for measuring retinotopy. - Stores orthogonal maps (e.g., altitude/azimuth; radius/theta) of responses to - specific stimuli and a combined polarity map from which to identify visual areas. + doc: 'DEPRECATED. Intrinsic signal optical imaging or widefield imaging for measuring + retinotopy. Stores orthogonal maps (e.g., altitude/azimuth; radius/theta) of responses + to specific stimuli and a combined polarity map from which to identify visual areas. This group does not store the raw responses imaged during retinotopic mapping or the stimuli presented, but rather the resulting phase and power maps after applying a Fourier transform on the averaged responses. diff --git a/docs/format/.readthedocs.yaml b/docs/format/.readthedocs.yaml new file mode 100644 index 00000000..5d6a2b62 --- /dev/null +++ b/docs/format/.readthedocs.yaml @@ -0,0 +1,27 @@ +# Read the Docs configuration file for https://nwb-schema.readthedocs.io/ +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.12" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/format/source/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +formats: all + +# Optionally set the version of Python and requirements required to build your docs +python: + install: + - requirements: requirements-doc.txt + +# Optionally include all submodules +submodules: + include: all + recursive: true diff --git a/docs/format/Makefile b/docs/format/Makefile index ab94d325..842f1083 100644 --- a/docs/format/Makefile +++ b/docs/format/Makefile @@ -159,7 +159,7 @@ changes: @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + $(SPHINXBUILD) -n -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." diff --git a/docs/format/make.bat b/docs/format/make.bat index d8642d31..72f39f9c 100644 --- a/docs/format/make.bat +++ b/docs/format/make.bat @@ -185,7 +185,7 @@ if "%1" == "changes" ( ) if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + %SPHINXBUILD% -n -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck if errorlevel 1 exit /b 1 echo. echo.Link check complete; look for any errors in the above output ^ diff --git a/docs/format/source/conf.py b/docs/format/source/conf.py index 349c8b59..8b0919b6 100644 --- a/docs/format/source/conf.py +++ b/docs/format/source/conf.py @@ -76,16 +76,16 @@ def setup(app): # General information about the project. project = u'NWB Format Specification' -copyright = u'2017-2023, Neurodata Without Borders' +copyright = u'2017-2024, Neurodata Without Borders' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '2.6.0' +version = '2.8.0-alpha' # The full version, including alpha/beta/rc tags. -release = '2.6.0' +release = '2.8.0-alpha' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/format/source/credits.rst b/docs/format/source/credits.rst index 3e209982..21ed29ca 100644 --- a/docs/format/source/credits.rst +++ b/docs/format/source/credits.rst @@ -18,11 +18,11 @@ NWB: Version 2.0.0 and later Documentation for Version 2 of the NWB format and later have been created by Oliver Ruebel and Andrew Tritt et al. in collaboration with the NWB community. -NWB:N: Version 1.0.x and earlier +NWB: Version 1.0.x and earlier -------------------------------- Version 1.0.5g (and earlier) of the NWB file format were created by Jeff Teeters et al. as -part of the first NWB pilot project. The documents for NWB:N 2 have been adopted from the +part of the first NWB pilot project. The documents for NWB 2 have been adopted from the final version of format docs released by the original NWB pilot project. diff --git a/docs/format/source/format_description.rst b/docs/format/source/format_description.rst index fcd15d97..a35b1be4 100644 --- a/docs/format/source/format_description.rst +++ b/docs/format/source/format_description.rst @@ -19,7 +19,7 @@ The NWB format uses the following main primitives to hierarchically organize neu The NWB format is formally described via formal specification documents using the `NWB specification language `_ . HDF5 currently serves as the main format for storing data in the NWB format (see http://nwb-storage.readthedocs.io/en/latest/ for details). -The `PyNWB `_ API is available to enable users to efficiently interact with NWB format files. +The `PyNWB `_ API is available to enable users to efficiently interact with NWB format files. The NWB format uses a modular design in which all main semantic components of the format have a unique *neurodata_type* (similar to a Class in object-oriented design)(:numref:`sec_neurodata_type_intro`). @@ -172,7 +172,7 @@ To ease development of extensions, the PyNWB_ (and `HDMF `_ used by PyNWB) API provides dedicated data structures that support programmatic creation and use of extensions. An example for extending NWB using PyNWB_ is available at -https://pynwb.readthedocs.io/en/stable/extensions.html and +https://pynwb.readthedocs.io/en/stable/tutorials/general/extensions.html and additional details are also available as part of the PyNWB_ tutorials https://pynwb.readthedocs.io/en/stable/tutorials/index.html . diff --git a/docs/format/source/format_release_notes.rst b/docs/format/source/format_release_notes.rst index 2b8a496b..2d9805ff 100644 --- a/docs/format/source/format_release_notes.rst +++ b/docs/format/source/format_release_notes.rst @@ -3,9 +3,25 @@ Release Notes ============= +2.8.0 (Upcoming) +------------------------ +Minor changes +^^^^^^^^^^^^^ +- Added support to set boundary metadata for ``SpatialSeries``. (#524) + +2.7.0 (February 7, 2024) +------------------------ + +Minor changes +^^^^^^^^^^^^^ +- Fixed typos in docstrings. (#560) +- Deprecated ``ImagingRetinotopy`` neurodata type. (#565) +- Modified ``OptogeneticSeries`` to allow 2D data, primarily in extensions of ``OptogeneticSeries``. (#564) +- Added optional ``stimulus_template`` column to ``IntracellularStimuliTable`` as part of the ``IntracellularRecordingsTable``. (#545) +- Added support for ``NWBDataInterface`` and ``DynamicTable`` in ``NWBFile.stimulus``. (#559) 2.6.0 (January 17, 2023) ------------------------ +------------------------ Minor changes ^^^^^^^^^^^^^ @@ -170,7 +186,7 @@ Bug Fixes - Use "text" data type for electrode table columns "location" and "group_name". Previously, only ASCII was allowed. -- Added to description to make electrode x,y,z consistent with CCF reference. http://help.brain-map.org/display/mousebrain/API#API-DownloadAtlas3-DReferenceModels +- Added to description to make electrode x,y,z consistent with CCF reference. https://allensdk.readthedocs.io/en/latest/reference_space.html - Added "position" dataset with compound data type x,y,z in ElectrodeGroup. @@ -281,7 +297,7 @@ Bug Fixes - NWB is cumbersome as a format for day-to-day work. There is a lot of overheard to save one-off analysis results to an NWB file. To save new datasets, a user has write an extension. This is a lot of work for a result that may just be tossed out. - "scratch" is now an optional top-level group under NWBFile that can hold NWBContainer groups and ScratchData datasets - The scratch space is explicitly for non-standardized data that is not intended for reuse - by others. Standard NWB:N types, and extensions if required, should always be used for any data that you + by others. Standard NWB types, and extensions if required, should always be used for any data that you intend to share. As such, published data should not include scratch data and a user should be able to ignore any data stored in scratch to use a file. - See https://github.com/NeurodataWithoutBorders/nwb-schema/issues/286 for details @@ -338,13 +354,13 @@ Added new base data types: ``NWBContainer``, ``NWBData``, ``NWBDataInterface`` **Specific Changes** * :ref:`NWBContainer ` defines a common base type for all Groups with a ``neurodata_type`` and - is now the base type of all main data group types in the NWB:N format, + is now the base type of all main data group types in the NWB format, including :ref:`TimeSeries `. This also means that all group types now inherit the required ``help`` and ``source`` attribute from ``NWBContainer``. A number of neurodata_types have been updated to add the missing ``help`` (see https://github.com/NeurodataWithoutBorders/nwb-schema/pull/37/files for details) * :ref:`NWBDataInterface ` extends :ref:`NWBContainer ` and replaces - ``Interface`` from NWB:N 1.x. It has been renamed to ease intuition. :ref:`NWBDataInterface ` + ``Interface`` from NWB 1.x. It has been renamed to ease intuition. :ref:`NWBDataInterface ` serves as base type for primary data (e.g., experimental or analysis data) and is used to distinguish in the schema between non-metadata data containers and metadata containers. (see https://github.com/NeurodataWithoutBorders/nwb-schema/pull/116/files for details) @@ -371,8 +387,8 @@ Support row-based and column-based tables rows via region-references to a single dataset (e.g., a set of electrodes), 2) make it easy to add rows by appending to a single dataset, 3) make it easy to read individual rows of a table (but require reading the full table to extract the data of a single column). - Row-based tables are used to simplify, e.g., the organization of electrode-metadata in NWB:N 2 (see above). - (See the `specification language release notes `_ + Row-based tables are used to simplify, e.g., the organization of electrode-metadata in NWB 2 (see above). + (See the `specification language release notes `_ for details about the addition of compound data types in the schema). * *Referencing rows in a row-based tables:* Subsets of rows can referenced directly via a region-reference to the @@ -390,7 +406,7 @@ Support row-based and column-based tables * *Referencing rows in column-based tables:* As :ref:`DynamicTable ` consist of multiple datasets (compared to row-based tables which consists of a single 1D dataset with a compound datatype) - is not possible to reference a set of rows with a single region reference. To address this issue, NWB:N defines + is not possible to reference a set of rows with a single region reference. To address this issue, NWB defines :ref:`DynamicTableRegion ` (added later in `PR634 (PyNWB) `_) dataset type, which stores a list of integer indices (row index) and also has an attribute ``table`` with the object reference to the corresponding :ref:`DynamicTable `. @@ -555,21 +571,21 @@ Improved storage of ROIs to produce the ROIs. * **Support 3D ROIs:** Allow users to add 3D ROIs collected from a multi-plane image. -**Changes:** The main types for storing ROIs in NWB:N 2 are :ref:`ImageSegmentation ` +**Changes:** The main types for storing ROIs in NWB 2 are :ref:`ImageSegmentation ` which stores 0 or more :ref:`PlaneSegmentation `. :ref:`PlaneSegmentation ` is a :ref:`DynamicTable ` for managing image segmentation results of a specific imaging plane. The ROIs are referenced by :ref:`RoiResponseSeries ` which stores the ROI responses over an -imaging plane. During the development of NWB:N 2 the management of ROIs has been improved several times. Here we +imaging plane. During the development of NWB 2 the management of ROIs has been improved several times. Here we outline the main changes (several of which were ultimately merged together in the :ref:`PlaneSegmentation ` type). -1. Added neurodata_type ``ImageMasks`` replacing ``ROI.img_mask`` (from NWB:N 1.x) with +1. Added neurodata_type ``ImageMasks`` replacing ``ROI.img_mask`` (from NWB 1.x) with **(a)** a 3D dataset with shape [num_rois, num_x_pixels, num_y_pixels] (i.e. an array of planar image masks) or **(b)** a 4D dataset with shape [num_rois, num_x_pixels, num_y_pixels, num_z_pixels] (i.e. an array of volumetric image masks) ``ImageMasks`` was subsequently merged with :ref:`PlaneSegmentation ` and is represented by the :ref:`VectorData ` table column ``image_mask`` in the table. -2. Added neurodata_type ``PixelMasks`` which replaces ROI.pix_mask/ROI.pix_mask_weight (from NWB:N 1.x) +2. Added neurodata_type ``PixelMasks`` which replaces ROI.pix_mask/ROI.pix_mask_weight (from NWB 1.x) with a table that has columns “x”, “y”, and “weight” (i.e. combining ROI.pix_mask and ROI.pix_mask_weight into a single table). ``PixelMasks`` was subsequently merged with :ref:`PlaneSegmentation ` and is represented by the :ref:`VectorData ` dataset ``pixel__mask`` that is referenced from the table @@ -581,7 +597,7 @@ outline the main changes (several of which were ultimately merged together in th 4. Added neurodata_type ``ROITable`` which defines a table for storing references to the image mask and pixel mask for each ROI (see item 1,2). The ``ROITable`` type was subsequently merged with the :ref:`PlaneSegmentation ` type and as such does no longer appear as a separate type in the - NWB:N 2 schema but :ref:`PlaneSegmentation ` takes the function of ``ROITable``. + NWB 2 schema but :ref:`PlaneSegmentation ` takes the function of ``ROITable``. 5. Added neurodata_type ``ROITableRegion`` for referencing a subset of elements in an ROITable. Subsequently ``ROITableRegion`` has been replaced by :ref:`DynamicTableRegion ` as the ``ROITable`` changed to a :ref:`DynamicTable ` and was merged with @@ -620,17 +636,17 @@ For additional details see also: Improved storage of unit-based data """"""""""""""""""""""""""""""""""" -In NWB:N 1.0.x data about spike units was stored across a number of different neurodata_types, specifically +In NWB 1.0.x data about spike units was stored across a number of different neurodata_types, specifically ``UnitTimes``, ``ClusterWaveforms``, and ``Clustering``. This structure had several critical shortcomings, -which were addressed in three main phases during the development of NWB:N 2. +which were addressed in three main phases during the development of NWB 2. -**Problem 1: Efficiency:** In NWB:N 1.x each unit was stored as a separate group ``unit_n`` containing the ``times`` +**Problem 1: Efficiency:** In NWB 1.x each unit was stored as a separate group ``unit_n`` containing the ``times`` and ``unit_description`` for unit with index ``n``. In cases where users have a very large number of units, this was problematic with regard to performance. To address this challenge ``UnitTimes`` has been -restructured in NWB:N 2 to use the new :ref:`VectorData ` , +restructured in NWB 2 to use the new :ref:`VectorData ` , :ref:`VectorIndex `, :ref:`ElementIdentifiers ` data structures -(see :ref:`sec-rn-vectordata-nwb2`).Specifically, NWB:N 2 replaced ``unit_n`` (from NWB:N 1.x, also referred to -by neurodata_type ``SpikeUnit`` in NWB:N 2beta) groups in ``UnitTimes`` with the following data: +(see :ref:`sec-rn-vectordata-nwb2`).Specifically, NWB 2 replaced ``unit_n`` (from NWB 1.x, also referred to +by neurodata_type ``SpikeUnit`` in NWB 2beta) groups in ``UnitTimes`` with the following data: * ``unit_ids`` : :ref:`ElementIdentifiers ` dataset for storing unique ids for each element * ``spike_times_index``: :ref:`VectorIndex ` dataset with region references into the spike times dataset @@ -651,17 +667,17 @@ See also `I116 (nwb-schema) ` (``unit_ids``) data structures. **Problem 2: Dynamic Metadata:** Users indicated that it was not easy to store user-defined metadata about units. -To address this challenge, NWB:N 2 added an optional top-level group ``units/`` (which was subsequently moved to +To address this challenge, NWB 2 added an optional top-level group ``units/`` (which was subsequently moved to ``/intervals/units``) which is a :ref:`DynamicTable ` with ``id`` and ``description`` columns and optional additional user-defined table columns. See `PR597 on PyNWB `_ for detailed code changes. See -the `PyNWB docs `__ for a +the `PyNWB docs `__ for a short tutorial on how to use unit metadata. See :ref:`NWBFile ` *Groups: /units* for an overview of the unit schema. **Problem 3: Usability:** Finally, users found that storing unit data was challenging due to the fact that the information was distributed across a number of different -types. To address this challenge, NWB:N 2.0 integrates ``UnitTimes``, ``ClusterWaveforms``, and ``Clustering`` (deprecated) +types. To address this challenge, NWB 2.0 integrates ``UnitTimes``, ``ClusterWaveforms``, and ``Clustering`` (deprecated) into the new column-based table ``units/`` (i.e., ``intervals/units``) (which still uses the optimized vector data storage to efficiently store spike times). See for discussions and `I674 on PyNWB `_ @@ -670,13 +686,13 @@ request `PR684 on PyNWB ` Improved specification of reference time stamp(s) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To improve the specification of reference time, NWB:N adopts ISO8061 for storing datetimes and adds +To improve the specification of reference time, NWB adopts ISO8061 for storing datetimes and adds ``timestamps_reference_time`` as explicit zero for all timestamps in addition to the ``session_start_time``. Improve standardization of reference time specification using ISO8061 @@ -753,9 +769,9 @@ Improved storage of epoch data TimeSeries that the Epoch applies to was stored as a single group. This structure is inefficient for storing large numbers of Epochs. -**Format Changes:** In NWB:N 2 epochs are stored via a :ref:`TimeIntervals ` table (i.e., a +**Format Changes:** In NWB 2 epochs are stored via a :ref:`TimeIntervals ` table (i.e., a :ref:`DynamicTable ` for storing time intervals) that is stored in the group ``/intervals/epochs``. -Over the course of the development of NWB:N 2 the epoch storage has been refined in several phases: +Over the course of the development of NWB 2 the epoch storage has been refined in several phases: - First, we create a new neurodata_type ``Epochs`` which was included in :ref:`NWBFile ` as the group ``epochs``. This simplified the extension of the epochs structure. ``/epochs`` at that point contained a @@ -781,12 +797,12 @@ Improved support for trial-based data **Change:** Add dedicated concept for storing trial data. -**Reason:** Users indicated that it was not easy to store trial data in NWB:N 1.x. +**Reason:** Users indicated that it was not easy to store trial data in NWB 1.x. **Format Changes:** Added optional group ``/intervals/trials/`` which is a :ref:`DynamicTable ` with ``id``, ``start_time``, and ``stop_time`` columns and optional additional user-defined table columns. See `PR536 on PyNWB `_ for detailed code changes. See -the `PyNWB docs `__ for a +the `PyNWB docs `__ for a short tutorial on how to use trials. See :ref:`NWBFile ` *Groups: /trials* for an overview of the trial schema. **Note:** Originally trials was added a top-level group trials which was then later moved to ``/intervals/trials`` as part of the generalization of time interval storage as part of @@ -821,7 +837,7 @@ Replaced Implicit Links/Data-Structures with Explicit Links **Change** Replace implicit links with explicit soft-links to the corresponding HDF5 objects where possible, i.e., use explicit HDF5 mechanisms for expressing basic links between data rather than implicit ones that require -users/developers to know how to use the specific data. In addition to links, NWB:N 2 adds support for object- +users/developers to know how to use the specific data. In addition to links, NWB 2 adds support for object- and region references, enabling the creation of datasets (i.e., arrays) that store links to other data objects (groups or datasets) or regions (i.e., subsets) of datasets. @@ -846,7 +862,7 @@ kind of links. - Integer array dataset ``electrode_idx`` of ```` is now a dataset ``electrodes`` of type :ref:`DynamicTableRegion ` pointing to a region of the ``ElectrodeTable`` stored in ``/general/extracellular_ephys/electrodes``. - Text dataset ``/general/extracellular_ephys//device`` is now a link ``/device`` - - The Epochs , Unit, Trial and other dynamic tables in NWB:N 2 also support (and use) region and object references + - The Epochs , Unit, Trial and other dynamic tables in NWB 2 also support (and use) region and object references to explicitly reference other data (e.g., vector data as part of the unit tables). @@ -950,13 +966,13 @@ Removed ``source`` field **Change:** Remove required attribute ``source`` from all neurodata_types -**Reason:** In NWB:N 1.0.x the attribute ``source`` was defined as a free text entry +**Reason:** In NWB 1.0.x the attribute ``source`` was defined as a free text entry intended for storage of provenance information. In practice, however, this attribute was often either ignored, contained no useful information, and/or was misused to encode custom metadata (that should have been defined via extensions). **Specific Change:** Removed attribute ``source`` from the core base neurodata_types -which effects a large number of the types throughout the NWB:N schema. For further +which effects a large number of the types throughout the NWB schema. For further details see `PR695 (PyNWB) `_) @@ -993,7 +1009,7 @@ Improved organization of processed data * With :ref:`NWBDataInterface ` now being a general base class of :ref:`TimeSeries `, this means that it is now possible to define data processing types that directly inherit from :ref:`TimeSeries `, - which was not possible in NWB:N 1.x. + which was not possible in NWB 1.x. * *Interface* has been renamed to *NWBDataInterface* to avoid confusion and ease intuition (see above) * All *Interface* types in the original format had fixed names. The fixed names have been replaced by specification of corresponding default names. This change enables storage of @@ -1010,7 +1026,7 @@ Simplified organization of acquisition data (not just TimeSeries). * ``/acquisition/timeseries`` and ``/acquisition/images`` have been removed * Created a new neurodata_type :ref:`Images ` for storing a collection of images to replace - ``acquisition/images`` and provide a more general container for use elsewhere in NWB:N (i.e., this is not + ``acquisition/images`` and provide a more general container for use elsewhere in NWB (i.e., this is not meant to replace :ref:`ImageSeries `) Other changes: @@ -1022,20 +1038,20 @@ Other changes: Improved governance and accessibility ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -**Change:** Updated release and documentation mechanisms for the NWB:N format specification +**Change:** Updated release and documentation mechanisms for the NWB format specification -**Reason:** Improve governance, ease-of-use, extensibility, and accessibility of the NWB:N format specification +**Reason:** Improve governance, ease-of-use, extensibility, and accessibility of the NWB format specification **Specific Changes** - - The NWB:N format specification is now released in separate Git repository + - The NWB format specification is now released in separate Git repository - Format specifications are released as YAML files (rather than via Python .py file included in the API) - Organized core types into a set of smaller YAML files to ease overview and maintenance - Converted all documentation documents to Sphinx reStructuredText to improve portability, maintainability, deployment, and public access - Sphinx documentation for the format are auto-generated from the YAML sources to ensure consistency between the specification and documentation - - The PyNWB API now provides dedicated data structured to interact with NWB:N specifications, enabling users to + - The PyNWB API now provides dedicated data structured to interact with NWB specifications, enabling users to programmatically access and generate format specifications @@ -1043,10 +1059,10 @@ Improved governance and accessibility Specification language changes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -**Change:** Numerous changes have been made to the specification language itself in NWB:N 2.0. Most changes to +**Change:** Numerous changes have been made to the specification language itself in NWB 2.0. Most changes to the specification language effect mainly how the format is specified, rather than the actual structure of the format. The changes that have implications on the format itself are described next. For an overview and discussion of the -changes to the specification language see `specification language release notes `_. +changes to the specification language see `specification language release notes `_. Specification of dataset dimensions """"""""""""""""""""""""""""""""""" @@ -1085,7 +1101,7 @@ Removed datasets defined via autogen of all datasets that were produced via autogen it was decided that all autogen datasets should be removed from the format. -**Reason** The main reasons for removal of autogen dataset is to ease use and maintenance of NWB:N files by +**Reason** The main reasons for removal of autogen dataset is to ease use and maintenance of NWB files by 1) avoiding redundant storage of information (i.e., improve normalization of data) and 2) avoiding dependencies between data (i.e., datasets having to be updated due to changes in other locations in a file). @@ -1134,7 +1150,7 @@ the specification, i.e., any objects that are not part of the specification are 1.0.x (09/2015 - 04/2017) ------------------------- -NWB:N 1.0.x has been deprecated. For documents relating to the 1.0.x schema please see +NWB 1.0.x has been deprecated. For documents relating to the 1.0.x schema please see `https://github.com/NeurodataWithoutBorders/specification_nwbn_1_0_x `_. diff --git a/docs/format/source/software_process.rst b/docs/format/source/software_process.rst index ce132e79..49996cd5 100644 --- a/docs/format/source/software_process.rst +++ b/docs/format/source/software_process.rst @@ -128,7 +128,7 @@ could point to two different versions of the schema. For the same reason, it is important that all public releases of the APIs, PyNWB and MatNWB, contain a publicly released version of nwb-schema. Starting with nwb-schema version 2.2.0, the dev branch and all public releases of PyNWB and MatNWB include only publicly released versions of nwb-schema. For more details, see the -[PyNWB software process documentation](https://pynwb.readthedocs.io/en/latest/software_process.html). +[PyNWB software process documentation](https://pynwb.readthedocs.io/en/stable/software_process.html). The [NWB Extensions Versioning Guidelines](https://nwb-extensions.github.io/versioning_guidelines) are used to guide versioning of the NWB core schema, as well as extensions to NWB. diff --git a/docs/storage/source/storage_release_notes.rst b/docs/storage/source/storage_release_notes.rst index 4666d142..c0a6f859 100644 --- a/docs/storage/source/storage_release_notes.rst +++ b/docs/storage/source/storage_release_notes.rst @@ -6,24 +6,24 @@ NWB Storage v2.1.1 ------------------ Fix wording in documentation of key mapping tables for group and dataset. -NWB:N - v2.1.0 +NWB - v2.1.0 -------------- Added documentation for new NWB key 'object_id' (see also format release notes for NWB 2.1.0: https://nwb-schema.readthedocs.io/en/latest/format_release_notes.html#september-2019). -NWB:N - v2.0.1 +NWB - v2.0.1 -------------- Added missing documentation on how format specification are cached in HDF5. -NWB:N - v2.0.0 +NWB - v2.0.0 --------------- Created separate reStructuredText documentation (i.e., this document) discuss and govern storage-related concerns. In particular this documents describes how primitives and keys described via the specification language are mapped to storage, in particular HDF5. -NWB:N - v1.0.x and earlier +NWB - v1.0.x and earlier -------------------------- -For version 1.0.x and earlier, there was no official separate document governing NWB:N storage concerns as -HDF5 was the only supported storage backend with implicit mapping between HDF5 types and NWB:N +For version 1.0.x and earlier, there was no official separate document governing NWB storage concerns as +HDF5 was the only supported storage backend with implicit mapping between HDF5 types and NWB language primitives. diff --git a/hdmf-common-schema b/hdmf-common-schema index 9b2580a2..5b4cbb31 160000 --- a/hdmf-common-schema +++ b/hdmf-common-schema @@ -1 +1 @@ -Subproject commit 9b2580a21647e06be54708fabf2d44cef73d32cb +Subproject commit 5b4cbb31dbafcff51ca70bf218f464b186568151 diff --git a/license.txt b/license.txt index ba6ea4d5..155961bd 100644 --- a/license.txt +++ b/license.txt @@ -1,4 +1,4 @@ -“nwb-schema” Copyright (c) 2017-2023, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. +“nwb-schema” Copyright (c) 2017-2024, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/requirements-doc.txt b/requirements-doc.txt index c2455afd..ff5e5934 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,2 +1,5 @@ hdmf-docutils pynwb +sphinx +sphinx_rtd_theme +matplotlib