Add data read

.github/workflows/codespell.yml

@@ -6,8 +6,8 @@ on:
     - main
 
   pull_request:
-    branches:
-    - main
+    #branches:
+    #- main
   workflow_dispatch:
 
 jobs:

.github/workflows/doxygen-build.yml

@@ -2,8 +2,8 @@ name: Doxygen build test
 
 on:
   pull_request:
-    branches:
-    - main
+    #branches:
+    #- main
 
 jobs:
   test:
@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           submodules: "true"
-      
+
       - name: Install dependencies - macos
         run: brew install hdf5 boost catch2 doxygen graphviz
 

.github/workflows/lint.yml

@@ -6,8 +6,8 @@ on:
     - main
 
   pull_request:
-    branches:
-    - main
+    #branches:
+    #- main
   workflow_dispatch:
 
 jobs:
@@ -18,4 +18,4 @@ jobs:
         - name: Checkout repository
           uses: actions/checkout@v4
         - name: Lint
-          run: cmake -D FORMAT_COMMAND=clang-format-14 -P cmake/lint.cmake
+          run: cmake -D FORMAT_COMMAND=clang-format -P cmake/lint.cmake

.github/workflows/tests.yml

@@ -6,8 +6,8 @@ on:
     - main
 
   pull_request:
-    branches:
-    - main
+    #branches:
+    #- main
   workflow_dispatch:
 
 jobs:
@@ -22,7 +22,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [macos-latest, ubuntu-latest]
-    
+
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -39,7 +39,7 @@ jobs:
         git checkout "v3.5.3"
         cmake -Bbuild -H. -DBUILD_TESTING=OFF
         sudo cmake --build build/ --target install
-    
+
     - name: Install dependencies - macos
       if: matrix.os == 'macos-latest'
       run: brew install hdf5 boost catch2
@@ -67,7 +67,7 @@ jobs:
 
   sanitize:
     needs: test
-    
+
     runs-on: ubuntu-latest
 
     steps:

CMakeLists.txt

@@ -19,19 +19,24 @@ include(cmake/variables.cmake)
 
 add_library(
     aqnwb_aqnwb
-    src/BaseIO.cpp
+    src/io/BaseIO.cpp
     src/Channel.cpp
-    src/hdf5/HDF5IO.cpp
+    src/io/hdf5/HDF5IO.cpp
+    src/io/hdf5/HDF5RecordingData.cpp
     src/nwb/NWBFile.cpp
     src/nwb/RecordingContainers.cpp
+    src/nwb/RegisteredType.cpp
     src/nwb/base/TimeSeries.cpp
     src/nwb/device/Device.cpp
     src/nwb/ecephys/ElectricalSeries.cpp
     src/nwb/ecephys/SpikeEventSeries.cpp
     src/nwb/file/ElectrodeGroup.cpp
     src/nwb/file/ElectrodeTable.cpp
     src/nwb/hdmf/base/Container.cpp
+    src/nwb/hdmf/base/Data.cpp
     src/nwb/hdmf/table/DynamicTable.cpp
+    src/nwb/hdmf/table/VectorData.cpp
+    src/nwb/hdmf/table/ElementIdentifiers.cpp
 )
 
 add_library(aqnwb::aqnwb ALIAS aqnwb_aqnwb)

docs/Doxyfile.in

@@ -6,14 +6,34 @@
 PROJECT_NAME = "@PROJECT_NAME@"
 PROJECT_NUMBER = "@PROJECT_VERSION@"
 
+
+# Expand the DEFINE_FIELD macro to create documentation also for functions created by
+# this macro as part of the RegisterType.hpp logic, which simplifies the creation
+# of functions for lazy read access. Unfortunately, expanding the macro directly
+# from the source code directly for some reason did not work with Doxygen and
+# using "EXPAND_AS_DEFINED = DEFINE_FIELD" also failed. As a workaround we here define
+# a simplified version of the macro as part of the PREDEFINED key to create a
+# simplified expansion of the macro for documentation purposes
+MACRO_EXPANSION = YES
+PREDEFINED           += "DEFINE_FIELD(name, storageObjectType, default_type, fieldPath, description)=/** description */ template<typename VTYPE = default_type> inline std::unique_ptr<IO::ReadDataWrapper<storageObjectType, VTYPE>> name() const;"
+EXPAND_ONLY_PREDEF   = YES
+
+
 # Add sources
 INPUT = "@PROJECT_SOURCE_DIR@/src" "@PROJECT_SOURCE_DIR@/docs/pages"
+RECURSIVE = YES
 EXAMPLE_PATH = "@PROJECT_SOURCE_DIR@/tests"  "@PROJECT_SOURCE_DIR@/.github/CODE_OF_CONDUCT.md"  "@PROJECT_SOURCE_DIR@/Legal.txt" "@PROJECT_SOURCE_DIR@/LICENSE"
 IMAGE_PATH = "@PROJECT_SOURCE_DIR@/resources/images"
 EXTRACT_ALL = YES
 RECURSIVE = YES
 OUTPUT_DIRECTORY = "@DOXYGEN_OUTPUT_DIRECTORY@"
 
+# Also show private members in the docs,
+EXTRACT_PRIVATE = YES
+# Also show static members in the docs
+EXTRACT_STATIC      = YES
+# HIDE_UNDOC_MEMBERS = YES
+
 # Enable Markdown support
 MARKDOWN_SUPPORT = YES
 

docs/pages/1_userdocs.dox

@@ -6,5 +6,6 @@
  *
  * - \subpage user_install_page
  * - \subpage workflow
+ * - \subpage read_page
  * - \subpage hdf5io
  */

docs/pages/2_devdocs.dox

@@ -7,6 +7,7 @@
  * - \subpage testing
  * - \subpage dev_docs_page
  * - \subpage nwb_schema_page
+ * - \subpage registered_type_page
  * - \subpage code_of_conduct_page
  * - \subpage license_page
  * - \subpage copyright_page

docs/pages/devdocs/documentation.dox

@@ -69,7 +69,7 @@
  *
  *  To display a code snippet from our example in the documentation we can use the
  *  ``\snippet <file> <label>`` Doxygen command in our ``*.dox`` documentation file.
- *  For example to just show the line where we create the \ref AQNWB::HDF5::HDF5IO "HDF5IO
+ *  For example to just show the line where we create the \ref AQNWB::IO::HDF5::HDF5IO "HDF5IO"
  *  object we use:
  *
  * \code{.sh}

docs/pages/devdocs/install.dox

@@ -102,7 +102,7 @@
  * cmake --build --preset=dev --target=<name of the target>
  * \endcode
  *
- * \subsubsection  devbuild_target_options_subsubsec Target options
+ * \subsection  devbuild_target_options_subsubsec Target options
  * - `format-check`: run the `clang-format` tool on the codebase to check for formatting errors
  * - `format-fix` :  run the `clang-format` tool on the codebase with `FIX=YES` to both check and automatically fix for formatting errors
  * - `spell-check`: run the `codespell` tool on the codebase to check for common spelling errors

docs/pages/devdocs/registered_types.dox

@@ -0,0 +1,208 @@
+/**
+ *
+ * \page registered_type_page Implementing a new Neurodata Type
+ *
+ * \tableofcontents
+ *
+ *
+ *
+ * New neurodata_types typically inherit from at least either \ref AQNWB::NWB::Container "Container"
+ * or \ref AQNWB::NWB::Data "Data", or a more specialized type of the two. In any case,
+ * all classes that represent a ``neurodata_type`` defined in the schema should be implemented
+ * as a subtype of \ref AQNWB::NWB::RegisteredType "RegisteredType".
+ *
+ * @section implement_registered_type How to Implement a RegisteredType
+ *
+ * To implement a subclass of \ref AQNWB::NWB::RegisteredType "RegisteredType", follow these steps:
+ *
+ * 1. Include the `RegisteredType.hpp` header file in your subclass header file.
+ *    @code
+ *    #include "nwb/RegisteredType.hpp"
+ *    @endcode
+ *
+ * 2. Define your subclass by inheriting from \ref AQNWB::NWB::RegisteredType "RegisteredType".
+ *    Ensure that your subclass implements a constructor with the arguments
+ *    `(const std::string& path, std::shared_ptr<IO::BaseIO> io)`,
+ *    as the "create" method expects this constructor signature.
+ *    @code
+ *    class MySubClass : public AQNWB::NWB::RegisteredType {
+ *    public:
+ *        MySubClass(const std::string& path, std::shared_ptr<IO::BaseIO> io)
+ *            : RegisteredType(path, io) {}
+ *
+ *        // Implement any additional methods or overrides here
+ *    };
+ *    @endcode
+ *
+ * 3. Use the \ref REGISTER_SUBCLASS macro to register your subclass. This should usually appear in
+ *    the header (`hpp`) file as part of the class definition:
+ *    @code
+ *    REGISTER_SUBCLASS(MySubClass, "my-namespace")
+ *    @endcode
+ *
+ * 4. In the corresponding source (`cpp`) file, initialize the static member to trigger the registration
+ *    using the \ref REGISTER_SUBCLASS_IMPL macro:
+ *    @code
+ *    #include "MySubClass.h"
+ *
+ *    // Initialize the static member to trigger registration
+ *    REGISTER_SUBCLASS_IMPL(MySubClass)
+ *    @endcode
+ *
+ * 5. To define getter methods for lazy read access to datasets and attributes that belong to our type,
+ *    we can use the \ref DEFINE_FIELD  macro. This macro creates a standard method for retrieving a
+ *    \ref AQNWB::IO::ReadDataWrapper "ReadDataWrapper" for lazy reading for the field:
+ *    @code
+ *    DEFINE_FIELD(getData, DatasetField, float, "data", The main data)
+ *    @endcode
+ *
+ * \warning
+ * To ensure proper function on read, the name of the class should match the name of the
+ * ``neurodata_type`` as defined in the schema. Similarly, "my-namespace" should match
+ * the name of the namespace in the schema (e.g., "core", "hdmf-common"). In this way
+ * we can look up the corresponding class for an object in a file based on the
+ * ``neurodata_type`` and ``namespace`` attributes stored in the file.
+ *
+ * \note
+ * A special version of the ``REGISTER_SUBCLASS`` macro, called ``REGISTER_SUBCLASS_WITH_TYPENAME``,
+ * allows setting the typename explicitly as a third argument. This is for the **special case**
+ * where we want to implement a class for a modified type that does not have its
+ * own `neurodata_type` in the NWB schema. An example is `ElectrodesTable` in NWB <v2.7, which
+ * did not have an assigned `neurodata_type`, but was implemented as a regular
+ * `DynamicTable`. To allow us to define a class `ElectrodeTable` to help with writing the table
+ * we can then use ``REGISTER_SUBCLASS_WITH_TYPENAME(ElectrodeTable, "core", "DynamicTable")``
+ * in the `ElectrodesTable` class. This ensures that the `neurodata_type` attribute is set
+ * correctly to `DynamicTable` on write instead of `ElectrodesTable`. However, on read this
+ * will by default not use the `ElectrodesTable` class but the regular `DynamicTable` class
+ * since that is what the schema is indicating to use. In the registry, the class will still
+ * be registered under the ``core::ElectrodesTable`` key, but with "DynamicTable" as the
+ * typename value and the `ElectrodesTable.getTypeName` automatic override returning
+ * the indicated typename instead of the classname.
+ *
+ *
+ * @subsection implement_registered_type_example Example: Implementing a new type
+ *
+ * *MySubClass.hpp*
+ * @code
+ * #pragma once
+ * #include "RegisteredType.hpp"
+ *
+ * class MySubClass : public AQNWB::NWB::RegisteredType
+ * {
+ * public:
+ *     MySubClass(const std::string& path, std::shared_ptr<IO::BaseIO> io)
+ *         : RegisteredType(path, io) {}
+ *
+ *     DEFINE_FIELD(getData, DatasetField, float, "data", The main data)
+ *
+ *     REGISTER_SUBCLASS(MySubClass, "my-namespace")
+ * };
+ * @endcode
+ *
+ * *MySubClass.cpp*
+ * @code
+ * #include "MySubClass.h"
+ *
+ * // Initialize the static member to trigger registration
+ * REGISTER_SUBCLASS_IMPL(MySubClass)
+ * @endcode
+ *
+ * @section type_registry How the Type Registry in RegisteredType Works
+ *
+ * The type registry in \ref AQNWB::NWB::RegisteredType "RegisteredType" allows for dynamic creation of registered subclasses by name. Here is how it works:
+ *
+ * 1. **Registry Storage**:
+ *    - The registry is implemented using 1) an `std::unordered_set` to store subclass names (which can be
+ *      accessed via \ref AQNWB::NWB::RegisteredType::getRegistry "getRegistry()") and
+ *      2) an `std::unordered_map` to store factory functions for creating instances of the subclasses
+ *      (which can be accessed via \ref AQNWB::NWB::RegisteredType::getFactoryMap() "getFactoryMap()").
+ *      The factory methods are the required constructor that uses the io and path as input.
+ *    - These are defined as static members within the \ref AQNWB::NWB::RegisteredType "RegisteredType" class.
+ *
+ * 2. **Registration**:
+ *    - The \ref AQNWB::NWB::RegisteredType::registerSubclass "registerSubclass" method is used to add a
+ *      subclass name and its corresponding factory function to the registry.
+ *    - This method is called via the `REGISTER_SUBCLASS` macro, which defines a static method (`registerSubclass()`)
+ *      and static member (`registered_`) to trigger the registration when the subclass is loaded.
+ *
+ * 3. **Dynamic Creation**:
+ *    - The \ref AQNWB::NWB::RegisteredType::create "create" method is used to create an instance of a registered subclass by name.
+ *    - This method looks up the subclass name in the registry and calls the corresponding factory function to create an instance.
+ *
+ * 4. **Automatic Registration**:
+ *    - The `REGISTER_SUBCLASS_IMPL` macro initializes the static member (`registered_`), which triggers the
+ *      \ref AQNWB::NWB::RegisteredType::registerSubclass "registerSubclass" method
+ *      and ensures that the subclass is registered when the program starts.
+ *
+ * 5. **Class Name and Namespace Retrieval**:
+ *    - The \ref AQNWB::NWB::RegisteredType::getTypeName "getTypeName" and
+ *      \ref AQNWB::NWB::RegisteredType::getNamespace "getNamespace" return the string name of the
+ *      class and namespace, respectively. The `REGISTER_SUBCLASS` macro implements an automatic
+ *      override of the methods to ensure the appropriate type and namespace string are returned.
+ *      These methods should, hence, not be manually overridden by subclasses, to ensure consistency
+ *      in type identification.
+ *
+ *
+ * @section use_registered_type_registry How to Use the RegisteredType Registry
+ *
+ * The \ref AQNWB::NWB::RegisteredType "RegisteredType" registry allows for dynamic creation and management of registered subclasses. Here is how you can use it:
+ *
+ * 1. **Creating Instances Dynamically**:
+ *    - Use the \ref AQNWB::NWB::RegisteredType::create "create" method to create an instance of a registered subclass by name.
+ *    - This method takes the subclass name, path, and a shared pointer to the IO object as arguments. This
+ *      illustrates how we can read a specific typed object in an NWB file.
+ *    \snippet tests/examples/test_RegisteredType_example.cpp example_RegisterType_get_type_instance
+ *
+ * 2. **Retrieving Registered Subclass Names**:
+ *    - Use the \ref AQNWB::NWB::RegisteredType::getRegistry "getRegistry" method to retrieve the
+ *      set of registered subclass names.
+ *      \snippet tests/examples/test_RegisteredType_example.cpp example_RegisterType_get_registered_names
+ *
+ * 3. **Retrieving the Factory Map**:
+ *    - Use the \ref AQNWB::NWB::RegisteredType::getFactoryMap "getFactoryMap" method to retrieve
+ *      the map of factory functions for creating instances of registered subclasses.
+ *      \snippet tests/examples/test_RegisteredType_example.cpp example_RegisterType_get_registered_factories
+ *
+ * @subsection use_registered_type_registry_example Example: Using the type registry
+ *
+ *  \snippet tests/examples/test_RegisteredType_example.cpp example_RegisterType_full
+ *
+ * @section use_the_define_field_macro How to Use the DEFINE_FIELD macro
+ *
+ * The \ref DEFINE_FIELD macro takes the following main inputs:
+ *
+ *  * ``name``: The name of the function to generate.
+ *  * ``storageObjectType`` : One of either \ref AQNWB::NWB::DatasetField "DatasetField" or
+ *    \ref AQNWB::NWB::AttributeField "AttributeField" to define the type of storage object used
+ *    to store the field.
+ *  * ``default_type`` : The default data type to use. If not known, we can use ``std::any``.
+ *	* ``fieldPath`` : Literal string with the relative path to the field within the schema of the
+ *     respective neurodata_type. This is automatically being expanded at runtime to the full path.
+ *  * ``description`` : Description of the field to include in the docstring for the docs
+ *
+ * All of these inputs are required. A typical example will look as follows:
+ *
+ * @code
+ * DEFINE_FIELD(getData, DatasetField, float, "data", The main data)
+ * @endcode
+ *
+ * The compiler will then expand this definition to create a new method
+ * called ``getData`` that will return a  \ref AQNWB::IO::ReadDataWrapper "ReadDataWrapper"
+ * for lazy reading for the field. The corresponding expanded function will look something like:
+ *
+ * @code
+ * template<typename VTYPE = float>
+ * inline std::unique_ptr<IO::ReadDataWrapper<DatasetField, VTYPE>> getData() const
+ * {
+ *     return std::make_unique<IO::ReadDataWrapper<DatasetField, VTYPE>>(
+ *          m_io,
+ *          AQNWB::mergePaths(m_path, fieldPath));
+ * }
+ * @endcode
+ *
+ * See \ref read_page for an example of how to use such methods (e.g.,
+ * \ref AQNWB::NWB::TimeSeries::readData "TimeSeries::readData" )
+ * for reading data fields from a file.
+ *
+ *
+ */