Merge pull request #1292 from flatironinstitute/dev

dev -> master for 1.10.3 release

README.md

@@ -3,12 +3,16 @@
 
 CaImAn
 ======
-<img src="https://github.com/flatironinstitute/CaImAn/blob/main/docs/LOGOS/Caiman_logo_FI.png" width="400" align="right">
+<img src="https://github.com/flatironinstitute/CaImAn/blob/main/docs/LOGOS/Caiman_logo_2.png" width="400" align="right">
 
 A Python toolbox for large-scale **Ca**lcium **Im**aging **An**alysis.    
 
 CaImAn implements a set of essential methods required to analyze calcium and voltage imaging data. It provides fast and scalable algorithms for motion correction, source extraction, spike deconvolution, and registering neurons across multiple sessions. It is suitable for both two-photon and one-photon fluorescence microscopy data, and can be run in both offline and online modes. Documentation is [here](https://caiman.readthedocs.io/en/latest/). 
 
+Caiman Central
+--------------
+- [Caiman Central](https://github.com/flatironinstitute/caiman_central) is the hub for sharing information about CaImAn. Information on quarterly community meetings, workshops, other events, and any other communications between the developers and the user community can be found there.
+
 # Quick start :rocket:
 Follow these three steps to get started quickly, from installation to working through a demo notebook. If you do not already have conda installed, [you can find it here](https://docs.conda.io/en/latest/miniconda.html). There is a video walkthrough of the following steps [here](https://youtu.be/b63zAmKihIY?si=m7WleTwdU0rJup_2).
 
@@ -27,10 +31,10 @@ Create a working directory called `caiman_data` that includes code samples and r
 ### Step 3: Try out a demo notebook
 Go into the working directory you created in Step 2, and open a Jupyter notebook:
 
-    cd <your home>/caiman_data/demos/notebooks  
+    cd <your home>/caiman_data/
     jupyter notebook 
 
-Jupyter will open. Click on `demo_pipeline.ipynb` to get started with a demo! 
+Jupyter will open. Navigate to demos/notebooks/ and click on `demo_pipeline.ipynb` to get started with a demo.
 
 > Note that what counts as `<your home>` in the first line depends on your OS/computer. Be sure to fill in your actual home directory. On Linux/Mac it is `~` while on Windows it will be something like `C:\Users\your_user_name\` 
 
@@ -59,11 +63,9 @@ The main use cases and notebooks are listed in the following table:
 A comprehensive list of references, where you can find detailed discussion of the methods and their development, can be found [here](https://caiman.readthedocs.io/en/master/CaImAn_features_and_references.html#references). 
 
 
-# Questions, help, news
-In addition to the demos, there are many ways to learn more about Caiman and receive help:
+# How to get help
 - [Online documentation](https://caiman.readthedocs.io/en/latest/) contains a lot of general information about Caiman, the parameters, how to interpret its outputs, and more.
 - [GitHub Discussions](https://github.com/flatironinstitute/CaImAn/discussions) is our preferred venue for users to ask for help.
-- [Caiman Central](https://github.com/flatironinstitute/caiman_central) is our central hub for sharing information about CaImAn. Tune in there for updates about quarterly community meetings, workshops, etc. 
 - The [Gitter forum](https://app.gitter.im/#/room/#agiovann_Constrained_NMF:gitter.im) is our old forum: we sometimes will ask people to join us there when something can best be solved in real time (e.g., installation problems).
 - If you have found a bug, we recommend searching the [issues at github](https://github.com/flatironinstitute/CaImAn/issues) and opening a new issue if you can't find the solution there. 
 - If there is a feature you would like to see implemented, feel free to come chat at the above forums or open an issue at Github.

VERSION

@@ -1,2 +1,2 @@
-1.10.2
+1.10.3
 

bin/caiman_gui.py

@@ -12,7 +12,7 @@
 from pyqtgraph.parametertree import Parameter, ParameterTree
 from scipy.sparse import csc_matrix
 
-import caiman as cm
+import caiman
 from caiman.source_extraction.cnmf.cnmf import load_CNMF
 from caiman.source_extraction.cnmf.params import CNMFParams
 
@@ -67,10 +67,10 @@ def make_color_img(img, gain=255, min_max=None, out_type=np.uint8):
                                           directory=d, filter=f + ';;*.mmap')[0]
 
 if fpath[-3:] == 'nwb':
-    mov = cm.load(cnm_obj.mmap_file,
+    mov = caiman.load(cnm_obj.mmap_file,
                   var_name_hdf5='acquisition/TwoPhotonSeries')
 else:
-    mov = cm.load(cnm_obj.mmap_file)
+    mov = caiman.load(cnm_obj.mmap_file)
 
 estimates = cnm_obj.estimates
 params_obj = cnm_obj.params
@@ -106,7 +106,7 @@ def selectionchange(self,i):
     cb.show()
 
 if not hasattr(estimates, 'Cn'):
-    estimates.Cn = cm.local_correlations(mov, swap_dim=False)
+    estimates.Cn = caiman.local_correlations(mov, swap_dim=False)
 #Cn = estimates.Cn
 
 # We rotate our components 90 degrees right because of incompatibility of pyqtgraph and pyplot

caiman/__init__.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python
 
 import pkg_resources
-from .base.movies import movie, load, load_movie_chain, _load_behavior, play_movie
-from .base.timeseries import concatenate
-from .cluster import start_server, stop_server
-from .mmapping import load_memmap, save_memmap, save_memmap_each, save_memmap_join
-from .summary_images import local_correlations
-#from .source_extraction import cnmf
+from caiman.base.movies import movie, load, load_movie_chain, _load_behavior, play_movie
+from caiman.base.timeseries import concatenate
+from caiman.cluster import start_server, stop_server
+from caiman.mmapping import load_memmap, save_memmap, save_memmap_each, save_memmap_join
+from caiman.summary_images import local_correlations
 
 __version__ = pkg_resources.get_distribution('caiman').version

caiman/base/movies.py

@@ -30,25 +30,18 @@
 import zarr
 from zipfile import ZipFile
 
-import caiman as cm
-
-from . import timeseries
+import caiman.base.timeseries
+import caiman.base.traces
+import caiman.mmapping
+import caiman.summary_images
+import caiman.utils.visualization
 
 try:
     cv2.setNumThreads(0)
 except:
     pass
 
-from . import timeseries as ts
-from .traces import trace
-
-from ..mmapping import load_memmap
-from ..utils import visualization
-from .. import summary_images as si
-from ..motion_correction import apply_shift_online, motion_correct_online
-
-
-class movie(ts.timeseries):
+class movie(caiman.base.timeseries.timeseries):
     """
     Class representing a movie. This class subclasses timeseries,
     that in turn subclasses ndarray
@@ -350,7 +343,7 @@ def extract_shifts(self, max_shift_w: int = 5, max_shift_h: int = 5, template=No
 
         template = template[ms_h:h_i - ms_h, ms_w:w_i - ms_w].astype(np.float32)
 
-        #% run algorithm, press q to stop it
+        # run algorithm, press q to stop it
         shifts = []    # store the amount of shift in each frame
         xcorrs = []
 
@@ -549,11 +542,11 @@ def removeBL(self, windowSize:int=100, quantilMin:int=8, in_place:bool=False, re
         myperc = partial(np.percentile, q=quantilMin, axis=-1)
         res = np.array(list(map(myperc,iter_win))).T
         if returnBL:
-                return cm.movie(cv2.resize(res,pixs.shape[::-1]),fr=self.fr).to3DFromPixelxTime(self.shape)
+                return caiman.movie(cv2.resize(res,pixs.shape[::-1]),fr=self.fr).to3DFromPixelxTime(self.shape)
         if (not in_place):            
             return (pixs-cv2.resize(res,pixs.shape[::-1])).to3DFromPixelxTime(self.shape)
         else:
-            self -= cm.movie(cv2.resize(res,pixs.shape[::-1]),fr=self.fr).to3DFromPixelxTime(self.shape) 
+            self -= caiman.movie(cv2.resize(res,pixs.shape[::-1]),fr=self.fr).to3DFromPixelxTime(self.shape) 
             return self
 
     def to2DPixelxTime(self, order='F'):
@@ -611,7 +604,7 @@ def computeDFF(self, secsWindow: int = 5, quantilMin: int = 8, method: str = 'on
                         **self.__dict__)
         numFramesNew, linePerFrame, pixPerLine = np.shape(mov_out)
 
-        #% compute baseline quickly
+        # compute baseline quickly
         logging.debug("binning data ...")
         sys.stdout.flush()
 
@@ -634,7 +627,7 @@ def computeDFF(self, secsWindow: int = 5, quantilMin: int = 8, method: str = 'on
                                    cval=0.0,
                                    prefilter=False)
 
-        #% compute DF/F
+        # compute DF/F
         if not in_place:
             if method == 'delta_f_over_sqrt_f':
                 mov_out = (mov_out - movBL) / np.sqrt(movBL)
@@ -884,7 +877,7 @@ def local_correlations(self,
         T = self.shape[0]
         Cn = np.zeros(self.shape[1:])
         if T <= 3000:
-            Cn = si.local_correlations(np.array(self),
+            Cn = caiman.summary_images.local_correlations(np.array(self),
                                        eight_neighbours=eight_neighbours,
                                        swap_dim=swap_dim,
                                        order_mean=order_mean)
@@ -894,7 +887,7 @@ def local_correlations(self,
             for jj, mv in enumerate(range(n_chunks - 1)):
                 logging.debug('number of chunks:' + str(jj) + ' frames: ' +
                               str([mv * frames_per_chunk, (mv + 1) * frames_per_chunk]))
-                rho = si.local_correlations(np.array(self[mv * frames_per_chunk:(mv + 1) * frames_per_chunk]),
+                rho = caiman.summary_images.local_correlations(np.array(self[mv * frames_per_chunk:(mv + 1) * frames_per_chunk]),
                                             eight_neighbours=eight_neighbours,
                                             swap_dim=swap_dim,
                                             order_mean=order_mean)
@@ -905,7 +898,7 @@ def local_correlations(self,
 
             logging.debug('number of chunks:' + str(n_chunks - 1) + ' frames: ' +
                           str([(n_chunks - 1) * frames_per_chunk, T]))
-            rho = si.local_correlations(np.array(self[(n_chunks - 1) * frames_per_chunk:]),
+            rho = caiman.summary_images.local_correlations(np.array(self[(n_chunks - 1) * frames_per_chunk:]),
                                         eight_neighbours=eight_neighbours,
                                         swap_dim=swap_dim,
                                         order_mean=order_mean)
@@ -956,7 +949,7 @@ def partition_FOV_KMeans(self,
         fovs = cv2.resize(np.uint8(fovs), (w1, h1), 1. / fx, 1. / fy, interpolation=cv2.INTER_NEAREST)
         return np.uint8(fovs), mcoef, distanceMatrix
 
-    def extract_traces_from_masks(self, masks: np.ndarray) -> trace:
+    def extract_traces_from_masks(self, masks: np.ndarray) -> caiman.base.traces.trace:
         """
         Args:
             masks: array, 3D with each 2D slice bein a mask (integer or fractional)
@@ -975,7 +968,7 @@ def extract_traces_from_masks(self, masks: np.ndarray) -> trace:
 
         pixelsA = np.sum(A, axis=1)
         A = A / pixelsA[:, None]       # obtain average over ROI
-        traces = trace(np.dot(A, np.transpose(Y)).T, **self.__dict__)
+        traces = caiman.base.traces.trace(np.dot(A, np.transpose(Y)).T, **self.__dict__)
         return traces
 
     def resize(self, fx=1, fy=1, fz=1, interpolation=cv2.INTER_AREA):
@@ -1013,7 +1006,7 @@ def resize(self, fx=1, fy=1, fz=1, interpolation=cv2.INTER_AREA):
                 if len(new_m) == 0:
                     new_m = m_tmp
                 else:
-                    new_m = timeseries.concatenate([new_m, m_tmp], axis=0)
+                    new_m = caiman.base.timeseries.concatenate([new_m, m_tmp], axis=0)
 
             return new_m
         else:
@@ -1566,7 +1559,7 @@ def load(file_name: Union[str, list[str]],
 
         elif extension == '.mmap':
             filename = os.path.split(file_name)[-1]
-            Yr, dims, T = load_memmap(
+            Yr, dims, T = caiman.mmapping.load_memmap(
                 os.path.join(                  # type: ignore # same dims typing issue as above
                     os.path.split(file_name)[0], filename))
             images = np.reshape(Yr.T, [T] + list(dims), order='F')
@@ -1666,10 +1659,10 @@ def load_movie_chain(file_list: list[str],
             m = m[:, top:h - bottom, left:w - right, z_top:d - z_bottom]
 
         mov.append(m)
-    return ts.concatenate(mov, axis=0)
+    return caiman.base.timeseries.concatenate(mov, axis=0)
 
 ####
-# This is only used for demo_behavior, and used to be part of cm.load(), activated with the
+# This is only used for demo_behavior, and used to be part of caiman.load(), activated with the
 # 'is_behavior' boolean flag.
 
 def _load_behavior(file_name:str) -> Any:
@@ -1976,7 +1969,7 @@ def from_zip_file_to_movie(zipfile_name: str, start_end:Optional[tuple] = None)
 
                     counter += 1
 
-    return cm.movie(mov[:counter])
+    return caiman.movie(mov[:counter])
 
 
 def from_zipfiles_to_movie_lists(zipfile_name: str, max_frames_per_movie: int = 3000,
@@ -2210,23 +2203,24 @@ def load_iter(file_name: Union[str, list[str]], subindices=None, var_name_hdf5:
             logging.error(f"File request:[{file_name}] not found!")
             raise Exception('File not found!')
 
-def get_file_size(file_name, var_name_hdf5='mov') -> tuple[tuple, Union[int, tuple]]:
-    """ Computes the dimensions of a file or a list of files without loading
+def get_file_size(file_name, var_name_hdf5:str='mov') -> tuple[tuple, Union[int, tuple]]:
+    """
+    Computes the dimensions of a file or a list of files without loading
     it/them in memory. An exception is thrown if the files have FOVs with
     different sizes
-        Args:
-            file_name: str/filePath or various list types
-                locations of file(s)
 
-            var_name_hdf5: 'str'
-                if loading from hdf5 name of the dataset to load
+    Args:
+        file_name:
+            locations of file(s)
+        var_name_hdf5:
+            if loading from hdf5 name of the dataset to load
 
-        Returns:
-            dims: tuple
-                dimensions of FOV
+    Returns:
+        dims: tuple
+            dimensions of FOV
+        T: int or tuple of int
+            number of timesteps in each file
 
-            T: int or tuple of int
-                number of timesteps in each file
     """
     # TODO There is a lot of redundant code between this, load(), and load_iter() that should be unified somehow
     if isinstance(file_name, pathlib.Path):
@@ -2314,8 +2308,7 @@ def get_file_size(file_name, var_name_hdf5='mov') -> tuple[tuple, Union[int, tup
         else:
             raise Exception('File not found!')
     elif isinstance(file_name, tuple):
-        from ...base.movies import load
-        dims = load(file_name[0], var_name_hdf5=var_name_hdf5).shape
+        dims = caiman.base.movies.load(file_name[0], var_name_hdf5=var_name_hdf5).shape
         T = len(file_name)
 
     elif isinstance(file_name, list):
@@ -2494,7 +2487,7 @@ def animate(i):
         anim = matplotlib.animation.FuncAnimation(fig, animate, frames=frames, interval=1, blit=True)
 
         # call our new function to display the animation
-        return visualization.display_animation(anim, fps=fr)
+        return caiman.utils.visualization.display_animation(anim, fps=fr)
 
     elif backend == 'embed_opencv':
         stopButton = widgets.ToggleButton(

caiman/base/rois.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 
 import cv2
 import json
@@ -21,7 +20,7 @@
 from typing import Any, Optional
 import zipfile
 
-from ..motion_correction import tile_and_correct
+from caiman.motion_correction import tile_and_correct
 
 try:
     cv2.setNumThreads(0)
@@ -242,7 +241,7 @@ def nf_match_neurons_in_binary_masks(masks_gt,
     cm_cnmf = [scipy.ndimage.center_of_mass(mm) for mm in masks_comp]
 
     if D is None:
-        #% find distances and matches
+        # find distances and matches
         # find the distance between each masks
         D = distance_masks([A_ben, A_cnmf], [cm_ben, cm_cnmf], min_dist, enclosed_thr=enclosed_thr)
 
@@ -252,7 +251,7 @@ def nf_match_neurons_in_binary_masks(masks_gt,
     matches = matches[0]
     costs = costs[0]
 
-    #%% compute precision and recall
+    # compute precision and recall
     TP = np.sum(np.array(costs) < thresh_cost) * 1.
     FN = np.shape(masks_gt)[0] - TP
     FP = np.shape(masks_comp)[0] - TP
@@ -264,7 +263,7 @@ def nf_match_neurons_in_binary_masks(masks_gt,
     performance['accuracy'] = (TP + TN) / (TP + FP + FN + TN)
     performance['f1_score'] = 2 * TP / (2 * TP + FP + FN)
     logging.debug(performance)
-    #%%
+
     idx_tp = np.where(np.array(costs) < thresh_cost)[0]
     idx_tp_ben = matches[0][idx_tp]    # ground truth
     idx_tp_cnmf = matches[1][idx_tp]   # algorithm - comp
@@ -471,7 +470,7 @@ def register_ROIs(A1,
     matches = matches[0]
     costs = costs[0]
 
-    #%% store indices
+    # store indices
 
     idx_tp = np.where(np.array(costs) < thresh_cost)[0]
     if len(idx_tp) > 0:
@@ -488,7 +487,7 @@ def register_ROIs(A1,
         non_matched1 = list(range(D[0].shape[0]))
         non_matched2 = list(range(D[0].shape[1]))
 
-    #%% compute precision and recall
+    # compute precision and recall
 
     FN = D[0].shape[0] - TP
     FP = D[0].shape[1] - TP
@@ -1001,7 +1000,7 @@ def getfloat():
     fill_color = get32()
     subtype = get16()
     if subtype != 0:
-        raise ValueError('roireader: ROI subtype %s not supported (!= 0)' % subtype)
+        raise ValueError(f'roireader: ROI subtype {subtype} not supported (!= 0)')
     options = get16()
     arrow_style = get8()
     arrow_head_size = get8()