Merge branch 'master' into dev

ISSUE_TEMPLATE.md

@@ -2,11 +2,22 @@ For better support, please use the template below to submit your issue. When you
 
 Sometimes errors while running CNMF occur during parallel processing which prevents the log to provide a meaningful error message. Please reproduce your error with setting `dview=None`.
 
-If you need to upgrade CaImAn follow the instructions given in the [wiki](https://github.com/flatironinstitute/CaImAn/wiki/Updating-CaImAn).
-
-- **Tell us about your operating system (Linux/macOS/Windows), Python version (3.x), working environment (Spyder/Jupyter Notebook/other), and (if applicable) which of the demo scripts you're using for your analysis**
-
-
+If you need to upgrade CaImAn follow the instructions given in the [documentation](https://caiman.readthedocs.io/en/master/Installation.html#upgrading).
+
+- **Tell us a bit about your setup**: 
+ 1. Operating system (Linux/macOS/Windows): 
+
+ 2. Python version (3.x): 
+
+ 3. Working environment (Python IDE/Jupyter Notebook/other):
+
+ 4. Which of the demo scripts you're using for your analysis (if applicable):
+
+ 5. CaImAn version*: 
+
+ 6. CaImAn installation process (`pip install .`/`pip install -e .`/conda): 
+
+*You can get the CaImAn version by creating a `params` object and then typing `params.data['caiman_version']`. If the field doesn't exist, type N/A and consider upgrading)
 
 - **Describe the issue that you are experiencing**
 

caiman/source_extraction/cnmf/cnmf.py

@@ -376,12 +376,12 @@ def fit_file(self, motion_correct=False, indices=None, include_eval=False):
         Cn[np.isnan(Cn)] = 0
         fit_cnm.save(fname_new[:-5]+'_init.hdf5')
         fit_cnm.params.change_params({'p': self.params.get('preprocess', 'p')})
-        # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
+        # RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
         cnm2 = fit_cnm.refit(images, dview=self.dview)
         cnm2.estimates.evaluate_components(images, cnm2.params, dview=self.dview)
-        #%% update object with selected components
+        # update object with selected components
         cnm2.estimates.select_components(use_object=True)
-        #%% Extract DF/F values
+        # Extract DF/F values
         cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)
         cnm2.estimates.Cn = Cn
         cnm2.save(cnm2.mmap_file[:-4] + 'hdf5')

caiman/source_extraction/cnmf/estimates.py

@@ -28,6 +28,10 @@
 
 
 class Estimates(object):
+    """
+    Class for storing and reusing the analysis results and performing basic
+    processing and plotting operations.
+    """
     def __init__(self, A=None, b=None, C=None, f=None, R=None, dims=None):
         """Class for storing the variables related to the estimates of spatial footprints, temporal traces,
         deconvolved neural activity, and background. Quality metrics are also stored. The class has methods
@@ -185,7 +189,7 @@ def plot_contours(self, img=None, idx=None, crd=None, thr_method='max',
             img = np.reshape(np.array(self.A.mean(1)), self.dims, order='F')
         if self.coordinates is None:  # not hasattr(self, 'coordinates'):
             self.coordinates = caiman.utils.visualization.get_contours(self.A, img.shape, thr=thr, thr_method=thr_method)
-        plt.figure()   
+        plt.figure()
         if params is not None:
             plt.suptitle('min_SNR=%1.2f, rval_thr=%1.2f, use_cnn=%i'
                          %(params.quality['SNR_lowest'],
@@ -570,7 +574,7 @@ def play_movie(self, imgs, q_max=99.75, q_min=2, gain_res=1,
 
         mov = caiman.concatenate((imgs[frame_range] - (not include_bck) * B,
                                   Y_rec + include_bck * B, Y_res * gain_res), axis=2)
-        
+
 
         if thr > 0:
             if save_movie:
@@ -810,8 +814,6 @@ def select_components(self, idx_components=None, use_object=False, save_discarde
 
     def restore_discarded_components(self):
         ''' Recover components that are filtered out with the select_components method
-        @param: None
-        @return: None
         '''
         if self.discarded_components is not None:
             for field in ['C', 'S', 'YrA', 'R', 'F_dff', 'g', 'bl', 'c1', 'neurons_sn', 'lam', 'cnn_preds','SNR_comp','r_values','coordinates']:
@@ -1019,7 +1021,7 @@ def deconvolve(self, params, dview=None, dff_flag=False):
         specified in params. Deconvolution on detrended and normalized (DF/F)
         traces can be performed by setting dff_flag=True. In this case the
         results of the deconvolution are stored in F_dff_dec and S_dff
-        
+
         Args:
             params: params object
                 Parameters of the algorithm
@@ -1063,7 +1065,7 @@ def deconvolve(self, params, dview=None, dff_flag=False):
         self.neurons_sn = [results[5][i] for i in order]
         self.lam = [results[8][i] for i in order]
         self.YrA = F - self.C
-        
+
         if dff_flag:
             if self.F_dff is None:
                 logging.warning('The F_dff field is empty. Run the method' +
@@ -1072,7 +1074,7 @@ def deconvolve(self, params, dview=None, dff_flag=False):
             else:
                 args_in = [(self.F_dff[jj], None, jj, 0, 0, self.g[jj], None,
                         args) for jj in range(F.shape[0])]
-    
+
                 if 'multiprocessing' in str(type(dview)):
                     results = dview.map_async(
                         constrained_foopsi_parallel, args_in).get(4294967)
@@ -1081,12 +1083,12 @@ def deconvolve(self, params, dview=None, dff_flag=False):
                                              args_in)
                 else:
                     results = list(map(constrained_foopsi_parallel, args_in))
-        
+
                 results = list(zip(*results))
                 order = list(results[7])
                 self.F_dff_dec = np.stack([results[0][i] for i in order])
                 self.S_dff = np.stack([results[1][i] for i in order])
-            
+
 
     def manual_merge(self, components, params):
         ''' merge a given list of components. The indices
@@ -1107,7 +1109,7 @@ def manual_merge(self, components, params):
                 set) of components, then use a list with a single (list)
                 element
             params: params object
-                
+
         Returns:
             self: estimates object
         '''
@@ -1320,6 +1322,8 @@ def remove_duplicates(self, predictions=None, r_values=None, dist_thr=0.1,
         return components_to_keep
 
     def masks_2_neurofinder(self, dataset_name):
+        """Return masks to neurofinder format
+        """
         if self.A_thr is None:
             raise Exception(
                 'You need to compute thresolded components before calling this method: use the threshold_components method')

caiman/source_extraction/cnmf/initialization.py

@@ -145,9 +145,21 @@ def initialize_components(Y, K=30, gSig=[5, 5], gSiz=None, ssub=1, tsub=1, nIter
                           SC_kernel='heat', SC_sigma=1, SC_thr=0, SC_normalize=True, SC_use_NN=False,
                           SC_nnn=20, lambda_gnmf=1):
     """
-    Initalize components
+    Initalize components. This function initializes the spatial footprints, temporal components,
+    and background which are then further refined by the CNMF iterations. There are four
+    different initialization methods depending on the data you're processing:
+        'greedy_roi': GreedyROI method used in standard 2p processing (default)
+        'corr_pnr': GreedyCorr method used for processing 1p data
+        'sparse_nmf': Sparse NMF method suitable for dendritic/axonal imaging
+        'graph_nmf': Graph NMF method also suitable for dendritic/axonal imaging
 
-    This method uses a greedy approach followed by hierarchical alternative least squares (HALS) NMF.
+    The GreedyROI method by default is not using the RollingGreedyROI method. This can
+    be changed through the binary flag 'rolling_sum'.
+
+    All the methods can be used for volumetric data except 'corr_pnr' which is only
+    available for 2D data.
+
+    It is also by default followed by hierarchical alternative least squares (HALS) NMF.
     Optional use of spatio-temporal downsampling to boost speed.
 
     Args:
@@ -188,8 +200,8 @@ def initialize_components(Y, K=30, gSig=[5, 5], gSiz=None, ssub=1, tsub=1, nIter
         img: optional [np 2d array]
             Image with which to normalize. If not present use the mean + offset
 
-        method_init: str
-            Initialization method 'greedy_roi', 'corr_pnr'. The latter can only be used for 2D data and it is compulsory for endoscopic one-photon data.
+        method_init: {'greedy_roi', 'corr_pnr', 'sparse_nmf', 'graph_nmf', 'pca_ica'}
+            Initialization method (default: 'greedy_roi')
 
         max_iter_snmf: int
             Maximum number of sparse NMF iterations
@@ -198,7 +210,7 @@ def initialize_components(Y, K=30, gSig=[5, 5], gSiz=None, ssub=1, tsub=1, nIter
             Sparsity penalty
 
         rolling_sum: boolean
-            Detect new components based on a rolling sum of pixel activity (default: True)
+            Detect new components based on a rolling sum of pixel activity (default: False)
 
         rolling_length: int
             Length of rolling window (default: 100)
@@ -322,7 +334,7 @@ def initialize_components(Y, K=30, gSig=[5, 5], gSiz=None, ssub=1, tsub=1, nIter
         Ain, Cin, _, b_in, f_in = sparseNMF(
             Y_ds, nr=K, nb=nb, max_iter_snmf=max_iter_snmf, alpha=alpha_snmf,
             sigma_smooth=sigma_smooth_snmf, remove_baseline=remove_baseline, perc_baseline=perc_baseline_snmf)
-    
+
     elif method == 'graph_nmf':
         Ain, Cin, _, b_in, f_in = graphNMF(
             Y_ds, nr=K, nb=nb, max_iter_snmf=max_iter_snmf, lambda_gnmf=lambda_gnmf,
@@ -568,7 +580,7 @@ def graphNMF(Y_ds, nr, max_iter_snmf=500, lambda_gnmf=1,
              perc_baseline=20, nb=1, truncate=2, tol=1e-3, SC_kernel='heat',
              SC_normalize=True, SC_thr=0, SC_sigma=1, SC_use_NN=False,
              SC_nnn=20):
-    
+
     m = scipy.ndimage.gaussian_filter(np.transpose(
     Y_ds, np.roll(np.arange(Y_ds.ndim), 1)), sigma=sigma_smooth,
     mode='nearest', truncate=truncate)
@@ -756,10 +768,10 @@ def finetune(Y, cin, nIter=5):
 
     Args:
         Y:  D1*d2*T*K patches
-    
+
         c: array T*K
             the inital calcium traces
-    
+
         nIter: int
             True indicates that time is listed in the last axis of Y (matlab format)
             and moves it in the front

caiman/source_extraction/cnmf/params.py

@@ -11,7 +11,8 @@
 from pprint import pformat
 
 class CNMFParams(object):
-
+    """Class for setting and changing the various parameters."""
+
     def __init__(self, fnames=None, dims=None, dxy=(1, 1),
                  border_pix=0, del_duplicates=False, low_rank_background=True,
                  memory_fact=1, n_processes=1, nb_patch=1, p_ssub=2, p_tsub=2,
@@ -521,10 +522,10 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1),
 
             gSig_filt: int or None, default: None
                 size of kernel for high pass spatial filtering in 1p data. If None no spatial filtering is performed
-            
+
             is3D: bool, default: False
                 flag for 3D recordings for motion correction
-                
+
             max_deviation_rigid: int, default: 3
                 maximum deviation in pixels between rigid shifts and shifts of individual patches
 
@@ -622,7 +623,7 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1),
             'SC_kernel': 'heat',         # kernel for graph affinity matrix
             'SC_sigma' : 1,              # std for SC kernel
             'SC_thr': 0,                 # threshold for affinity matrix
-            'SC_normalize': True,        # standardize entries prior to 
+            'SC_normalize': True,        # standardize entries prior to
                                          # computing affinity matrix
             'SC_use_NN': False,          # sparsify affinity matrix by using
                                          # only nearest neighbors
@@ -634,7 +635,7 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1),
             'gSiz': gSiz,
             'init_iter': init_iter,
             'kernel': None,           # user specified template for greedyROI
-            'lambda_gnmf' :1,         # regularization weight for graph NMF  
+            'lambda_gnmf' :1,         # regularization weight for graph NMF
             'maxIter': 5,             # number of HALS iterations
             'max_iter_snmf': 500,
             'method_init': method_init,    # can be greedy_roi, greedy_pnr sparse_nmf, local_NMF
@@ -798,7 +799,7 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1),
             if isinstance(self.data['fnames'], str):
                 self.data['fnames'] = [self.data['fnames']]
             if self.motion['is3D']:
-                T = get_file_size(self.data['fnames'], var_name_hdf5=self.data['var_name_hdf5'])[0][0]                
+                T = get_file_size(self.data['fnames'], var_name_hdf5=self.data['var_name_hdf5'])[0][0]
             else:
                 T = get_file_size(self.data['fnames'], var_name_hdf5=self.data['var_name_hdf5'])[1]
             if len(self.data['fnames']) > 1:
@@ -909,6 +910,8 @@ def __eq__(self, other):
         return True
 
     def to_dict(self):
+        """Returns the params class as a dictionary with subdictionaries for each
+        catergory."""
         return {'data': self.data, 'spatial_params': self.spatial, 'temporal_params': self.temporal,
                 'init_params': self.init, 'preprocess_params': self.preprocess,
                 'patch_params': self.patch, 'online': self.online, 'quality': self.quality,
@@ -925,6 +928,14 @@ def __repr__(self):
         return 'CNMFParams:\n\n' + '\n\n'.join(formatted_outputs)
 
     def change_params(self, params_dict, verbose=False):
+        """ Method for updating the params object by providing a single dictionary.
+        For each key in the provided dictionary the method will search in all
+        subdictionaries and will update the value if it finds a match.
+
+        Args:
+            params_dict: dictionary with parameters to be changed and new values
+            verbose: bool (False). Print message for all keys
+        """
         for gr in list(self.__dict__.keys()):
             self.set(gr, params_dict, verbose=verbose)
         for k, v in params_dict.items():

caiman/source_extraction/cnmf/utilities.py

@@ -1032,6 +1032,7 @@ def get_file_size(file_name, var_name_hdf5='mov'):
 
             else:
                 raise Exception('Unknown file type')
+            dims = tuple(dims)
         else:
             raise Exception('File not found!')
     elif isinstance(file_name, list):