diff --git a/cellbender/VERSION.txt b/cellbender/VERSION.txt index 6833eaa..9fc80f9 100644 --- a/cellbender/VERSION.txt +++ b/cellbender/VERSION.txt @@ -1 +1 @@ -0.3.1.dev0 \ No newline at end of file +0.3.2 \ No newline at end of file diff --git a/cellbender/remove_background/cli.py b/cellbender/remove_background/cli.py index 29f7928..fee0479 100644 --- a/cellbender/remove_background/cli.py +++ b/cellbender/remove_background/cli.py @@ -117,14 +117,22 @@ def validate_args(args) -> argparse.Namespace: args.fpr = fpr_list_correct_dtypes # Ensure that "exclude_features" specifies allowed features. - # As of CellRanger 6.0, the possible features are: + # As of CellRanger 7.2, the possible features are: # Gene Expression # Antibody Capture # CRISPR Guide Capture # Custom # Peaks + # Multiplexing Capture + # VDJ + # VDJ-T + # VDJ-T-GD + # VDJ-B + # Antigen Capture allowed_features = ['Gene Expression', 'Antibody Capture', - 'CRISPR Guide Capture', 'Custom', 'Peaks'] + 'CRISPR Guide Capture', 'Custom', 'Peaks', + 'Multiplexing Capture', 'VDJ', 'VDJ-T', + 'VDJ-T-GD', 'VDJ-B', 'Antigen Capture'] for feature in args.exclude_features: if feature not in allowed_features: sys.stdout.write(f"Specified '{feature}' using --exclude-feature-types, " diff --git a/cellbender/remove_background/tests/benchmarking/run_benchmark.py b/cellbender/remove_background/tests/benchmarking/run_benchmark.py index efb8520..ab31ca3 100644 --- a/cellbender/remove_background/tests/benchmarking/run_benchmark.py +++ b/cellbender/remove_background/tests/benchmarking/run_benchmark.py @@ -141,7 +141,9 @@ def cromshell_submit(wdl: str, submit_cmd = ['cromshell', 'submit', tmp_wdl, inputs, + '--options-json', options, + '--dependencies-zip', dependencies_zip] # submit job diff --git a/cellbender/remove_background/tests/benchmarking/run_benchmark_result_tabulation.py b/cellbender/remove_background/tests/benchmarking/run_benchmark_result_tabulation.py index ffc5e5a..e5c997c 100644 --- a/cellbender/remove_background/tests/benchmarking/run_benchmark_result_tabulation.py +++ b/cellbender/remove_background/tests/benchmarking/run_benchmark_result_tabulation.py @@ -69,7 +69,7 @@ def get_cromshell_output_h5(workflow: str, grep: str = '_out.h5') -> Union[str, """Use cromshell list-outputs to get the relevant file gsURL""" output = grep_from_command(['cromshell', 'list-outputs', workflow], grep=grep) - out = output[:-1].decode().split('\n') + out = output.decode().lstrip('run_cellbender_benchmark.h5_array: ').rstrip('\n').split('\n') if len(out) > 1: return out else: @@ -95,18 +95,18 @@ def metadata_from_workflow_id(workflow: str) -> Tuple[str, str, Optional[str]]: # git hash output = grep_from_command(['cromshell', 'metadata', workflow], grep='"git_hash":') - git_hash = output[17:-3].decode() + git_hash = output.decode().split('"git_hash": ')[-1].lstrip('"').split('"')[0] # input file output = grep_from_command(['cromshell', 'metadata', workflow], grep='run_cellbender_benchmark.cb.input_file_unfiltered') - input_file = output[58:-3].decode() + input_file = 'gs://' + output.decode().split('gs://')[-1].split('"')[0] # truth file output = grep_from_command(['cromshell', 'metadata', workflow], grep='run_cellbender_benchmark.cb.truth_file') if 'null' not in output.decode(): - truth_file = output[47:-3].decode() + truth_file = 'gs://' + output.decode().split('gs://')[-1].split('"')[0] else: truth_file = None diff --git a/cellbender/remove_background/train.py b/cellbender/remove_background/train.py index 8d77808..961e06e 100644 --- a/cellbender/remove_background/train.py +++ b/cellbender/remove_background/train.py @@ -152,7 +152,6 @@ def run_training(model: RemoveBackgroundPyroModel, # Initialize train and tests ELBO with empty lists. train_elbo = [] - test_elbo = [] lr = [] epoch_checkpoint_freq = 1000 # a large number... it will be recalculated @@ -212,16 +211,15 @@ def run_training(model: RemoveBackgroundPyroModel, if epoch % test_freq == 0: model.eval() total_epoch_loss_test = evaluate_epoch(svi, test_loader) - test_elbo.append(-total_epoch_loss_test) model.loss['test']['epoch'].append(epoch) model.loss['test']['elbo'].append(-total_epoch_loss_test) logger.info("[epoch %03d] average test loss: %.4f" % (epoch, total_epoch_loss_test)) # Check whether test ELBO has spiked beyond specified conditions. - if (epoch_elbo_fail_fraction is not None) and (len(test_elbo) > 2): - current_diff = max(0., test_elbo[-2] - test_elbo[-1]) - overall_diff = np.abs(test_elbo[-2] - test_elbo[0]) + if (epoch_elbo_fail_fraction is not None) and (len(model.loss['test']['elbo']) > 2): + current_diff = max(0., model.loss['test']['elbo'][-2] - model.loss['test']['elbo'][-1]) + overall_diff = np.abs(model.loss['test']['elbo'][-2] - model.loss['test']['elbo'][0]) fractional_spike = current_diff / overall_diff if fractional_spike > epoch_elbo_fail_fraction: raise ElboException( @@ -245,15 +243,20 @@ def run_training(model: RemoveBackgroundPyroModel, # Check on the final test ELBO to see if it meets criteria. if final_elbo_fail_fraction is not None: - best_test_elbo = max(test_elbo) - if test_elbo[-1] < best_test_elbo: - final_best_diff = best_test_elbo - test_elbo[-1] - initial_best_diff = best_test_elbo - test_elbo[0] - if (final_best_diff / initial_best_diff) > final_elbo_fail_fraction: + best_test_elbo = max(model.loss['test']['elbo']) + if model.loss['test']['elbo'][-1] < best_test_elbo: + final_best_diff = best_test_elbo - model.loss['test']['elbo'][-1] + initial_best_diff = best_test_elbo - model.loss['test']['elbo'][0] + if initial_best_diff == 0: raise ElboException( - f'Training failed because final test loss {test_elbo[-1]:.2f} ' + f"Training failed because there was no improvement from the initial test loss {model.loss['test']['elbo'][0]:.2f}. " + f"Final test loss was {model.loss['test']['elbo'][-1]}" + ) + elif (final_best_diff / initial_best_diff) > final_elbo_fail_fraction: + raise ElboException( + f"Training failed because final test loss {model.loss['test']['elbo'][-1]:.2f} " f'is not sufficiently close to best test loss {best_test_elbo:.2f}, ' - f'compared to the initial test loss {test_elbo[0]:.2f}. ' + f"compared to the initial test loss {model.loss['test']['elbo'][0]:.2f}. " f'Fractional difference is {final_best_diff / initial_best_diff:.2f}, ' f'which is > specified final_elbo_fail_fraction {final_elbo_fail_fraction:.2f}' ) @@ -284,14 +287,14 @@ def run_training(model: RemoveBackgroundPyroModel, logger.info(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) # Check final ELBO meets conditions. - if (final_elbo_fail_fraction is not None) and (len(test_elbo) > 1): - best_test_elbo = max(test_elbo) - if -test_elbo[-1] >= -best_test_elbo * (1 + final_elbo_fail_fraction): - raise ElboException(f'Training failed because final test loss ({-test_elbo[-1]:.4f}) ' + if (final_elbo_fail_fraction is not None) and (len(model.loss['test']['elbo']) > 1): + best_test_elbo = max(model.loss['test']['elbo']) + if -model.loss['test']['elbo'][-1] >= -best_test_elbo * (1 + final_elbo_fail_fraction): + raise ElboException(f"Training failed because final test loss ({-model.loss['test']['elbo'][-1]:.4f}) " f'exceeds best test loss ({-best_test_elbo:.4f}) by >= ' f'{100 * final_elbo_fail_fraction:.1f}%') # Free up all the GPU memory we can once training is complete. torch.cuda.empty_cache() - return train_elbo, test_elbo + return train_elbo, model.loss['test']['elbo'] diff --git a/docs/source/_static/remove_background/v0.3.2_hgmm.png b/docs/source/_static/remove_background/v0.3.2_hgmm.png new file mode 100644 index 0000000..a9ca57f Binary files /dev/null and b/docs/source/_static/remove_background/v0.3.2_hgmm.png differ diff --git a/docs/source/changelog/index.rst b/docs/source/changelog/index.rst index 378c73b..f16d7a6 100644 --- a/docs/source/changelog/index.rst +++ b/docs/source/changelog/index.rst @@ -11,33 +11,37 @@ edge case bug fixes, speedups, and small new features might bump up the last digit of the version number. For example, the difference between 0.2.1 and 0.2.0 represents this kind of small change. -Version 0.1.0 + +Version 0.3.2 ------------- -This was the initial release. The output count matrix was constructed via -imputation, so that there were no explicit guarantees that CellBender would -only subtract counts and never add. +Small improvements aimed at reducing memory footprint, along with bug fixes. -This version has been deprecated, and we do not recommend using it any longer. +Improvements: -- Imputes the "denoised" count matrix using a variational autoencoder +- Make posterior generation more memory efficient -Version 0.2.0 -------------- +New features: -A significant overhaul of the model and the output generation procedure were -undertaken to explicitly guarantee that CellBender only subtracts counts and -never adds. The output is not constructed by imputation or smoothing, and -CellBender intentionally tries to modify the raw data as little as possible in -order to achieve denoising. A nominal false positive rate is approximately -controlled at the level of the entire dataset, to prevent removal of too much -signal. +- WDL workflow updates to facilitate automatic retries on failure +- Added to list of allowed feature types to match 2024.04 CellRanger definitions -- Uses a variational autoencoder as a prior +Bug fixes: -- Computes the "denoised" count matrix using a MAP estimate and posterior regularization +- Fix bug with MTX inputs for WDL +- Fix Windows bug during posterior generation +- Fix report generation bugs on Mac and Windows + + +(Version 0.3.1 -- redacted) +--------------------------- + +WARNING: redacted + +If you managed to obtain a copy of v0.3.1 before it was redacted, do not use it. An integer +overflow bug caused outputs to be incorrect in nearly all cases. For more information, see +`github issue 347 here `_. - - CellBender never adds counts Version 0.3.0 ------------- @@ -84,6 +88,37 @@ a workflow using Google Colab on a GPU for free. hundreds of samples in automated pipelines. This file can be parsed to look for indications that a sample may need to be re-run. + +Version 0.2.0 +------------- + +A significant overhaul of the model and the output generation procedure were +undertaken to explicitly guarantee that CellBender only subtracts counts and +never adds. The output is not constructed by imputation or smoothing, and +CellBender intentionally tries to modify the raw data as little as possible in +order to achieve denoising. A nominal false positive rate is approximately +controlled at the level of the entire dataset, to prevent removal of too much +signal. + +- Uses a variational autoencoder as a prior + +- Computes the "denoised" count matrix using a MAP estimate and posterior regularization + + - CellBender never adds counts + + +Version 0.1.0 +------------- + +This was the initial release. The output count matrix was constructed via +imputation, so that there were no explicit guarantees that CellBender would +only subtract counts and never add. + +This version has been deprecated, and we do not recommend using it any longer. + +- Imputes the "denoised" count matrix using a variational autoencoder + + Human-mouse mixture benchmark ----------------------------- @@ -137,3 +172,14 @@ v0.3.0 .. image:: /_static/remove_background/v0.3.0_hgmm.png :width: 750 px + +This represents a real improvement over the results published in the paper. + +v0.3.2 +~~~~~~ + +.. image:: /_static/remove_background/v0.3.2_hgmm.png + :width: 750 px + +This appears identical to v0.3.0, as the changes were intended to fix bugs and +reduce memory footprint. diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst index ead0ce7..2f47629 100644 --- a/docs/source/reference/index.rst +++ b/docs/source/reference/index.rst @@ -185,4 +185,5 @@ The information contained in the posterior can be used to quantitatively answer questions such as "What is the probability that the number of viral gene counts in this cell is nonzero?" For help with these kinds of computations, please open a -`github issue `_. +`github issue `_, or see +the `semi-worked example on the github issue here `_. diff --git a/wdl/cellbender_remove_background.wdl b/wdl/cellbender_remove_background.wdl index f7507ec..d3a964b 100644 --- a/wdl/cellbender_remove_background.wdl +++ b/wdl/cellbender_remove_background.wdl @@ -90,6 +90,7 @@ task run_cellbender_remove_background_gpu { git clone -q https://github.com/broadinstitute/CellBender.git /cromwell_root/CellBender cd /cromwell_root/CellBender git checkout -q ~{dev_git_hash__} + yes | pip install -U pip setuptools yes | pip install --no-cache-dir -U -e /cromwell_root/CellBender pip list cd /cromwell_root diff --git a/wdl/cellbender_remove_background_azure.wdl b/wdl/cellbender_remove_background_azure.wdl new file mode 100644 index 0000000..503a465 --- /dev/null +++ b/wdl/cellbender_remove_background_azure.wdl @@ -0,0 +1,250 @@ +version 1.0 + +## Copyright Broad Institute, 2022 +## +## LICENSING : +## This script is released under the WDL source code license (BSD-3) +## (see LICENSE in https://github.com/openwdl/wdl). + +task run_cellbender_remove_background_gpu { + + input { + + # File-related inputs + String sample_name + File input_file_unfiltered # all barcodes, raw data + File? barcodes_file # for MTX and NPZ formats, the bacode information is in a separate file + File? genes_file # for MTX and NPZ formats, the gene information is in a separate file + File? checkpoint_file # start from a saved checkpoint + File? truth_file # only for developers using simulated data + + # Outputs + String? output_bucket_base_directory # Google bucket path + + # Docker image with CellBender + String? docker_image = "us.gcr.io/broad-dsde-methods/cellbender:0.3.0" + + # Used by developers for testing non-dockerized versions of CellBender + String? dev_git_hash__ # leave blank to run CellBender normally + + # Method configuration inputs + Int? expected_cells + Int? total_droplets_included + Float? force_cell_umi_prior + Float? force_empty_umi_prior + String? model + Int? low_count_threshold + String? fpr # in quotes: floats separated by whitespace: the output false positive rate(s) + Int? epochs + Int? z_dim + String? z_layers # in quotes: integers separated by whitespace + Float? empty_drop_training_fraction + Float? learning_rate + String? exclude_feature_types # in quotes: strings separated by whitespace + String? ignore_features # in quotes: integers separated by whitespace + Float? projected_ambient_count_threshold + Float? checkpoint_mins + Float? final_elbo_fail_fraction + Float? epoch_elbo_fail_fraction + Int? num_training_tries + Float? learning_rate_retry_mult + Int? posterior_batch_size + Boolean? estimator_multiple_cpu + Boolean? constant_learning_rate + Boolean? debug + + # Hardware-related inputs + String? hardware_zones = "us-east1-d us-east1-c us-central1-a us-central1-c us-west1-b" + Int? hardware_disk_size_GB = 50 + Int? hardware_boot_disk_size_GB = 20 + Int? hardware_preemptible_tries = 0 + Int? hardware_max_retries = 0 + Int? hardware_cpu_count = 4 + Int? hardware_memory_GB = 32 + String? hardware_gpu_type = "nvidia-tesla-t4" + String? nvidia_driver_version = "470.82.01" # need >=465.19.01 for CUDA 11.3 + + } + + # For development only: install a non dockerized version of CellBender + Boolean install_from_git = (if defined(dev_git_hash__) then true else false) + + # Compute the output bucket directory for this sample: output_bucket_base_directory/sample_name/ + String output_bucket_directory = (if defined(output_bucket_base_directory) + then sub(select_first([output_bucket_base_directory]), "/+$", "") + "/${sample_name}/" + else "") + + command { + + set -e # fail the workflow if there is an error + set -x + + # install a specific commit of cellbender from github if called for (-- developers only) + if [[ ~{install_from_git} == true ]]; then + echo "Uninstalling pre-installed cellbender" + yes | pip uninstall cellbender + echo "Installing cellbender from github" + # this more succinct version is broken in some older versions of cellbender + echo "pip install --no-cache-dir -U git+https://github.com/broadinstitute/CellBender.git@~{dev_git_hash__}" + # yes | pip install --no-cache-dir -U git+https://github.com/broadinstitute/CellBender.git@~{dev_git_hash__} + # this should always work + git clone -q https://github.com/broadinstitute/CellBender.git /cromwell_root/CellBender + cd /cromwell_root/CellBender + git checkout -q ~{dev_git_hash__} + yes | pip install --no-cache-dir -U -e /cromwell_root/CellBender + pip list + cd /cromwell_root + fi + + # put the barcodes_file in the right place, if it is provided + if [[ ! -z "~{barcodes_file}" ]]; then + dir=$(dirname ~{input_file_unfiltered}) + if [[ "~{input_file_unfiltered}" == *.npz ]]; then + name="row_index.npy" + elif [[ "~{barcodes_file}" == *.gz ]]; then + name="barcodes.tsv.gz" + else + name="barcodes.tsv" + fi + echo "Moving barcodes file to "$dir"/"$name + echo "mv ~{barcodes_file} "$dir"/"$name + [ -f $dir/$name ] || mv ~{barcodes_file} $dir/$name + fi + + # put the genes_file in the right place, if it is provided + if [[ ! -z "~{genes_file}" ]]; then + dir=$(dirname ~{input_file_unfiltered}) + if [[ "~{input_file_unfiltered}" == *.npz ]]; then + name="col_index.npy" + elif [[ "~{genes_file}" == *.gz ]]; then + name="features.tsv.gz" + else + name="genes.tsv" + fi + echo "Moving genes file to "$dir"/"$name + echo "mv ~{genes_file} "$dir"/"$name + [ -f $dir/$name ] || mv ~{genes_file} $dir/$name + fi + + # use the directory as the input in the case of an MTX file + if [[ "~{input_file_unfiltered}" == *.mtx* ]]; then + input=$(dirname ~{input_file_unfiltered}) + else + input=~{input_file_unfiltered} + fi + + cellbender remove-background \ + --input $input \ + --output "~{sample_name}_out.h5" \ + ~{"--checkpoint " + checkpoint_file} \ + ~{"--expected-cells " + expected_cells} \ + ~{"--total-droplets-included " + total_droplets_included} \ + ~{"--fpr " + fpr} \ + ~{"--model " + model} \ + ~{"--low-count-threshold " + low_count_threshold} \ + ~{"--epochs " + epochs} \ + ~{"--force-cell-umi-prior " + force_cell_umi_prior} \ + ~{"--force-empty-umi-prior " + force_empty_umi_prior} \ + ~{"--z-dim " + z_dim} \ + ~{"--z-layers " + z_layers} \ + ~{"--empty-drop-training-fraction " + empty_drop_training_fraction} \ + ~{"--exclude-feature-types " + exclude_feature_types} \ + ~{"--ignore-features " + ignore_features} \ + ~{"--projected-ambient-count-threshold " + projected_ambient_count_threshold} \ + ~{"--learning-rate " + learning_rate} \ + ~{"--checkpoint-mins " + checkpoint_mins} \ + ~{"--final-elbo-fail-fraction " + final_elbo_fail_fraction} \ + ~{"--epoch-elbo-fail-fraction " + epoch_elbo_fail_fraction} \ + ~{"--num-training-tries " + num_training_tries} \ + ~{"--learning-rate-retry-mult " + learning_rate_retry_mult} \ + ~{"--posterior-batch-size " + posterior_batch_size} \ + ~{true="--estimator-multiple-cpu " false=" " estimator_multiple_cpu} \ + ~{true="--constant-learning-rate " false=" " constant_learning_rate} \ + ~{true="--debug " false=" " debug} \ + ~{"--truth " + truth_file} + + # copy outputs to google bucket if output_bucket_base_directory is supplied + echo ~{output_bucket_directory} + # commenting out b/c cant use gsutil in azure + if [[ ! -z "~{output_bucket_directory}" ]]; then + echo "Copying output data to ~{output_bucket_directory} using gsutil cp" + gsutil -m cp ~{sample_name}_out* ~{output_bucket_directory} + fi + + } + + output { + File log = "${sample_name}_out.log" + File pdf = "${sample_name}_out.pdf" + File cell_csv = "${sample_name}_out_cell_barcodes.csv" + Array[File] metrics_array = glob("${sample_name}_out*_metrics.csv") # a number of outputs depending on "fpr" + Array[File] report_array = glob("${sample_name}_out*_report.html") # a number of outputs depending on "fpr" + Array[File] h5_array = glob("${sample_name}_out*.h5") # a number of outputs depending on "fpr" + String output_dir = "${output_bucket_directory}" + File ckpt_file = "ckpt.tar.gz" + } + + runtime { + docker: "${docker_image}" + bootDiskSizeGb: hardware_boot_disk_size_GB + disks: "local-disk ${hardware_disk_size_GB} HDD" + memory: "${hardware_memory_GB}G" + cpu: hardware_cpu_count + zones: "${hardware_zones}" + preemptible: hardware_preemptible_tries + checkpointFile: "ckpt.tar.gz" + maxRetries: hardware_max_retries # can be used in case of a PAPI error code 2 failure to install GPU drivers + } + meta { + author: "Stephen Fleming" + email: "sfleming@broadinstitute.org" + description: "WDL that runs CellBender remove-background on a GPU on Google Cloud hardware. See the [CellBender GitHub repo](https://github.com/broadinstitute/CellBender) and [read the documentation](https://cellbender.readthedocs.io/en/v0.3.0/reference/index.html#command-line-options) for more information." + } + + parameter_meta { + sample_name : + {help: "Name which will be prepended to output files and which will be used to construct the output google bucket file path, if output_bucket_base_directory is supplied, as output_bucket_base_directory/sample_name/"} + input_file_unfiltered : + {help: "Input file. This must be raw data that includes all barcodes. See http://cellbender.readthedocs.io for more information on file formats, but importantly, this must be one file, and cannot be a pointer to a directory that contains MTX and TSV files."} + barcodes_file : + {help: "Supply this only if your input_file_unfiltered is a sparse NPZ matrix from Optimus lacking metadata."} + genes_file : + {help: "Supply this only if your input_file_unfiltered is a sparse NPZ matrix from Optimus lacking metadata."} + output_bucket_base_directory : + {help: "Optional google bucket gsURL. If provided, the workflow will create a subfolder called sample_name in this directory and copy outputs there. (Note that the output data would then exist in two places.) Helpful for organization."} + docker_image : + {help: "CellBender docker image. Not all CellBender docker image tags will be compatible with this WDL.", + suggestions: ["us.gcr.io/broad-dsde-methods/cellbender:0.3.0"]} + checkpoint_file : + {help: "Optional gsURL for a checkpoint file created using a previous run ('ckpt.tar.gz') on the same dataset (using the same CellBender docker image). This can be used to create a new output with a different --fpr without re-doing the training run."} + truth_file : + {help: "Optional file only used by CellBender developers or those trying to benchmark CellBender remove-background on simulated data. Normally, this input would not be supplied."} + hardware_preemptible_tries : + {help: "If nonzero, CellBender will be run on a preemptible instance, at a lower cost. If preempted, your run will not start from scratch, but will start from a checkpoint that is saved by CellBender and recovered by Cromwell. For example, if hardware_preemptible_tries is 2, your run will attempt twice using preemptible instances, and if the job is preempted both times before completing, it will finish on a non-preemptible machine. The cost savings is significant. The potential drawback is that preemption wastes time."} + hardware_max_retries : + {help: "If nonzero when CellBender exits without success it will be retried. If one also sets the memory_retry_multiplier workflow option, and the exit happens to be detected as an out of memory error, then the retry will also increase the memory allocated to the next run. The potential benefit is that one can start CellBender with less memory, and memory will be increased only when needed. The potential drawback is that the job will be retried even if the error is not a memory error."} + checkpoint_mins : + {help: "Time in minutes between creation of checkpoint files. Bear in mind that Cromwell copies checkpoints to a bucket every ten minutes."} + hardware_gpu_type : + {help: "Specify the type of GPU that should be used. Ensure that the selected hardware_zones have the GPU available.", + suggestions: ["nvidia-tesla-t4", "nvidia-tesla-k80"]} + } + +} + +workflow cellbender_remove_background { + + call run_cellbender_remove_background_gpu + + output { + File log = run_cellbender_remove_background_gpu.log + File summary_pdf = run_cellbender_remove_background_gpu.pdf + File cell_barcodes_csv = run_cellbender_remove_background_gpu.cell_csv + Array[File] metrics_csv_array = run_cellbender_remove_background_gpu.metrics_array + Array[File] html_report_array = run_cellbender_remove_background_gpu.report_array + Array[File] h5_array = run_cellbender_remove_background_gpu.h5_array + String output_directory = run_cellbender_remove_background_gpu.output_dir + File checkpoint_file = run_cellbender_remove_background_gpu.ckpt_file + } + +}