Merge pull request #1408 from broadinstitute/staging

staging --> master

beta-pipelines/skylab/slidetags/SlideTags.wdl

@@ -1,6 +1,7 @@
 version 1.0
 
 import "scripts/spatial-count.wdl" as SpatialCount
+import "scripts/positioning.wdl" as Positioning
 
 workflow SlideTags {
 
@@ -10,25 +11,28 @@ workflow SlideTags {
         String id
         Array[String] fastq_paths
         Array[String] pucks
-        Int mem_GiB = 64
-        Int disk_GiB = 128
-        String docker = "us.gcr.io/broad-gotc-prod/slide-tags:1.0.0"
+        Array[String] rna_paths
+        String sb_path
+        String docker = "us.gcr.io/broad-gotc-prod/slide-tags:1.1.0"
      }
 
     parameter_meta {
         fastq_paths: "Array of paths to spatial fastq files"
         pucks: "Array of paths to puck files"
-        mem_GiB: "Memory in GiB to allocate to the task"
-        disk_GiB: "Disk in GiB to allocate to the task"
         docker: "Docker image to use"
     }
 
     call SpatialCount.count as spatial_count {
         input:
             fastq_paths = fastq_paths,
             pucks = pucks,
-            mem_GiB = mem_GiB,
-            disk_GiB = disk_GiB,
+            docker = docker
+     }
+
+    call Positioning.generate_positioning as positioning {
+        input:
+            rna_paths = rna_paths,
+            sb_path = spatial_count.sb_counts,
             docker = docker
      }
 

beta-pipelines/skylab/slidetags/scripts/CREDITS.md

@@ -6,6 +6,10 @@ This project uses code from the following sources:
   URL: [https://github.com/MacoskoLab/Macosko-Pipelines/blob/main/spatial-count/spatial-count.wdl](https://github.com/MacoskoLab/Macosko-Pipelines/blob/main/spatial-count/spatial-count.wdl)  
   This code was adapted and modified from the Macosko Lab's pipeline repository.
 
+- **Positioning Workflow**
+  URL: [https://github.com/MacoskoLab/Macosko-Pipelines/blob/main/positioning/positioning.wdl](https://github.com/MacoskoLab/Macosko-Pipelines/blob/main/positioning/positioning.wdl)  
+  This code was adapted and modified from the Macosko Lab's pipeline repository.
+
 Additional modifications include output handling and script download changes.
 
 Please refer to the original source for the full context of the workflow.

beta-pipelines/skylab/slidetags/scripts/positioning.wdl

@@ -0,0 +1,90 @@
+version 1.0
+
+task generate_positioning {
+  input {
+    Array[String] rna_paths
+    String sb_path
+    Int mem_GiB  = 128
+    Int disk_GiB = 128
+    Int nthreads = 16
+    String docker
+  }
+  command <<<
+    set -euo pipefail
+    set -x
+    echo "<< starting spatial-count >>"
+
+    gcloud config set storage/process_count 16 # is this set by user?
+    gcloud config set storage/thread_count  2 # is this set by user?
+
+    # Download the scripts -- these need to be changed -- also need to add to docker   
+    wget https://raw.githubusercontent.com/MacoskoLab/Macosko-Pipelines/ee005109446f58764509ee47ff51c212ce8dabe3/positioning/positioning.R
+    wget https://raw.githubusercontent.com/MacoskoLab/Macosko-Pipelines/6a78716aa08a9f2506c06844f7e3fd491b03aa8b/positioning/load_matrix.R
+    wget https://raw.githubusercontent.com/MacoskoLab/Macosko-Pipelines/a7fc86abbdd3d46461c500e7d024315d88a97e9a/positioning/run-positioning.R
+
+    echo "RNA: ~{sep=' ' rna_paths}"
+    echo "SB: ~{sb_path}"
+
+    # Assert that the RNA files exist
+    rnas=(~{sep=' ' rna_paths})
+    for rna in "${rnas[@]}" ; do
+        if ! gsutil stat "$rna" &> /dev/null ; then
+            echo "ERROR: gsutil stat command failed on file $rna"
+            exit 1
+        fi
+    done
+
+    # Download the RNA
+    echo "Downloading RNA:"
+    mkdir RNA
+    gcloud storage cp ~{sep=' ' rna_paths} RNA
+
+    # Assert that the SB file exists
+    if ! gsutil stat "~{sb_path}" &> /dev/null ; then
+        echo "ERROR: gsutil stat command failed on file ~{sb_path}"
+        exit 1
+    fi
+
+    # Download the SB
+    echo "Downloading SB:"
+    mkdir SB
+    gcloud storage cp ~{sb_path} SB
+
+    # Run the script
+    echo ; echo "Running run-positioning.R"
+    Rscript run-positioning.R RNA SB output
+
+    # Upload the results
+    ls output/* 
+
+    if [[ -f output/seurat.qs ]] ; then
+        echo "true" > DONE
+    else
+        echo ; echo "ERROR: CANNOT FIND: seurat.qs"
+    fi
+
+    echo; echo "Writing logs:"
+    echo; echo "RNA size:"; du -sh RNA
+    echo; echo "SB size:"; du -sh SB
+    echo; echo "output size:"; du -sh output
+    echo; echo "FREE SPACE:"; df -h
+
+    echo "tar files/logs"
+    cat stdout stderr > positioning.log
+    tar -zcvf output.tar.gz output
+    echo "<< completed positioning >>"
+  >>>
+
+  output {
+    File output_file = "output.tar.gz"
+    File positioning_log = "positioning.log"
+  }
+
+  runtime {
+    docker: docker
+    memory: "~{mem_GiB} GB"
+    disks: "local-disk ~{disk_GiB} SSD"
+    cpu: nthreads
+  }
+
+}

beta-pipelines/skylab/slidetags/scripts/spatial-count.wdl

@@ -4,8 +4,9 @@ task count {
   input {
     Array[String] fastq_paths
     Array[String] pucks
-    Int mem_GiB
-    Int disk_GiB
+    Int mem_GiB = 64
+    Int disk_GiB = 128
+    Int nthreads = 1
     String docker
   }
   command <<<
@@ -74,7 +75,6 @@ task count {
   >>>
 
   output {
-    Boolean DONE = read_boolean("DONE")
     File sb_counts = "SBcounts.h5"
     File spatial_log = "spatial-count.log"
 
@@ -83,8 +83,7 @@ task count {
     docker: docker
     memory: "~{mem_GiB} GB"
     disks: "local-disk ~{disk_GiB} SSD"
-    cpu: 1
-    preemptible: 0
+    cpu: nthreads
   }
 }
 

pipeline_versions.txt

@@ -1,40 +1,40 @@
 Pipeline Name	Version	Date of Last Commit
-CheckFingerprint	 1.0.21	2024-09-06 
+CheckFingerprint	 1.0.22	2024-10-28 
 RNAWithUMIsPipeline	 1.0.17	2024-09-06 
 AnnotationFiltration	 1.2.6	2024-09-06 
-UltimaGenomicsWholeGenomeGermline	 1.1.0	2024-09-06 
-WholeGenomeGermlineSingleSample	 3.3.1	2024-09-17 
-ExomeGermlineSingleSample	 3.2.1	2024-09-17 
+UltimaGenomicsWholeGenomeGermline	 1.1.1	2024-10-28 
+WholeGenomeGermlineSingleSample	 3.3.2	2024-10-28 
+ExomeGermlineSingleSample	 3.2.2	2024-10-28 
 JointGenotypingByChromosomePartTwo	 1.5.1	2024-09-10 
 JointGenotypingByChromosomePartOne	 1.5.1	2024-09-10 
-ReblockGVCF	 2.3.0	2024-09-06 
+ReblockGVCF	 2.3.1	2024-10-28 
 JointGenotyping	 1.7.1	2024-09-10 
 UltimaGenomicsJointGenotyping	 1.2.1	2024-09-10 
-VariantCalling	 2.2.2	2024-09-06 
-UltimaGenomicsWholeGenomeCramOnly	 1.0.21	2024-09-06 
+VariantCalling	 2.2.3	2024-10-28 
+UltimaGenomicsWholeGenomeCramOnly	 1.0.22	2024-10-28 
 GDCWholeGenomeSomaticSingleSample	 1.3.3	2024-09-06 
-BroadInternalRNAWithUMIs	 1.0.34	2024-09-06 
-BroadInternalUltimaGenomics	 1.1.0	2024-09-06 
-BroadInternalArrays	 1.1.12	2024-09-06 
+BroadInternalRNAWithUMIs	 1.0.35	2024-09-06 
+BroadInternalUltimaGenomics	 1.1.1	2024-10-28 
+BroadInternalArrays	 1.1.13	2024-10-28 
 BroadInternalImputation	 1.1.13	2024-09-06 
-Arrays	 2.6.28	2024-09-06 
+Arrays	 2.6.29	2024-10-28 
 ValidateChip	 1.16.6	2024-09-06 
 MultiSampleArrays	 1.6.2	2024-08-02 
 Imputation	 1.1.14	2024-09-06 
-IlluminaGenotypingArray	 1.12.22	2024-09-06 
-ExternalWholeGenomeReprocessing	 2.3.1	2024-09-17 
-ExternalExomeReprocessing	 3.3.1	2024-09-17 
+IlluminaGenotypingArray	 1.12.23	2024-10-28 
+ExternalWholeGenomeReprocessing	 2.3.2	2024-10-28 
+ExternalExomeReprocessing	 3.3.2	2024-10-28 
 CramToUnmappedBams	 1.1.3	2024-08-02 
-WholeGenomeReprocessing	 3.3.1	2024-09-17 
-ExomeReprocessing	 3.3.1	2024-09-17 
+WholeGenomeReprocessing	 3.3.2	2024-10-28 
+ExomeReprocessing	 3.3.2	2024-10-28 
 BuildIndices	 3.0.0	2023-12-06 
 scATAC	 1.3.2	2023-08-03 
 snm3C	 4.0.4	2024-08-06 
-Multiome	 5.7.1	2024-10-18 
-PairedTag	 1.7.1	2024-10-18 
+Multiome	 5.8.0	2024-10-23 
+PairedTag	 1.8.0	2024-10-23 
 MultiSampleSmartSeq2	 2.2.22	2024-09-11 
-MultiSampleSmartSeq2SingleNucleus	 2.0.1	2024-09-24 
-Optimus	 7.7.0	2024-09-24 
-atac	 2.3.2	2024-10-18 
+MultiSampleSmartSeq2SingleNucleus	 2.0.2	2024-10-23 
+Optimus	 7.8.0	2024-10-23 
+atac	 2.4.0	2024-10-23 
 SmartSeq2SingleSample	 5.1.21	2024-09-11 
-SlideSeq	 3.4.2	2024-09-24 
+SlideSeq	 3.4.3	2024-10-24 

pipelines/broad/arrays/single_sample/Arrays.changelog.md

@@ -1,3 +1,8 @@
+# 2.6.29
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 2.6.28
 2024-09-06 (Date of Last Commit)
 

pipelines/broad/arrays/single_sample/Arrays.wdl

@@ -23,7 +23,7 @@ import "../../../../tasks/broad/Utilities.wdl" as utils
 
 workflow Arrays {
 
-  String pipeline_version = "2.6.28"
+  String pipeline_version = "2.6.29"
 
   input {
     String chip_well_barcode

pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.changelog.md

@@ -1,3 +1,8 @@
+# 2.3.1
+2024-10-28 (Date of Last Commit)
+
+* Updated GATK for Validate Variants, which reduces the memory requirements for the task when an interval list is not provided
+
 # 2.3.0
 2024-09-06 (Date of Last Commit)
 

pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.wdl

@@ -6,7 +6,7 @@ import "../../../../../../tasks/broad/Utilities.wdl" as utils
 
 workflow ReblockGVCF {
 
-  String pipeline_version = "2.3.0"
+  String pipeline_version = "2.3.1"
 
 
   input {

pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.changelog.md

@@ -1,3 +1,8 @@
+# 3.2.2
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 3.2.1
 2024-09-17 (Date of Last Commit)
 

pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.wdl

@@ -45,7 +45,7 @@ import "../../../../../../tasks/broad/Utilities.wdl" as utils
 # WORKFLOW DEFINITION
 workflow ExomeGermlineSingleSample {
 
-  String pipeline_version = "3.2.1"
+  String pipeline_version = "3.2.2"
 
 
   input {

pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.changelog.md

@@ -1,3 +1,8 @@
+# 1.1.1
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 1.1.0
 2024-09-06 (Date of Last Commit)
 

pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.wdl

@@ -50,7 +50,7 @@ workflow UltimaGenomicsWholeGenomeGermline {
     filtering_model_no_gt_name: "String describing the optional filtering model; default set to rf_model_ignore_gt_incl_hpol_runs"
   }
 
-  String pipeline_version = "1.1.0"
+  String pipeline_version = "1.1.1"
 
 
   References references = alignment_references.references

pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.changelog.md

@@ -1,3 +1,8 @@
+# 3.3.2
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 3.3.1
 2024-09-17 (Date of Last Commit)
 

pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl

@@ -40,7 +40,7 @@ import "../../../../../../structs/dna_seq/DNASeqStructs.wdl"
 workflow WholeGenomeGermlineSingleSample {
 
 
-  String pipeline_version = "3.3.1"
+  String pipeline_version = "3.3.2"
 
 
   input {

pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.changelog.md

@@ -1,3 +1,8 @@
+# 2.2.3
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 2.2.2
 2024-09-06 (Date of Last Commit)
 

pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl

@@ -9,7 +9,7 @@ import "../../../../../tasks/broad/DragenTasks.wdl" as DragenTasks
 workflow VariantCalling {
 
 
-  String pipeline_version = "2.2.2"
+  String pipeline_version = "2.2.3"
 
 
   input {

pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.changelog.md

@@ -1,3 +1,8 @@
+# 1.0.22
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 1.0.21
 2024-09-06 (Date of Last Commit)
 

pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.wdl

@@ -43,7 +43,7 @@ workflow UltimaGenomicsWholeGenomeCramOnly {
     save_bam_file: "If true, then save intermeidate ouputs used by germline pipeline (such as the output BAM) otherwise they won't be kept as outputs."
   }
 
-  String pipeline_version = "1.0.21"
+  String pipeline_version = "1.0.22"
 
   References references = alignment_references.references
 

pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.changelog.md

@@ -1,3 +1,8 @@
+# 1.12.23
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 1.12.22
 2024-09-06 (Date of Last Commit)
 

pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl

@@ -21,7 +21,7 @@ import "../../../../tasks/broad/Qc.wdl" as Qc
 
 workflow IlluminaGenotypingArray {
 
-  String pipeline_version = "1.12.22"
+  String pipeline_version = "1.12.23"
 
   input {
     String sample_alias

pipelines/broad/internal/arrays/single_sample/BroadInternalArrays.changelog.md

@@ -1,3 +1,8 @@
+# 1.1.13
+2024-10-28 (Date of Last Commit)
+
+* Updated the docker in the ValidateVCF task; this does not affect this pipeline
+
 # 1.1.12
 2024-09-06 (Date of Last Commit)