Merge pull request #391 from UPHL-BioNGS/update-20241203

Update 20241203
UPHL-BioNGS · Dec 9, 2024 · 021cd6f · 021cd6f
2 parents 42b1fde + 1b4796e
commit 021cd6f
Show file tree

Hide file tree

Showing 92 changed files with 4,594 additions and 3,956 deletions.
diff --git a/.dockstore.yml b/.dockstore.yml
@@ -1,6 +1,6 @@
 version: "v.3.7.20230502"
 workflows:
-   - name: Cecret
-     subclass: NFL
-     publish: True
-     primaryDescriptorPath: ./nextflow.config
+  - name: Cecret
+    subclass: NFL
+    publish: True
+    primaryDescriptorPath: ./nextflow.config
diff --git a/.github/workflows/check_versions.yml b/.github/workflows/check_versions.yml
@@ -3,7 +3,7 @@ name: Check versions
 on:
   workflow_dispatch:
   schedule:
-    - cron: '30 4 * * TUE'
+    - cron: "30 4 * * TUE"
 
 jobs:
   check:
@@ -16,7 +16,7 @@ jobs:
 
       - name: Checkout LFS objects
         run: git lfs checkout
-        
+
       - name: versions
         run: |
           issue_text="Needing updates:  " 
@@ -54,7 +54,7 @@ jobs:
           echo $issue_text
           cat versions.txt
           echo 'ISSUE_TEXT='$issue_text >> $GITHUB_ENV
-          
+
       - name: Create Issue
         run: |
           gh issue create --title "Version check" --body "$ISSUE_TEXT" --repo $GITHUB_REPOSITORY

diff --git a/.github/workflows/github_actions.config b/.github/workflows/github_actions.config
@@ -1,134 +1,29 @@
 process {
 	errorStrategy = { task.attempt < 2 ? 'retry' : 'terminate'}
-	withName:aci{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:artic{
-		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-	}
-	withName:artic_read_filtering{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:bbnorm{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:bcftools_variants{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:bwa{
-    		publishDir = [ path: "cecret", mode: 'link', pattern: 'logs/*/*log' ]
-  	}
-	withName:download{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:fasta_prep{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:summary{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:unzip{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:fastp{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:fastqc{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:freyja_variants{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:freyja_demix{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:freyja_aggregate{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:heatcluster{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:igv_reports{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:iqtree2{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:ivar_consensus{
-    		memory = '4 GB'
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:ivar_variants{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:ivar_trim{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:kraken2{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:mafft{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:minimap2{
-    		publishDir = [ path: "cecret", mode: 'link', pattern: 'logs/*/*log' ]
-  	}
-	withName:multiqc_combine{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:nextclade_dataset{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:nextclade{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:pango_collapse{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:pangolin{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:phytreeviz{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:samtools_stats{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:samtools_coverage{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:samtools_flagstat{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:samtools_depth{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:samtools_ampliconstats{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:samtools_plot_ampliconstats{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:samtools_sort{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:samtools_filter{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:samtools_ampliconclip{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:samtools_markdup{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:seqyclean{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:snpdists{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
-  	}
-	withName:vadr{
-    		publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
+	publishDir = [ path: "cecret", mode: 'link', saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
+    withLabel:process_single {
+        cpus   = { 1    }
+        memory = { 6.GB }
+        time   = { 4.h  }
+    }
+    withLabel:process_low {
+        cpus   = { 2     }
+        memory = { 12.GB }
+        time   = { 4.h   }
+    }
+    withLabel:process_medium {
+        cpus   = { 2     }
+        memory = { 12.GB }
+        time   = { 4.h   }
+    }
+    withLabel:process_high {
+        cpus   = { 2     }
+        memory = { 12.GB }
+        time   = { 4.h   }
+    }
+
+	withName: "IVAR.*" {
+    	memory = '4 GB'
   	}
 }
 
diff --git a/.github/workflows/sample_sheet.csv b/.github/workflows/sample_sheet.csv
@@ -0,0 +1,6 @@
+sample,fastq_1,fastq_2
+paired_test,reads/SRR13957123_1.fastq.gz,reads/SRR13957123_2.fastq.gz
+single_test,reads/SRR25097869.fastq.gz,single
+wastewater_test,reads/SRR31220305_1.fastq.gz,reads/SRR31220305_2.fastq.gz
+fasta_test,fastas/SRR13957123DB.fa,fasta
+nanopore_test,nanopore/SRR31458670_1.fastq.gz,ont
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -5,7 +5,6 @@ on: [pull_request, workflow_dispatch]
 run-name: Download fastq and fasta and run
 
 jobs:
-
   test:
     runs-on: ubuntu-20.04
     steps:
@@ -16,31 +15,31 @@ jobs:
 
       - name: Checkout LFS objects
         run: git lfs checkout
-          
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
-          
+
       - name: Run Cecret
         run: |
           docker --version
-          
+
           mkdir reads
           wget -q ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR139/023/SRR13957123/SRR13957123_1.fastq.gz
           wget -q ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR139/023/SRR13957123/SRR13957123_2.fastq.gz
           mv *fastq.gz reads/.
-          
+
           mkdir -p fastas
           wget -q https://raw.githubusercontent.com/StaPH-B/docker-builds/master/tests/SARS-CoV-2/SRR13957123.consensus.fa
           mv SRR13957123.consensus.fa SRR13957123DB.fa
           mv *fa fastas/.
-          
-          nextflow run . -profile docker -c .github/workflows/github_actions.config --primer_set ncov_V3 --maxcpus 2 --medcpus 2 --cleaner 'fastp' --aligner 'minimap2' --mpileup_depth 200 
+
+          nextflow run . -profile docker -c .github/workflows/github_actions.config --primer_set ncov_V3 --cleaner 'fastp' --aligner 'minimap2' --mpileup_depth 200 --fastas fastas --reads reads
 
           ls cecret*
             
           cat cecret*/cecret_results.txt
 
       - name: Clean
-        run: rm -rf work .nextflow*
+        run: rm -rf work .nextflow*
diff --git a/.github/workflows/test_kraken2.yml b/.github/workflows/test_kraken2.yml
@@ -5,7 +5,6 @@ on: [pull_request, workflow_dispatch]
 run-name: Test profile
 
 jobs:
-
   test:
     runs-on: ubuntu-20.04
     steps:
@@ -16,25 +15,25 @@ jobs:
 
       - name: Checkout LFS objects
         run: git lfs checkout
-          
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
 
       - name: Download Kraken2 Database
         run: |
-            mkdir kraken2_db
-            cd kraken2_db
-            wget https://genome-idx.s3.amazonaws.com/kraken/k2_viral_20240112.tar.gz 
-            tar -zxvf k2_viral_20240112.tar.gz
-            cd ../
-          
+          mkdir kraken2_db
+          cd kraken2_db
+          wget https://genome-idx.s3.amazonaws.com/kraken/k2_viral_20240112.tar.gz 
+          tar -zxvf k2_viral_20240112.tar.gz
+          cd ../
+
       - name: Run Cecret
         run: |
           nextflow run . -profile docker,test \
             -c .github/workflows/github_actions.config \
-            --maxcpus 2 --medcpus 2 --cleaner 'fastp' --aligner 'minimap2' --mpileup_depth 200 --vadr false \
+            --cleaner 'fastp' --aligner 'minimap2' --mpileup_depth 200 --vadr false \
             --kraken2_db kraken2_db --kraken2
 
           ls cecret*
@@ -43,7 +42,7 @@ jobs:
 
       - name: Kraken2 results
         run: |
-          wc -l cecret/kraken2/*_kraken2_report.txt
+          wc -l cecret*/kraken2/*_kraken2_report.txt
 
       - name: Clean
-        run: rm -rf work .nextflow*
+        run: rm -rf work .nextflow*
diff --git a/.github/workflows/test_mpx.yml b/.github/workflows/test_mpx.yml
@@ -5,18 +5,17 @@ on: [pull_request, workflow_dispatch]
 run-name: MPX
 
 jobs:
-
   test:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
         uses: actions/checkout@master
-          
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
-      
+
       - name: Download reads
         run: |
           mkdir reads
@@ -27,7 +26,7 @@ jobs:
 
       - name: Run Cecret
         run: |
-          nextflow run . -profile docker,mpx -c .github/workflows/github_actions.config --maxcpus 2 --medcpus 2
+          nextflow run . -profile docker,mpx -c .github/workflows/github_actions.config --single_reads reads
 
           ls cecret*
             

diff --git a/.github/workflows/test_mpx_idt.yml b/.github/workflows/test_mpx_idt.yml
@@ -5,13 +5,12 @@ on: [pull_request, workflow_dispatch]
 run-name: MPX with IDT Primers
 
 jobs:
-
   mpx_idt:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
         uses: actions/checkout@master
-          
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
@@ -27,7 +26,7 @@ jobs:
 
       - name: Run Cecret
         run: |
-          nextflow run . -profile docker,mpx_idt -c .github/workflows/github_actions.config --maxcpus 2 --medcpus 2
+          nextflow run . -profile docker,mpx_idt -c .github/workflows/github_actions.config --reads reads
 
           ls cecret*
             

diff --git a/.github/workflows/test_mpx_primalseq.yml b/.github/workflows/test_mpx_primalseq.yml
@@ -5,18 +5,17 @@ on: [pull_request, workflow_dispatch]
 run-name: mpx_primalseq
 
 jobs:
-
   test:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
         uses: actions/checkout@master
-          
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
-      
+
       - name: Download reads
         run: |
           mkdir reads
@@ -27,7 +26,7 @@ jobs:
 
       - name: Run Cecret
         run: |
-          nextflow run . -profile docker,mpx_primalseq -c .github/workflows/github_actions.config --maxcpus 2 --medcpus 2
+          nextflow run . -profile docker,mpx_primalseq -c .github/workflows/github_actions.config --reads reads
 
           ls cecret*