From 910d7ba6688612464a2c7e7ddcfa4e6b90253c34 Mon Sep 17 00:00:00 2001 From: LaraFuhrmann <55209716+LaraFuhrmann@users.noreply.github.com> Date: Fri, 6 Sep 2024 15:42:59 +0200 Subject: [PATCH] update feature-benchmark branch (#158) * adding viloca to the workflow * fix file paths * update conig_schema * adding paired_end_read_merger * update rule for paired_en_read_merger + add conda env definition * update rule for paired_en_read_merger + add conda env definition * update enviroment * add paired_end_read_merger * update paired_end_read_merger rule * add conda package to env * Viloca touch ups - snakemake handles `directory()` - VILOCA 1.0.0 on bioconda - use GitLab for smallgenomeutilites due to GitHub LFS limitations * Linting * Generate copnfig manual * Documentation touch ups - update paper: V-pipe 3.0 pre-print - WorkflowHub's required documentation.md points to our documentation - Update Tutorial intro page: point to installer * improve tutorial_0_install.md * Links to bioconda in installation tutorial * Singularity+conda support * Update README.md with viloca reference * Add issue templates * Contributing guidelines * Disable linter for templates * bugfix: viloca - typo fix - all: depends on correct CSV * bugfix: bump versions of bcftools and cyvcf2 - recent change of numpy in bioconda breaks older cyvcf * [bump] LolliPop 0.4.0 - support new `filters` option * Update of conf manual * [bump] LolliPop v0.4.1 --------- Co-authored-by: Ivan Blagoev Topolsky Co-authored-by: Prajwal Kulkarni Co-authored-by: Ivan Blagoev Topolsky --- .github/ISSUE_TEMPLATE/01_report_bug.md | 28 +++ .github/ISSUE_TEMPLATE/02_request_feature.md | 17 ++ .github/ISSUE_TEMPLATE/03_ask.md | 5 + .github/ISSUE_TEMPLATE/04_discuss.md | 5 + .github/ISSUE_TEMPLATE/config.yml | 1 + .mega-linter.yml | 2 + .snakemake-workflow-catalog.yml | 2 +- CONTRIBUTING.md | 57 +++++ README.md | 6 +- config/config.html | 231 +++++++++--------- docs/README.md | 20 +- docs/tutorial_0_install.md | 16 +- .../resources/local_haplotype_setup/README.md | 2 +- workflow/Snakefile | 3 + workflow/documentation.md | 20 ++ workflow/envs/bcftools.yaml | 4 +- workflow/envs/lollipop.yaml | 2 +- workflow/envs/paired_end_read_merger.yaml | 22 ++ workflow/envs/viloca.yaml | 8 + workflow/rules/common.smk | 11 +- workflow/rules/signatures.smk | 5 +- workflow/rules/snv.smk | 128 +++++++++- workflow/schemas/config_schema.json | 95 ++++++- 23 files changed, 547 insertions(+), 143 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/01_report_bug.md create mode 100644 .github/ISSUE_TEMPLATE/02_request_feature.md create mode 100644 .github/ISSUE_TEMPLATE/03_ask.md create mode 100644 .github/ISSUE_TEMPLATE/04_discuss.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 CONTRIBUTING.md create mode 100644 workflow/documentation.md create mode 100644 workflow/envs/paired_end_read_merger.yaml create mode 100644 workflow/envs/viloca.yaml diff --git a/.github/ISSUE_TEMPLATE/01_report_bug.md b/.github/ISSUE_TEMPLATE/01_report_bug.md new file mode 100644 index 00000000..3e9e9522 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/01_report_bug.md @@ -0,0 +1,28 @@ +--- +name: 🐛 Report a bug +about: Tell us if something is broken or needs attention +labels: t:bug, help wanted, good first issue, needs triage +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior, e.g.: +1. V-pipe configuration file used '...' +2. Samples TSV file used '....' +3. Commands executed '.vpipe --core 4 ...' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. Linux, Mac OS] + - Version [e.g. v3.0.0, master branch] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/02_request_feature.md b/.github/ISSUE_TEMPLATE/02_request_feature.md new file mode 100644 index 00000000..2bcedb8f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/02_request_feature.md @@ -0,0 +1,17 @@ +--- +name: 🙋 Request a feature +about: Suggest an improvement, tell about your idea +labels: t:feat, help wanted, good first issue, needs triage +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when ... + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/03_ask.md b/.github/ISSUE_TEMPLATE/03_ask.md new file mode 100644 index 00000000..3dceabc5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/03_ask.md @@ -0,0 +1,5 @@ +--- +name: ❓ Ask +about: Ask a question about this project +labels: t:ask, needs triage +--- diff --git a/.github/ISSUE_TEMPLATE/04_discuss.md b/.github/ISSUE_TEMPLATE/04_discuss.md new file mode 100644 index 00000000..ec714d61 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/04_discuss.md @@ -0,0 +1,5 @@ +--- +name: 💬 Discuss +about: Talk to the team +labels: t:talk, needs triage +--- diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..3ba13e0c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.mega-linter.yml b/.mega-linter.yml index f76eb615..1de43e4e 100644 --- a/.mega-linter.yml +++ b/.mega-linter.yml @@ -20,3 +20,5 @@ SHOW_ELAPSED_TIME: true DEFAULT_BRANCH: master # any further fix to LICENSE.md will break licensee's detection scheme MARKDOWN_MARKDOWNLINT_FILTER_REGEX_EXCLUDE: LICENSE\.md +# GitHub passes HTML comments verbatime from template so we can't use +FILTER_REGEX_EXCLUDE: (\.github/ISSUE_TEMPLATE/.*\.md) diff --git a/.snakemake-workflow-catalog.yml b/.snakemake-workflow-catalog.yml index 0b5f012b..acc7d8e1 100644 --- a/.snakemake-workflow-catalog.yml +++ b/.snakemake-workflow-catalog.yml @@ -2,5 +2,5 @@ usage: software-stack-deployment: conda: true singularity: false - singularity+conda: false + singularity+conda: true report: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..367aad01 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,57 @@ + +# Contributing to V-pipe + +A big welcome and thank you for considering contributing to V-pipe! It’s people like you that make it a reality for users in our community. + +Reading and following these guidelines will help us make the contribution process easy and effective for everyone involved. It also communicates that you agree to respect the time of the developers managing and developing these open source projects. In return, we will reciprocate that respect by addressing your issue, assessing changes, and helping you finalize your pull requests. + +## Quicklinks + +* [Getting Started](#getting-started) + * [Issues](#issues) + * [Pull Requests](#pull-requests) +* [Getting Help](#getting-help) + +## Getting Started + +Contributions are made to this repo via Issues and Pull Requests (PRs). A few general guidelines that cover both: + +- Search for existing Issues and PRs before creating your own. +- We work hard to makes sure issues are handled in a timely manner but, depending on the impact, it could take a while to investigate the root cause. A friendly ping in the comment thread to the submitter or a contributor can help draw attention if your issue is blocking. + +### Issues + +Issues should be used to report problems with the V-pipe workflow, request a new feature, or to discuss potential changes before a PR is created. When you create a new Issue, a template will be loaded that will guide you through collecting and providing the information we need to investigate. + +If you find an Issue that addresses the problem you're having, please add your own reproduction information to the existing issue rather than creating a new one. Adding a [reaction](https://github.blog/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) can also help be indicating to our maintainers that a particular problem is affecting more than just the reporter. + +### Pull Requests + +PRs to our workflow are always welcome and can be a quick way to get your fix or improvement slated for the next release. In general, PRs should: + +- Target our staging branch: [rubicon](https://github.com/cbg-ethz/V-pipe/tree/rubicon) +- Only fix/add the functionality in question **OR** address wide-spread whitespace/style issues, not both. +- Add unit or integration tests for fixed or changed functionality (if a test suite already exists). + - Or at least provide a minimalist example dataset +- Address a single concern in the least number of changed lines as possible. +- Include documentation in the repo or on our `docs/` directory. + +For changes that address core functionality or would require breaking changes (e.g. a major release), it's best to open an Issue to discuss your proposal first. This is not required but can save time creating and reviewing changes. + +In general, we follow the ["fork-and-pull" Git workflow](https://github.com/susam/gitpr) + +1. Fork the repository to your own Github account +2. Clone the project to your machine +3. Create a branch locally with a succinct but descriptive name +4. Commit changes to the branch +5. Following any formatting and testing guidelines specific to this repo + - We rely on [snakefmt](https://github.com/snakemake/snakefmt) for Snakemake files + - We use [Mega-Linter](https://megalinter.io) for the remaining files (Python (Black), Jupyter (Jupyfmt), Markdown (Markdownlint), Bash (Shellcheck), Perl (Perlcritic), Docker (Hadolint)) + - Ask us for help if you have trouble linting your code +6. Push changes to your fork +7. Open a PR in our repository and follow the PR template so that we can efficiently review the changes. + +## Getting Help + +Join us in the [V-pipe Gitter channel](https://gitter.im/V-pipe/community) (also [accessible over matrix](https://matrix.to/#/#V-pipe_community:gitter.im?utm_source=gitter) from your favorite client) and post your question there to reach out the devs. +For further inquiries, you can also contact the V-pipe Dev Team by opening a ticket at [v-pipe@bsse.ethz.ch](mailto:v-pipe@bsse.ethz.ch). diff --git a/README.md b/README.md index 4eb3cbac..e961012c 100644 --- a/README.md +++ b/README.md @@ -162,9 +162,9 @@ Other dependencies are managed by using isolated conda environments per rule, an If you use this software in your research, please cite: -Posada-Céspedes S., Seifert D., Topolsky I., Jablonski K.P., Metzner K.J., and Beerenwinkel N. 2021. -"V-pipe: a computational pipeline for assessing viral genetic diversity from high-throughput sequencing data." -_Bioinformatics_, January. doi:[10.1093/bioinformatics/btab015](https://doi.org/10.1093/bioinformatics/btab015). +Fuhrmann, L., Jablonski, K. P., Topolsky, I., Batavia, A. A., Borgsmueller, N., Icer Baykal, P., Carrara, M. ... & Beerenwinkel, (2023). +"V-Pipe 3.0: A Sustainable Pipeline for Within-Sample Viral Genetic Diversity Estimation." +_bioRxiv_, doi:[10.1101/2023.10.16.562462](https://doi.org/10.1101/2023.10.16.562462). ## Contributions diff --git a/config/config.html b/config/config.html index 10359a1e..e0dc6e21 100644 --- a/config/config.html +++ b/config/config.html @@ -1,22 +1,22 @@ - V-pipe configuration

V-pipe configuration

Type: object

The V-pipe workflow can be customized through the configuration file config.yaml or config.json or, for backward compatibility with the legacy INI-style format used in V-pipe v1.x/2.x, vpipe.config. This configuration file is a text file written using a basic structure composed of sections, properties and values. When using YAML or JSON format use these languages associative array/dictionaries in two levels for sections and properties. When using the older INI format, sections are expected in squared brackets, and properties are followed by corresponding values.

Further more, it is possible to specify additional options on the command line using Snakemake’s --configfile to pass additional YAML/JSON configuration files, and/or using Snakemake’s --config to pass sections and properties in a YAML Flow style/JSON syntax.

The order of precedence is:
command line options (--config, --configfile) >> default configuration file (config/config.yaml or config.yaml) >> legacy configuration INI (vpipe.config) >> Virus-specific base config (virus_based_config) >> default values

Example: For instance, we suggest providing as input a tabular file specifying sample unique identifiers (e.g., patient identifiers), and dates for different sequencing runs related to the same patient. The name of this file (here, samples.tsv) can be provided by specifying the section as input and the property as samples_file, as follows in the example below.

In this document, we provide a comprehensive list of all user-configurable options stratified by sections.
Example:

input:
-  samples_file: samples.tsv
-

root general
Type: object Default: {}

This section of the configuration provides general options that control the overall behavior of the pipeline.

root general virus_base_config
Type: string Default: ""

We provide virus-specific base configuration files which contain handy defaults for, e.g., HIV and SARS-CoV-2. Check the git repository’s config subdirectory to learn about them.
Examples:

hiv
-
sars-cov-2
-

root general preprocessor
Type: enum (of string) Default: "prinseq"

By default trimming and clipping of reads is performed by PRINSEQ 1 – a versatile raw read processor for short-reads with many customization options, that we use mostly for Illumina short-read sequencing.
Some other sequencing platforms, e.g., Oxford Nanopore Technologies, are not compatible with this software and usually perform quality control during the fast5 basecalling and demultiplexing anyway, e.g., by Guppy. Use skip to avoid performing preprocessing such already quality-trimmed fastq files.

  1. Schmieder, R. and Edwards, R. Quality control and preprocessing of metagenomic datasets. Bioinformatics. 2011. ↩

Must be one of:

  • "prinseq"
  • "skip"
  • ""

Example:

skip
-

root general aligner
Type: enum (of string) Default: "ngshmmalign"

There are three options for mapping reads, either using ngshmmalign, BWA MEM (bwa) 1, Bowtie 2 (bowtie) 2, or minimap2 (minimap)3. To use a different aligner than the default, indicate which aligner you want to use by setting the property aligner.

Note: Some virus-specific base configuration specified in virus_base_config might change this option’s default to a more appropriate aligner for that virus, e.g., depending on its usual diversity and mutation rate.
You are still free to override that default in your configuration shall the need arise.

  1. Li, H. Aligning sequence reads, clone sequences and assembly contigs with BWA-MEM. 2013. ↩

  2. Langmead, B. and Salzberg, S. Fast gapped-read alignment with Bowtie 2. Nature Methods. 2012. ↩

  3. Li, H. Minimap2: pairwise alignment for nucleotide sequences. Bioinformatics. 2018. ↩

Must be one of:

  • "ngshmmalign"
  • "bwa"
  • "bowtie"
  • "minimap"

Example:

minimap
-

root general primers_trimmer
Type: enum (of string) Default: "ivar"

There are two options available for trimming primers, either using iVar trim (ivar) 1 or Samtools ampliconclip (samtools) 2. iVar trim is used by default. If you prefer to use Samtools ampliconclip, then indicate so in the configuration file as in the example

  1. Grubaugh, N. et al. An amplicon-based sequencing framework for accurately measuring intrahost virus diversity using PrimalSeq and iVar. Genome Biology. 2019. ↩

  2. Danecek P, et al. Twelve years of SAMtools and BCFtools. GigaScience. 2021 ↩

Must be one of:

  • "ivar"
  • "samtools"

Example:

samtools
-

root general snv_caller
Type: enum (of string) Default: "shorah"

There are two options available for calling single nucleotide variants, either using ShoRAH (shorah) 1 or LoFreq (lofreq) 2. ShoRAH is used by default. If you prefer to use LoFreq, then indicate so in the configuration file as in the example

  1. Zagordi, O. et al. ShoRAH: estimating the genetic diversity of a mixed sample from next-generation sequencing data. BMC Bioinformatics. 2011. ↩

  2. Wilm, A. et al. LoFreq: A sequence-quality aware, ultra-sensitive variant caller for uncovering cell-population heterogeneity from high-throughput sequencing datasets. Nucleic Acids Res. 2012. ↩

Must be one of:

  • "shorah"
  • "lofreq"

Example:

lofreq
-

root general haplotype_reconstruction
Type: enum (of string) Default: "savage"

There are three options available for haplotype reconstruction, namely SAVAGE 1, HaploClique 2 or PredictHaplo 3. SAVAGE is used by default. If you wish to use HaploClique, then indicate it in the configuration file as in the example.

  1. Baaijens, J. A. et al., De novo assembly of viral quasispecies using overlap graphs. Genome Res. 2017. ↩

  2. Töpfer, A. et al. Viral quasispecies assembly via maximal clique finding. PLOS Computational Biology. 2014. ↩

  3. Prabhakaran, S. et al. HIV haplotype inference using a propagating dirichlet process mixture model. IEEE/ACM transactions on computational biology and bioinformatics 11.1. 2013. ↩

Must be one of:

  • "savage"
  • "haploclique"
  • "predicthaplo"

Example:

haploclique
-

root general threads
Type: integer Default: 1

This option should be used to specify the default number of threads for all multi-threaded rules. That is, unless the number of threads is specified for each rule, this value is set as default.

Value must be greater or equal to 1


Example:

4
-

root general checksum
Type: enum (of string) Default: "md5"

Sets the algorithm to be used when computing checksums for uploadable data.

Must be one of:

  • "md5"
  • "sha1"
  • "sha256"
  • "sha224"
  • "sha384"
  • "sha512"
  • "xxh64"
  • "xxh32"
  • "xxh128"

Example:

sha256
-

root general temp_prefix
Type: string Default: ""

Some step of V-pipe produce temporary files such as, e.g., decompressed intermediate — i.e. files which aren’t kept long-term but are deleted after all steps that needed them have finished. By default, these files are written in the output data directory. This option, makes it is possible to write them in a different directory instead. Use this option to, e.g., leverage a faster cluster-local storage or avoid wasting backup space on a snapshotted storage. You might want to consult the documentation provided by your HPC.
Examples:

temp
-
/cluster/scratch
-

root general tsvbased
Type: enum (of integer) Default: 1

Specify whether TSV files like coverage and base counts should be 1-based (i.e.: the first base pair position is called 1) like standard practice used in biology and most text formats such as VCF and GFF, or should be 0-based (i.e.: the first base pair position is called 0) like in several Python tools such as pysam and the BED format.

By default V-pipe uses 1-based TSV file (position column starts with 1), but this option change the behaviour.

Must be one of:

  • 0
  • 1

Example:

0
+ V-pipe configuration 
V-pipe configuration
   
 
Type: object
 
The V-pipe workflow can be customized through the configuration file config.yaml or config.json or, for backward compatibility with the legacy INI-style format used in V-pipe v1.x/2.x, vpipe.config. This configuration file is a text file written using a basic structure composed of sections, properties and values. When using YAML or JSON format use these languages associative array/dictionaries in two levels for sections and properties. When using the older INI format, sections are expected in squared brackets, and properties are followed by corresponding values.
 
Further more, it is possible to specify additional options on the command line using Snakemake’s --configfile to pass additional YAML/JSON configuration files, and/or using Snakemake’s --config to pass sections and properties in a YAML Flow style/JSON syntax.
 
The order of precedence is:
 command line options (--config, --configfile) >> default configuration file (config/config.yaml or config.yaml) >> legacy configuration INI (vpipe.config) >> Virus-specific base config (virus_based_config) >> default values
 
Example: For instance, we suggest providing as input a tabular file specifying sample unique identifiers (e.g., patient identifiers), and dates for different sequencing runs related to the same patient. The name of this file (here, samples.tsv) can be provided by specifying the section as input and the property as samples_file, as follows in the example below.
 
In this document, we provide a comprehensive list of all user-configurable options stratified by sections.
  
 
Example:
 
input:
+  samples_file: samples.tsv
+
 
 
 
 
 
  
 
 
 
 
root    general
Type: object Default: {}
 
This section of the configuration provides general options that control the overall behavior of the pipeline.
  
 
 
 
  
 
 
 
 
root    general    virus_base_config
Type: string Default: ""
 
We provide virus-specific base configuration files which contain handy defaults for, e.g., HIV and SARS-CoV-2. Check the git repository’s config subdirectory to learn about them.
  
 
Examples:
 
hiv
+
 
sars-cov-2
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    general    preprocessor
Type: enum (of string) Default: "prinseq"
 
By default trimming and clipping of reads is performed by PRINSEQ 1 – a versatile raw read processor for short-reads with many customization options, that we use mostly for Illumina short-read sequencing.
 Some other sequencing platforms, e.g., Oxford Nanopore Technologies, are not compatible with this software and usually perform quality control during the fast5 basecalling and demultiplexing anyway, e.g., by Guppy. Use skip to avoid performing preprocessing such already quality-trimmed fastq files.
 
 
 
     
  1.  
    Schmieder, R. and Edwards, R. Quality control and preprocessing of metagenomic datasets. Bioinformatics. 2011. ↩
     
    2.  
 
 
 
Must be one of:
 
  • "prinseq"
  • "skip"
  • ""
 
 
 
Example:
 
skip
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    general    aligner
Type: enum (of string) Default: "ngshmmalign"
 
There are three options for mapping reads, either using ngshmmalign, BWA MEM (bwa) 1, Bowtie 2 (bowtie) 2, or minimap2 (minimap)3. To use a different aligner than the default, indicate which aligner you want to use by setting the property aligner.
 
Note: Some virus-specific base configuration specified in virus_base_config might change this option’s default to a more appropriate aligner for that virus, e.g., depending on its usual diversity and mutation rate.
 You are still free to override that default in your configuration shall the need arise.
 
 
 
     
  1.  
    Li, H. Aligning sequence reads, clone sequences and assembly contigs with BWA-MEM. 2013. ↩
     
    2.  
  2.  
    Langmead, B. and Salzberg, S. Fast gapped-read alignment with Bowtie 2. Nature Methods. 2012. ↩
     
    3.  
  3.  
    Li, H. Minimap2: pairwise alignment for nucleotide sequences. Bioinformatics. 2018. ↩
     
    4.  
 
 
 
Must be one of:
 
  • "ngshmmalign"
  • "bwa"
  • "bowtie"
  • "minimap"
 
 
 
Example:
 
minimap
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    general    primers_trimmer
Type: enum (of string) Default: "ivar"
 
There are two options available for trimming primers, either using iVar trim (ivar) 1 or Samtools ampliconclip (samtools) 2. iVar trim is used by default. If you prefer to use Samtools ampliconclip, then indicate so in the configuration file as in the example
 
 
 
     
  1.  
    Grubaugh, N. et al. An amplicon-based sequencing framework for accurately measuring intrahost virus diversity using PrimalSeq and iVar. Genome Biology. 2019. ↩
     
    2.  
  2.  
    Danecek P, et al. Twelve years of SAMtools and BCFtools. GigaScience. 2021 ↩
     
    3.  
 
 
 
Must be one of:
 
  • "ivar"
  • "samtools"
 
 
 
Example:
 
samtools
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    general    snv_caller
Type: enum (of string) Default: "shorah"
 
There are three options available for calling single nucleotide variants, either using ShoRAH (shorah) 1, LoFreq (lofreq) 2 or VILOCA (viloca) [^7] . ShoRAH is used by default. If you prefer to use LoFreq, then indicate so in the configuration file as in the example
 
 
 
     
  1.  
    Zagordi, O. et al. ShoRAH: estimating the genetic diversity of a mixed sample from next-generation sequencing data. BMC Bioinformatics. 2011. ↩
     
    2.  
  2.  
    Wilm, A. et al. LoFreq: A sequence-quality aware, ultra-sensitive variant caller for uncovering cell-population heterogeneity from high-throughput sequencing datasets. Nucleic Acids Res. 2012. ↩
     
    3.  
 
 
 
Must be one of:
 
  • "shorah"
  • "lofreq"
  • "viloca"
 
 
 
Example:
 
lofreq
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    general    haplotype_reconstruction
Type: enum (of string) Default: "savage"
 
There are three options available for haplotype reconstruction, namely SAVAGE 1, HaploClique 2 or PredictHaplo 3. SAVAGE is used by default. If you wish to use HaploClique, then indicate it in the configuration file as in the example.
 
 
 
     
  1.  
    Baaijens, J. A. et al., De novo assembly of viral quasispecies using overlap graphs. Genome Res. 2017. ↩
     
    2.  
  2.  
    Töpfer, A. et al. Viral quasispecies assembly via maximal clique finding. PLOS Computational Biology. 2014. ↩
     
    3.  
  3.  
    Prabhakaran, S. et al. HIV haplotype inference using a propagating dirichlet process mixture model. IEEE/ACM transactions on computational biology and bioinformatics 11.1. 2013. ↩
     
    4.  
 
 
 
Must be one of:
 
  • "savage"
  • "haploclique"
  • "predicthaplo"
 
 
 
Example:
 
haploclique
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    general    threads
Type: integer Default: 1
 
This option should be used to specify the default number of threads for all multi-threaded rules. That is, unless the number of threads is specified for each rule, this value is set as default.
  
Value must be greater or equal to 1
 
 
Example:
 
4
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    general    checksum
Type: enum (of string) Default: "md5"
 
Sets the algorithm to be used when computing checksums for uploadable data.
 
 
Must be one of:
 
  • "md5"
  • "sha1"
  • "sha256"
  • "sha224"
  • "sha384"
  • "sha512"
  • "xxh64"
  • "xxh32"
  • "xxh128"
 
 
 
Example:
 
sha256
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    general    temp_prefix
Type: string Default: ""
 
Some step of V-pipe produce temporary files such as, e.g., decompressed intermediate — i.e. files which aren’t kept long-term but are deleted after all steps that needed them have finished. By default, these files are written in the output data directory. This option, makes it is possible to write them in a different directory instead. Use this option to, e.g., leverage a faster cluster-local storage or avoid wasting backup space on a snapshotted storage. You might want to consult the documentation provided by your HPC.
  
 
Examples:
 
temp
+
 
/cluster/scratch
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    general    tsvbased
Type: enum (of integer) Default: 1
 
Specify whether TSV files like coverage and base counts should be 1-based (i.e.: the first base pair position is called 1) like standard practice used in biology and most text formats such as VCF and GFF, or should be 0-based (i.e.: the first base pair position is called 0) like in several Python tools such as pysam and the BED format.
 
By default V-pipe uses 1-based TSV file (position column starts with 1), but this option change the behaviour.
 
 
Must be one of:
 
  • 0
  • 1
 
 
 
Example:
 
0
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    general    id_separator
Type: string Default: "/"
 
Charater to use when assembling the two levels (e.g.: sample and a date), into a column title to be used in a report TSV file
 
E.g., with this sample file
 
patient1    20100113
 patient1    20110202
 patient2    20081130
-
 
the coverage TSV file’s column will be called patient1/20100113, patient1/20110202 and patient2/20081130.
  
 
Example:
 
-
-
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input
Type: object Default: {}
 
Properties in this section of the configuration control the input of the pipeline.
  
 
 
 
  
 
 
 
 
root    input    datadir
Type: string Default: "samples/"
 
The input file for the workflow will be searched in this directory.
 
V-pipe expects the input samples to be organized in a two-level directory hierarchy.
 
     
  • The first level can be, e.g., patient samples or biological replicates of an experiment.
    •  
  • The second level can be, e.g., different sampling dates or different sequencing runs of the same sample.
    •  
  • Inside that directory, the sub-directory raw_data holds the sequencing data in FASTQ format (optionally compressed with GZip).
    •  
 
For example:
 
 
📁samples
+
 
the coverage TSV file’s column will be called patient1/20100113, patient1/20110202 and patient2/20081130.
  
 
Example:
 
-
+
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input
Type: object Default: {}
 
Properties in this section of the configuration control the input of the pipeline.
  
 
 
 
  
 
 
 
 
root    input    datadir
Type: string Default: "samples/"
 
The input file for the workflow will be searched in this directory.
 
V-pipe expects the input samples to be organized in a two-level directory hierarchy.
 
     
  • The first level can be, e.g., patient samples or biological replicates of an experiment.
    •  
  • The second level can be, e.g., different sampling dates or different sequencing runs of the same sample.
    •  
  • Inside that directory, the sub-directory raw_data holds the sequencing data in FASTQ format (optionally compressed with GZip).
    •  
 
For example:
 
📁samples
 ├──📁patient1
 │  ├──📁20100113
 │  │  └──📁raw_data
@@ -31,55 +31,55 @@
       └──📁raw_data
          ├──🧬patient2_20081130_R1.fastq.gz
          └──🧬patient2_20081130_R2.fastq.gz
-
 
  
 
Examples:
 
tests/data/hiv/
-
 
tests/data/sars-cov-2/
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    paired
Type: boolean Default: true
 
Indicate whether the input sequencing reads correspond to paired-end reads.
 
Paired-ended reads need to be in split files with _R1 and _R2 suffixes:
 
 
📁raw_data
+
  
 
Examples:
 
tests/data/hiv/
+
 
tests/data/sars-cov-2/
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    paired
Type: boolean Default: true
 
Indicate whether the input sequencing reads correspond to paired-end reads.
 
Paired-ended reads need to be in split files with _R1 and _R2 suffixes:
 
📁raw_data
 ├──🧬patient2_20081130_R1.fastq.gz
 └──🧬patient2_20081130_R2.fastq.gz
-
 
  
 
Example:
 
False
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    fastq_suffix
Type: string Default: ""
 
V-pipe expects paired-end reads to be in files that end in _R1 and _R2 exactly right before the file extension, e.g., _R1.fastq.gz, because this is how the workflow finds and recognizes them.
 
But Illumina’s bcl2fastq demultiplexer might introduce additional strings, e.g., _R2_001.fast.gz or, depending on its mismatches settings, e.g., _R2_001_MM_1.fast.gz. Use this options to specify anything which should go between the _R1 and _R2 endings and the file extension.
  
 
Examples:
 
_001
-
 
_001_MM_1
+
  
 
Example:
 
False
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    fastq_suffix
Type: string Default: ""
 
V-pipe expects paired-end reads to be in files that end in _R1 and _R2 exactly right before the file extension, e.g., _R1.fastq.gz, because this is how the workflow finds and recognizes them.
 
But Illumina’s bcl2fastq demultiplexer might introduce additional strings, e.g., _R2_001.fast.gz or, depending on its mismatches settings, e.g., _R2_001_MM_1.fast.gz. Use this options to specify anything which should go between the _R1 and _R2 endings and the file extension.
  
 
Examples:
 
_001
+
 
_001_MM_1
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    samples_file
Type: string Default: "config/samples.tsv"
 
File containing sample unique identifiers and dates as tab-separated values, e.g.,
 
patient1    20100113
 patient1    20110202
 patient2    20081130
-
 
Here, we have two samples from patient 1 and one sample from patient 2. By default, V-pipe searches for a file named samples.tsv, if this file does not exist, a list of samples is built by globbing datadir directory contents.
 
Optionally, the samples file can contain a third column specifying the read length. This is particularly useful when samples are sequenced using protocols with different read lengths.
 
Optionally, a fourth column can contain a short name of a protocol (e.g.: v3) that is detailed in the file specified in input => protocols_file. This is useful if protocol details such as primers change over time, e.g. to adapt to new variants with SNV breaking primer binding affinity.
 
Standardized Snakemake workflows place their tables inside the config/ subdirectory, but using this options you can specify alternate locations, e.g., the current working directory (as done in legacy V-pipe v1.x/2.x).
  
 
Example:
 
samples.tsv
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    protocols_file
Type: string Default: ""
 
When different samples have been processed with different library protocols, this file specifies a lookup table with per-protocol specific (primers bed and fasta), eg.:
 
 
v41:
-  name: SARS-CoV-2 ARTIC V4.1
-  inserts_bedfile: references/primers/v41/SARS-CoV-2.insert.bed
-  primers_bedfile: references/primers/v41/SARS-CoV-2.primer.bed
-  primers_file: references/primers/v41/SARS-CoV-2.tsv
-  primers_fasta: references/primers/v41/SARS-CoV-2.primer.fasta
-v4:
-  name: SARS-CoV-2 ARTIC V4
-  inserts_bedfile: references/primers/v4/SARS-CoV-2.insert.bed
-  primers_bedfile: references/primers/v4/SARS-CoV-2.primer.bed
-  primers_file: references/primers/v4/SARS-CoV-2.tsv
-  primers_fasta: references/primers/v4/ARTIC_v4.fasta
-v3:
-  name: SARS-CoV-2 ARTIC V3
-  inserts_bedfile: references/primers/v3/nCoV-2019.insert.bed
-  primers_bedfile: references/primers/v3/nCoV-2019.primer.bed
-  primers_file: references/primers/v3/nCoV-2019.tsv
-  primers_fasta: references/primers/v3/ARTIC_v3.fasta
-
 
 
The short name can then be referenced in the samples TSV table file:
 
sample_a    20211108    250 v3
+
 
Here, we have two samples from patient 1 and one sample from patient 2. By default, V-pipe searches for a file named samples.tsv, if this file does not exist, a list of samples is built by globbing datadir directory contents.
 
Optionally, the samples file can contain a third column specifying the read length. This is particularly useful when samples are sequenced using protocols with different read lengths.
 
Optionally, a fourth column can contain a short name of a protocol (e.g.: v3) that is detailed in the file specified in input => protocols_file. This is useful if protocol details such as primers change over time, e.g. to adapt to new variants with SNV breaking primer binding affinity.
 
Standardized Snakemake workflows place their tables inside the config/ subdirectory, but using this options you can specify alternate locations, e.g., the current working directory (as done in legacy V-pipe v1.x/2.x).
  
 
Example:
 
samples.tsv
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    protocols_file
Type: string Default: ""
 
When different samples have been processed with different library protocols, this file specifies a lookup table with per-protocol specific (primers bed and fasta), eg.:
 
v41:
+  name: SARS-CoV-2 ARTIC V4.1
+  inserts_bedfile: references/primers/v41/SARS-CoV-2.insert.bed
+  primers_bedfile: references/primers/v41/SARS-CoV-2.primer.bed
+  primers_file: references/primers/v41/SARS-CoV-2.tsv
+  primers_fasta: references/primers/v41/SARS-CoV-2.primer.fasta
+v4:
+  name: SARS-CoV-2 ARTIC V4
+  inserts_bedfile: references/primers/v4/SARS-CoV-2.insert.bed
+  primers_bedfile: references/primers/v4/SARS-CoV-2.primer.bed
+  primers_file: references/primers/v4/SARS-CoV-2.tsv
+  primers_fasta: references/primers/v4/ARTIC_v4.fasta
+v3:
+  name: SARS-CoV-2 ARTIC V3
+  inserts_bedfile: references/primers/v3/nCoV-2019.insert.bed
+  primers_bedfile: references/primers/v3/nCoV-2019.primer.bed
+  primers_file: references/primers/v3/nCoV-2019.tsv
+  primers_fasta: references/primers/v3/ARTIC_v3.fasta
+
 
The short name can then be referenced in the samples TSV table file:
 
sample_a    20211108    250 v3
 sample_b    20220214    250 v4
-
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
  
 
Example:
 
resources/sars-cov-2/primers.yaml
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    read_length
Type: integer Default: 250
 
Default for those samples whose read length isn’t specified explicitly in the optional third column of the samples.tsv table.
  
 
Example:
 
100
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    primers_bedfile
Type: string Default: ""
 
A bed file with primers position to trim the alignment output
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
 
Note: individual sample can override this using the 4th column in the samples TSV table file and the protocols YAML look-up table.
  
 
Example:
 
resources/sars-cov-2/primers/v3/nCoV-2019.primer.bed
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    inserts_bedfile
Type: string Default: ""
 
A bed file with inserts position of the multiplex PCR output to use with amplicon-based analysis.
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
 
Note: individual sample can override this using the 4th column in the samples TSV table file and the protocols YAML look-up table.
  
 
Example:
 
resources/sars-cov-2/primers/v3/nCoV-2019.primer.bed
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    trim_percent_cutoff
Type: number Default: 0.8
 
Using this parameter, the user can specify the read-length threshold that should be applied during the quality trimming as a percentage (0 < trim_percent_cutoff < 1).
  
Value must be greater or equal to 0 and lesser or equal to 1
 
 
Example:
 
0.9
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    reference
Type: string Default: "results/cohort_consensus.fasta"
 
Reference sequence to use for the alignment step
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default to a reference for that virus.
 You are still free to override that default in your configuration shall the need arise.
  
 
Examples:
 
resources/hiv/HXB2.fasta
-
 
resources/sars-cov-2/NC_045512.2.fasta
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    gff_directory
Type: string Default: ""
 
A directory containing gff files that can be optionally used to annotate the reference genome in the visualization, e.g., with genes, mature products, protein domains, regions of interests, etc.
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
  
 
Examples:
 
resources/hiv/gffs/
-
 
resources/sars-cov-2/gffs/
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    metainfo_file
Type: string Default: ""
 
An associative array providing user-friendly name to display for each annotation .gff file in the gff_directory
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
  
 
Examples:
 
resources/hiv/metainfo.yaml
-
 
resources/sars-cov-2/metainfo.yaml
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    genes_gff
Type: string Default: ""
 
The specific annotation GFF file that has provides the genes position along the genome for reports that mention specific genes such frameshift-deletions-check.
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
 
Note: if not set, V-pipe will try auto-selecting a .gff file from the gff_directory.
  
 
Examples:
 
resources/hiv/gffs/GCF_000864765.1_ViralProj15476_genomic.gff
-
 
resources/sars-cov-2/gffs/Genes_NC_045512.2.GFF3
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    primers_file
Type: string Default: ""
 
A table with primers to display on the visualization
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
 
Note: individual sample can override this using the 4th column in the samples TSV table file and the protocols YAML look-up table.
  
 
Example:
 
resources/sars-cov-2/primers/v3/nCoV-2019.tsv
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    variants_def_directory
Type: string Default: ""
 
Directory holding a list of COJAC YAML definitions of variants of concern that will be used for search of variant signatures
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
  
 
Example:
 
resources/sars-cov-2/voc/
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    phylogeny_data
Type: string Default: ""
 
A FASTQ file with sequences of interest
 
Note: These sequences are used, together with the consensus sequence, to build a phylogenetic tree.
  
 
Example:
 
resources/sars-cov-2/phylogeny/selected_covid_sequences.fasta
-
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output
Type: object Default: {}
 
Properties in this section of the configuration control the output of the pipeline.
  
 
 
 
  
 
 
 
 
root    output    datadir
Type: string Default: "results"
 
The workflow will write its output files into this directory. This will follow the same structure as for the input.
 
For each sample, V-pipe produces several output files that are located in the corresponding sample-specific directory. First, the alignment file and consensus sequences are located in the alignments and references subdirectories, respectively. Second, output files containing SNVs and viral haplotypes are located in the variants subdirectories.
 
Using the sample example as in the input section, the output files for the two patient samples will be located in the following subdirectories:
 
 
📁results
+
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
  
 
Example:
 
resources/sars-cov-2/primers.yaml
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    read_length
Type: integer Default: 250
 
Default for those samples whose read length isn’t specified explicitly in the optional third column of the samples.tsv table.
  
 
Example:
 
100
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    primers_bedfile
Type: string Default: ""
 
A bed file with primers position to trim the alignment output
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
 
Note: individual sample can override this using the 4th column in the samples TSV table file and the protocols YAML look-up table.
  
 
Example:
 
resources/sars-cov-2/primers/v3/nCoV-2019.primer.bed
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    inserts_bedfile
Type: string Default: ""
 
A bed file with inserts position of the multiplex PCR output to use with amplicon-based analysis.
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
 
Note: individual sample can override this using the 4th column in the samples TSV table file and the protocols YAML look-up table.
  
 
Example:
 
resources/sars-cov-2/primers/v3/nCoV-2019.primer.bed
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    trim_percent_cutoff
Type: number Default: 0.8
 
Using this parameter, the user can specify the read-length threshold that should be applied during the quality trimming as a percentage (0 < trim_percent_cutoff < 1).
  
Value must be greater or equal to 0 and lesser or equal to 1
 
 
Example:
 
0.9
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    reference
Type: string Default: "results/cohort_consensus.fasta"
 
Reference sequence to use for the alignment step
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default to a reference for that virus.
 You are still free to override that default in your configuration shall the need arise.
  
 
Examples:
 
resources/hiv/HXB2.fasta
+
 
resources/sars-cov-2/NC_045512.2.fasta
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    gff_directory
Type: string Default: ""
 
A directory containing gff files that can be optionally used to annotate the reference genome in the visualization, e.g., with genes, mature products, protein domains, regions of interests, etc.
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
  
 
Examples:
 
resources/hiv/gffs/
+
 
resources/sars-cov-2/gffs/
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    metainfo_file
Type: string Default: ""
 
An associative array providing user-friendly name to display for each annotation .gff file in the gff_directory
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
  
 
Examples:
 
resources/hiv/metainfo.yaml
+
 
resources/sars-cov-2/metainfo.yaml
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    genes_gff
Type: string Default: ""
 
The specific annotation GFF file that has provides the genes position along the genome for reports that mention specific genes such frameshift-deletions-check.
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
 
Note: if not set, V-pipe will try auto-selecting a .gff file from the gff_directory.
  
 
Examples:
 
resources/hiv/gffs/GCF_000864765.1_ViralProj15476_genomic.gff
+
 
resources/sars-cov-2/gffs/Genes_NC_045512.2.GFF3
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    primers_file
Type: string Default: ""
 
A table with primers to display on the visualization
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
 
Note: individual sample can override this using the 4th column in the samples TSV table file and the protocols YAML look-up table.
  
 
Example:
 
resources/sars-cov-2/primers/v3/nCoV-2019.tsv
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    variants_def_directory
Type: string Default: ""
 
Directory holding a list of COJAC YAML definitions of variants of concern that will be used for search of variant signatures
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
  
 
Example:
 
resources/sars-cov-2/voc/
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    input    phylogeny_data
Type: string Default: ""
 
A FASTQ file with sequences of interest
 
Note: These sequences are used, together with the consensus sequence, to build a phylogenetic tree.
  
 
Example:
 
resources/sars-cov-2/phylogeny/selected_covid_sequences.fasta
+
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output
Type: object Default: {}
 
Properties in this section of the configuration control the output of the pipeline.
  
 
 
 
  
 
 
 
 
root    output    datadir
Type: string Default: "results"
 
The workflow will write its output files into this directory. This will follow the same structure as for the input.
 
For each sample, V-pipe produces several output files that are located in the corresponding sample-specific directory. First, the alignment file and consensus sequences are located in the alignments and references subdirectories, respectively. Second, output files containing SNVs and viral haplotypes are located in the variants subdirectories.
 
Using the sample example as in the input section, the output files for the two patient samples will be located in the following subdirectories:
 
📁results
 ├──📁patient1
 │  ├──📁20100113
 │  │  ├──📁alignments
@@ -117,72 +117,75 @@
      |  └──snvs.vcf
      └──📁global
         └──contigs_stage_c.fasta
-
 
 
     
  • Standardized Snakemake workflows place their output in a results subdirectory
    •  
  • If your prefer the output written, e.g., to the same samples/ subdirectory as the input (as used to be done in legacy V-pipe v1.x/2.x), you can use this options you can specify alternate target locations.
    •  
  
 
Example:
 
samples
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    cohortdir
Type: string Default: ""
 
In addition, V-pipe can optionally generate a few cohort-wide results, such as a current cohort consensus fasta file, or a TSV file containing the frequencies of all minor alleles that differ from the consensus among analyzed samples.
 By default, these output files are located at the base of the output datadir, outside of the two-level per sample structure:
 
 
results
+
 
     
  • Standardized Snakemake workflows place their output in a results subdirectory
    •  
  • If your prefer the output written, e.g., to the same samples/ subdirectory as the input (as used to be done in legacy V-pipe v1.x/2.x), you can use this options you can specify alternate target locations.
    •  
  
 
Example:
 
samples
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    cohortdir
Type: string Default: ""
 
In addition, V-pipe can optionally generate a few cohort-wide results, such as a current cohort consensus fasta file, or a TSV file containing the frequencies of all minor alleles that differ from the consensus among analyzed samples.
 By default, these output files are located at the base of the output datadir, outside of the two-level per sample structure:
 
results
 ├──minority_variants.tsv
 ├──cohort_consensus.fasta
 ├──patient1
 │  ├──20100113
 │  │  ├──alignments
 …
-
 
 
If you prefer instead, e.g., such cohort-wide results behind written in a subdirectory of the working directory at the same level as the datadirs, you can use this options you can specify alternate subdirectory relative to the datadir property. (Use .. prefix if you want instead your cohort-wide results to be in a directory at the sample level as samples/ and results/. See the example below to recreate the variants/ directory used by legacy V-pipe v1.x/2.x).
  
 
Example:
 
../variants
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    QA
Type: boolean Default: false
 
V-pipe can produce several outputs to assess the quality of the output of its steps, e.g., checking whether a sample’s consensus sequence generated by bctfools does result in frameshifting indels and writing a report in sample’s …/references/frameshift_deletions_check.tsv. Such reports can be useful when submitting sequences to GISAID.
 
This option turns on such QA features.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    trim_primers
Type: boolean Default: false
 
This option indicates that the samples come from PCR amplification and the primers should be trimmed from amplicons in the alignment file. The trimmed read are written to each sample’s …/variants/SNVs/REF_aln_trim.bam.
 
Using this option requires either specifying a primers bed file in input => protocols_file, or using a 4 column input samples TSV file and specify a protocol look-up YAML file in input => protocols_file.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    snv
Type: boolean Default: false
 
This option selects whether the SNV caller step should be executed and its output written to each sample’s …/variants/SNVs/snvs.csv.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    local
Type: boolean Default: false
 
This option activates local haplotype reconstruction (only available when using ShoRAH).
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    global
Type: boolean Default: false
 
This option turns on global haplotype reconstruction.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    visualization
Type: boolean Default: false
 
This option selects whether to generate HTML visualization of the SNVs in each sample’s …/visualization/index.html.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    diversity
Type: boolean Default: false
 
This option turns on the computation of diversity measures in each sample.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    dehumanized_raw_reads
Type: boolean Default: false
 
This option turns on dehumanization of the raw reads (i.e. removal of host’s reads) and generates the file dehuman.cram. This is useful to prepare raw reads for upload on public databases such as, e.g. ENA (European Nucleotide Archive).
 
This only applies to the upload and does not affect the main workflow.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    upload
Type: boolean Default: false
 
This option can be used for assistance in incremental upload of data. See section upload for an example.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications
Type: object Default: {}
 
The path to the different software packages can be specified using this section.
 
It is especially useful when dependencies are not obtained via conda such as VICUNA, and when the software packages are not in the PATH.
 
Note we strongly recommend to use conda environments, by adding the --use-conda flag to the V-pipe execution command, e.g. ./vpipe --use-conda. If you prefer to use your own installations, this section allows you to specify the location of the executable files.
  
 
Example:
 
bwa: /path/to/bwa
-haploclique: /path/to/haploclique
-
 
 
 
 
 
  
 
 
 
 
root    applications    gunzip
Type: string Default: "gunzip"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    zstd
Type: string Default: "zstd"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    xsv
Type: string Default: "xsv"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    prinseq
Type: string Default: "prinseq-lite.pl"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    fastqc
Type: string Default: "fastqc"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    vicuna
Type: string Default: "vicuna"
 
Due to a special license, VICUNA is not available from bioconda and must be installed from its original website.
 Use this option to specify where you have installed its executable.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    indelfixer
Type: string Default: "InDelFixer"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    consensusfixer
Type: string Default: "ConsensusFixer"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    picard
Type: string Default: "picard"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    bwa
Type: string Default: "bwa"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    bowtie_idx
Type: string Default: "bowtie2-build"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    bowtie
Type: string Default: "bowtie2"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    minimap
Type: string Default: "minimap2"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    samtools
Type: string Default: "samtools"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    extract_consensus
Type: string Default: "extract_consensus"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    matcher
Type: string Default: "matcher"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    frameshift_deletions_checks
Type: string Default: "frameshift_deletions_checks"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    mafft
Type: string Default: "mafft"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    ngshmmalign
Type: string Default: "ngshmmalign"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    convert_reference
Type: string Default: "convert_reference"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    extract_seq
Type: string Default: "extract_seq"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    coverage_stats
Type: string Default: "coverage_stats"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    remove_gaps_msa
Type: string Default: "remove_gaps_msa"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    ivar
Type: string Default: "ivar"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    aln2basecnt
Type: string Default: "aln2basecnt"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    gather_coverage
Type: string Default: "gather_coverage"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    minority_freq
Type: string Default: "minority_freq"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    extract_coverage_intervals
Type: string Default: "extract_coverage_intervals"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    shorah
Type: string Default: "shorah shotgun"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    lofreq
Type: string Default: "lofreq"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    bcftools
Type: string Default: "bcftools"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    haploclique
Type: string Default: "haploclique"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    compute_mds
Type: string Default: "compute_mds"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    savage
Type: string Default: "savage"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    predicthaplo
Type: string Default: "predicthaplo"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    cojac
Type: string Default: "cojac"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    lollipop
Type: string Default: "lollipop"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    gunzip
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    gunzip    mem
Type: integer Default: 32
 
 
 
 
 
 
 
 
  
 
 
 
 
root    gunzip    time
Type: integer Default: 60
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    extract
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    extract    mem
Type: integer Default: 4096
 
 
 
 
 
 
 
 
  
 
 
 
 
root    extract    time
Type: integer Default: 20
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    preprocessing
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    preprocessing    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    preprocessing    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    preprocessing    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/preprocessing.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    preprocessing    extra
Type: string Default: "-ns_max_n 4 -min_qual_mean 30 -trim_qual_left 30 -trim_qual_right 30 -trim_qual_window 10"
 
We use software PRINSEQ 1 for quality control. By default, we use options -ns_max_n 4 -min_qual_mean 30 -trim_qual_left 30 -trim_qual_right 30 -trim_qual_window 10, which indicates to trim reads using a sliding window with size 10 bp, and trim bases if their quality scores are less than 30. Additionally, reads are filtered out if the average quality score is below 30 and if they contain more than 4 N’s. The user can choose to overwrite the default settings or use additional parameters by using the property extra. E.g., if many reads are filtered out in this step, the user can choose to lower the quality threshold as indicated in the example.
 Please do not modify PRINSEQ options -out_format, -out_good, nor -min_len. Instead of using -min_len to define threshold on the read length after trimming, use input => trim_percent_cutoff.
 
 
 
     
  1.  
    Schmieder, R. and Edwards, R. Quality control and preprocessing of metagenomic datasets. Bioinformatics. 2011. ↩
     
    2.  
 
  
 
Example:
 
-ns_max_n 4 -min_qual_mean 20 -trim_qual_left 20 -trim_qual_right 20 -trim_qual_window 10
-
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    fastqc
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    fastqc    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    fastqc    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    fastqc    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/fastqc.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    fastqc    threads
Type: integer Default: 6
 
 
 
 
 
 
 
 
  
 
 
 
 
root    fastqc    no_group
Type: boolean Default: false
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna
Type: object Default: {}
 
NOTE The conda environment for this rule doesn’t work properly. The package on the bioconda channel, mvicuna, is slightly different from VICUNA and it has different command-line arguments. Moreover, VICUNA and mvicuna are no longer maintained. In the future, this rule will be deprecated.
  
 
 
 
  
 
 
 
 
root    initial_vicuna    mem
Type: integer Default: 1000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna    time
Type: integer Default: 600
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/initial_vicuna.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna_msa
Type: object Default: {}
 
NOTE Obtaining a initial reference de novo is implemented for more than one sample.
  
 
 
 
  
 
 
 
 
root    initial_vicuna_msa    mem
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna_msa    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna_msa    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna_msa    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/initial_vicuna_msa.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    create_vicuna_initial
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    create_vicuna_initial    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    hmm_align
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    hmm_align    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    hmm_align    time
Type: integer Default: 1435
 
 
 
 
 
 
 
 
  
 
 
 
 
root    hmm_align    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    hmm_align    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/hmm_align.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    hmm_align    leave_msa_temp
Type: boolean Default: false
 
This option is useful for debugging purposes.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    hmm_align    extra
Type: string Default: ""
 
Pass additional options to run ngshmmalign
 
V-pipe uses option -R <path/to/initial_reference>, thus option -r arg is not allowed. Also, instead of passing -l via the property extra, set leave_msa_temp to True. Lastly, please do not modify options -o arg, -w arg, -t arg, and -N arg. These are already managed by V-pipe.
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    sam2bam
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    sam2bam    mem
Type: integer Default: 5000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    sam2bam    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
 
 
 
root    sam2bam    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/sam2bam.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_QA
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    bwa_QA    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_QA    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_QA    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_QA    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/bwa_QA.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_QA    ref_panel
Type: string Default: ""
 
Panel of diverse references against which to align reads as a QA step
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
  
 
Example:
 
resources/hiv/5-Virus-Mix.fasta
-
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_QA
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    coverage_QA    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_QA    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_QA    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_QA    target
Type: string Default: "HXB2:6614-6812,7109-7217,7376-7478,7601-7634"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    msa
Type: object Default: {}
 
This rule takes all previously aligned reads by hmm_align. Therefore, resources should be allocated accordingly.
  
 
 
 
  
 
 
 
 
root    msa    mem
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    msa    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    msa    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    msa    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/msa.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    convert_to_ref
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    convert_to_ref    mem
Type: integer Default: 8000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    convert_to_ref    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    convert_to_ref    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_bwa_index
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    ref_bwa_index    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_bwa_index    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_bwa_index    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/bwa_align.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_align
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    bwa_align    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_align    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_align    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_align    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/bwa_align.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_align    extra
Type: string Default: ""
 
With property extra, users can pass additional options to run BWA MEM. For more details on BWA MEM configurable options refer to the software documentation.
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_bowtie_index
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    ref_bowtie_index    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_bowtie_index    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_bowtie_index    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/bowtie_align.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    bowtie_align    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/bowtie_align.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    phred
Type: enum (of string) Default: "--phred33"
 
Indicate if qualities are Phred+33 (default) or Phred+64 (--phred64).
 
 
Must be one of:
 
  • "--phred33"
  • "--phred64"
 
 
 
Example:
 
--phred64
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    preset
Type: string Default: "--local --sensitive-local"
 
Specify Bowtie 2 presets.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    maxins
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    extra
Type: string Default: ""
 
Pass additional options to run Bowtie 2. V-pipe handles the input and output files, as well as the reference sequence. Thus, do not modify these options
 For more details on Bowtie 2 configurable options refer to the software documentation.
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_minimap_index
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    ref_minimap_index    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_minimap_index    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_minimap_index    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/minimap_align.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    minimap_align    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align    preset
Type: enum (of string) Default: "sr"
 
Specify minimap2 preset options. See minimape’s documentation for details about each of the presets.
 
 
Must be one of:
 
  • "map-ont"
  • "map-hifi"
  • "map-pb"
  • "asm5"
  • "asm10"
  • "asm20"
  • "splice"
  • "splice:hq"
  • "sr"
  • "ava-pb"
  • "ava-ont"
 
 
 
Example:
 
map-ont
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align    secondary
Type: boolean Default: false
 
By default V-pipe ignores Minimap2’s secondary alignment(s) and only considers the primary one (A secondary alignment occurs when a given read could align reasonably well to more than one place). This flags turns back on Minimap2’s secondary alignments, and also includes their sequences in the output BAM file.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/minimap_align.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align    extra
Type: string Default: ""
 
With property extra, users can pass additional options to run minimap2. For more details on minimap2 configurable options refer to the software documentation.
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    primerstrim
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    primerstrim    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    primerstrim    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    primerstrim    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/primerstrim.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    consensus_sequences    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences    min_coverage
Type: integer Default: 50
 
Minimum read depth for reporting variants per locus.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences    n_coverage
Type: integer Default: 5
 
Read count below which ambiguous base ’n’ is reported.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences    qual_thrd
Type: integer Default: 15
 
Minimum phred quality score for a base to be included.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences    min_freq
Type: number Default: 0.05
 
Minimum frequency for an ambiguous nucleotide.
  
Value must be greater or equal to 0 and lesser or equal to 1
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/bcftools.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    max_coverage
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    mask_coverage_threshold
Type: integer Default: 10
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    ambiguous_base_coverage_threshold
Type: number Default: 0.05
 
Value must be greater or equal to 0 and lesser or equal to 1
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consseq_QA
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    consseq_QA    mem
Type: integer Default: 4096
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consseq_QA    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consseq_QA    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consseq_QA    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/consseq_qa.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    frameshift_deletions_checks
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    frameshift_deletions_checks    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    frameshift_deletions_checks    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
  
 
 
 
 
 
  
 
 
 
 
root    frameshift_deletions_checks    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    basecounts
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    basecounts    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    basecounts    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
 
 
 
root    basecounts    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    basecounts    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    coverage    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minor_variants
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    minor_variants    mem
Type: integer Default: 1000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minor_variants    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minor_variants    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minor_variants    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minor_variants    min_coverage
Type: integer Default: 100
 
Minimum read depth for reporting variants per locus.
  
 
Example:
 
50
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minor_variants    frequencies
Type: boolean Default: false
 
Output a numpy array file containing frequencies of all bases, including gaps and also the most abundant base across samples.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    coverage_intervals    mem
Type: integer Default: 1000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals    overlap
Type: boolean Default: false
 
Construct intervals based on overlapping windows of the read alignment. By default, regions with high coverage are built based on the position-wise read depth.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals    coverage
Type: integer Default: 50
 
Minimum read depth. A region spanning the reference genome is returned if coverage is set to 0.
  
 
Example:
 
0
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals    liberal
Type: boolean Default: true
 
Indicate whether to apply a more liberal shifting on intervals’ right-endpoint.
  
 
Example:
 
False
-
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    snv    mem
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    time
Type: integer Default: 2880
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/snv.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    consensus
Type: boolean Default: true
 
Indicate whether to use the cohort-consensus sequence from the analyzed samples (output from minor_variants rule located in the cohort-wide output results/cohort_onsensus.fasta) or the reference sequence by setting this option to False.
  
 
Example:
 
False
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    alpha
Type: number Default: 0.1
 
Hyperparameter used for instantiating a new cluster.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    ignore_indels
Type: boolean Default: false
 
Ignore SNVs adjacent to indels.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    posterior_threshold
Type: number Default: 0.9
 
Value must be greater or equal to 0 and lesser or equal to 1
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    coverage
Type: integer Default: 0
 
Omit windows with coverage less than this value.
  
 
Example:
 
50
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    shift
Type: integer Default: 3
 
ShoRAH performs local haplotype reconstruction on windows of the read alignment. The overlap between these windows is defined by the window shifts. By default, it is set to 3, i.e., apart from flanking regions each position is covered by 3 windows.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    keep_files
Type: boolean Default: false
 
Indicate whether to move files produced in previous/interrupted runs to subdirectory named old
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    localscratch
Type: string Default: ""
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    samtools_index
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    samtools_index    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    samtools_index    time
Type: integer Default: 20
 
 
 
 
 
 
 
 
  
 
 
 
 
root    samtools_index    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/lofreq.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    lofreq
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    lofreq    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    lofreq    time
Type: integer Default: 60
 
 
 
 
 
 
 
 
  
 
 
 
 
root    lofreq    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/lofreq.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    lofreq    consensus
Type: boolean Default: true
 
Indicate whether to use the cohort-consensus sequence from the analyzed samples (output from minor_variants rule located in the cohort-wide output results/cohort_onsensus.fasta) or the reference sequence by setting this option to False.
  
 
Example:
 
False
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    lofreq    extra
Type: string Default: ""
 
Pass additional options to run lofreq call
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    alignment_coverage
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    alignment_coverage    mem
Type: integer Default: 1000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    alignment_coverage    time
Type: integer Default: 60
 
 
 
 
 
 
 
 
  
 
 
 
 
root    alignment_coverage    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    alignment_coverage    coverage
Type: integer Default: 5
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    stats
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    stats    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/sam2bam.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    savage
Type: object Default: {}
 
NOTE This rule only works in Linux.
  
 
 
 
  
 
 
 
 
root    savage    mem
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    savage    time
Type: integer Default: 1435
 
 
 
 
 
 
 
 
  
 
 
 
 
root    savage    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    savage    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/savage.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    savage    split
Type: integer Default: 20
 
Size of the batches of reads to be processed by SAVAGE. It is recommended that 500 < coverage/split < 1000.
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    haploclique    mem
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    time
Type: integer Default: 1435
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/haploclique.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    relax
Type: boolean Default: true
 
If set to True (default) a predefined set of parameter values is used for drawing edges between reads in the read graph.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    no_singletons
Type: boolean Default: true
 
Singletons are defined as proposed haplotypes which are supported by a single read. If this property is set to True, singletons are discarded.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    no_prob0
Type: boolean Default: true
 
If set to True (default) probability of the overhangs is ignored.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    clique_size_limit
Type: integer Default: 3
 
Sets a threshold to limit the size of cliques.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    max_num_cliques
Type: integer Default: 10000
 
Indicates the maximum number of clique to be considered in the next iteration.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    extra_parameters
Type: string Default: ""
 
Additional parameters to be passed to haploclique.
 
Warning: this won’t overwrite the other options (e.g. clique_size_limi and max_num_cliques should still be set via their own respective properties, do not pass parameters --limit_clique_size= nor --max_cliques= via this extra property).
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique_visualization
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    haploclique_visualization    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique_visualization    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique_visualization    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique_visualization    region_start
Type: integer Default: 0
 
Use to specify a region of interest.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique_visualization    region_end
Type: integer Default: -1
 
Use to specify a region of interest.
  
 
Examples:
 
9719
-
 
29836
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique_visualization    msa
Type: string Default: ""
 
When the ground truth is available (e.g., simulation studies), a multiple sequence alignment of types making up the population can be provided, and additional checks are performed.
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    predicthaplo
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    predicthaplo    mem
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    predicthaplo    time
Type: integer Default: 1435
 
 
 
 
 
 
 
 
  
 
 
 
 
root    predicthaplo    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    predicthaplo    read_min_length
Type: integer Default: 0
 
 
 
 
 
 
 
 
  
 
 
 
 
root    predicthaplo    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/predicthaplo.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    amplicons
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    amplicons    mem
Type: integer Default: 256
 
 
 
 
 
 
 
 
  
 
 
 
 
root    amplicons    time
Type: integer Default: 10
 
 
 
 
 
 
 
 
  
 
 
 
 
root    amplicons    mincooc
Type: integer Default: 2
 
Minimal number of coocurrences to search for in amplicon. Lowering this property to 1 will make COJAC also look for amplicon with singleton mutations.
  
 
Example:
 
1
-
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    cooc
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    cooc    mem
Type: integer Default: 8192
 
 
 
 
 
 
 
 
  
 
 
 
 
root    cooc    time
Type: integer Default: 45
 
 
 
 
 
 
 
 
  
 
 
 
 
root    cooc    threads
Type: integer Default: 1
 
 
 
 
 
 
 
 
  
 
 
 
 
root    cooc    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/cojac.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    cooc    out_format
Type: enum (of string) Default: "lines"
 
Format of the output CSV.
 
     
  • lines(default) - each amplicon a separate entry on a separate line.
    •  
  • columns - one column per amplicon
    •  
 
 
Must be one of:
 
  • "lines"
  • "columns"
 
 
 
Example:
 
columns
-
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    mutlist
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    mutlist    mem
Type: integer Default: 256
 
 
 
 
 
 
 
 
  
 
 
 
 
root    mutlist    time
Type: integer Default: 10
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    sigmut
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    sigmut    mem
Type: integer Default: 4096
 
 
 
 
 
 
 
 
  
 
 
 
 
root    sigmut    time
Type: integer Default: 15
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline
Type: object Default: {}
 
This section is used to set up a timeline of the samples. Some output, e.g., deconvolution of quasispecies mixture using LolliPop, need to have a time component. By default it calls a script that uses regular expressions and look-up tables to extract this information from the samples’ own names. But by using the properties script and options and adapting the environment provided in property conda, it is possible to heavily customize the actions (e.g. it is possible to query an external database instead). For inspiration, see the default script file_parser.py.
  
 
 
 
  
 
 
 
 
root    timeline    mem
Type: integer Default: 1024
 
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline    time
Type: integer Default: 15
 
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/timeline.yaml"
 
The default environment only provides regular expression functions (python-reges) but depending on your needs you would want to provide a custom environment with additional tools (e.g. drivers to query a databse, etc.)
  
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline    threads
Type: integer Default: 1
 
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline    local
Type: boolean Default: true
 
Don’t dispatch the timeline rule to the cluster for execution, run locally.
  
 
Example:
 
False
+
 
If you prefer instead, e.g., such cohort-wide results behind written in a subdirectory of the working directory at the same level as the datadirs, you can use this options you can specify alternate subdirectory relative to the datadir property. (Use .. prefix if you want instead your cohort-wide results to be in a directory at the sample level as samples/ and results/. See the example below to recreate the variants/ directory used by legacy V-pipe v1.x/2.x).
  
 
Example:
 
../variants
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    QA
Type: boolean Default: false
 
V-pipe can produce several outputs to assess the quality of the output of its steps, e.g., checking whether a sample’s consensus sequence generated by bctfools does result in frameshifting indels and writing a report in sample’s …/references/frameshift_deletions_check.tsv. Such reports can be useful when submitting sequences to GISAID.
 
This option turns on such QA features.
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    trim_primers
Type: boolean Default: false
 
This option indicates that the samples come from PCR amplification and the primers should be trimmed from amplicons in the alignment file. The trimmed read are written to each sample’s …/variants/SNVs/REF_aln_trim.bam.
 
Using this option requires either specifying a primers bed file in input => protocols_file, or using a 4 column input samples TSV file and specify a protocol look-up YAML file in input => protocols_file.
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    snv
Type: boolean Default: false
 
This option selects whether the SNV caller step should be executed and its output written to each sample’s …/variants/SNVs/snvs.csv.
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    local
Type: boolean Default: false
 
This option activates local haplotype reconstruction (only available when using ShoRAH or VILOCA).
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    global
Type: boolean Default: false
 
This option turns on global haplotype reconstruction.
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    visualization
Type: boolean Default: false
 
This option selects whether to generate HTML visualization of the SNVs in each sample’s …/visualization/index.html.
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    diversity
Type: boolean Default: false
 
This option turns on the computation of diversity measures in each sample.
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    dehumanized_raw_reads
Type: boolean Default: false
 
This option turns on dehumanization of the raw reads (i.e. removal of host’s reads) and generates the file dehuman.cram. This is useful to prepare raw reads for upload on public databases such as, e.g. ENA (European Nucleotide Archive).
 
This only applies to the upload and does not affect the main workflow.
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    output    upload
Type: boolean Default: false
 
This option can be used for assistance in incremental upload of data. See section upload for an example.
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications
Type: object Default: {}
 
The path to the different software packages can be specified using this section.
 
It is especially useful when dependencies are not obtained via conda such as VICUNA, and when the software packages are not in the PATH.
 
Note we strongly recommend to use conda environments, by adding the --use-conda flag to the V-pipe execution command, e.g. ./vpipe --use-conda. If you prefer to use your own installations, this section allows you to specify the location of the executable files.
  
 
Example:
 
bwa: /path/to/bwa
+haploclique: /path/to/haploclique
+
 
 
 
 
 
  
 
 
 
 
root    applications    gunzip
Type: string Default: "gunzip"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    zstd
Type: string Default: "zstd"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    xsv
Type: string Default: "xsv"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    prinseq
Type: string Default: "prinseq-lite.pl"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    fastqc
Type: string Default: "fastqc"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    vicuna
Type: string Default: "vicuna"
 
Due to a special license, VICUNA is not available from bioconda and must be installed from its original website.
 Use this option to specify where you have installed its executable.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    indelfixer
Type: string Default: "InDelFixer"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    consensusfixer
Type: string Default: "ConsensusFixer"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    picard
Type: string Default: "picard"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    bwa
Type: string Default: "bwa"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    bowtie_idx
Type: string Default: "bowtie2-build"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    bowtie
Type: string Default: "bowtie2"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    minimap
Type: string Default: "minimap2"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    samtools
Type: string Default: "samtools"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    extract_consensus
Type: string Default: "extract_consensus"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    matcher
Type: string Default: "matcher"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    frameshift_deletions_checks
Type: string Default: "frameshift_deletions_checks"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    paired_end_read_merger
Type: string Default: "paired_end_read_merger"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    mafft
Type: string Default: "mafft"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    ngshmmalign
Type: string Default: "ngshmmalign"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    convert_reference
Type: string Default: "convert_reference"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    extract_seq
Type: string Default: "extract_seq"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    coverage_stats
Type: string Default: "coverage_stats"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    remove_gaps_msa
Type: string Default: "remove_gaps_msa"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    ivar
Type: string Default: "ivar"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    aln2basecnt
Type: string Default: "aln2basecnt"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    gather_coverage
Type: string Default: "gather_coverage"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    minority_freq
Type: string Default: "minority_freq"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    extract_coverage_intervals
Type: string Default: "extract_coverage_intervals"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    shorah
Type: string Default: "shorah shotgun"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    viloca
Type: string Default: "viloca run"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    lofreq
Type: string Default: "lofreq"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    bcftools
Type: string Default: "bcftools"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    haploclique
Type: string Default: "haploclique"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    compute_mds
Type: string Default: "compute_mds"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    savage
Type: string Default: "savage"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    predicthaplo
Type: string Default: "predicthaplo"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    cojac
Type: string Default: "cojac"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    applications    lollipop
Type: string Default: "lollipop"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    gunzip
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    gunzip    mem
Type: integer Default: 32
 
 
 
 
 
 
 
 
  
 
 
 
 
root    gunzip    time
Type: integer Default: 60
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    extract
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    extract    mem
Type: integer Default: 4096
 
 
 
 
 
 
 
 
  
 
 
 
 
root    extract    time
Type: integer Default: 20
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    preprocessing
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    preprocessing    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    preprocessing    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    preprocessing    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/preprocessing.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    preprocessing    extra
Type: string Default: "-ns_max_n 4 -min_qual_mean 30 -trim_qual_left 30 -trim_qual_right 30 -trim_qual_window 10"
 
We use software PRINSEQ 1 for quality control. By default, we use options -ns_max_n 4 -min_qual_mean 30 -trim_qual_left 30 -trim_qual_right 30 -trim_qual_window 10, which indicates to trim reads using a sliding window with size 10 bp, and trim bases if their quality scores are less than 30. Additionally, reads are filtered out if the average quality score is below 30 and if they contain more than 4 N’s. The user can choose to overwrite the default settings or use additional parameters by using the property extra. E.g., if many reads are filtered out in this step, the user can choose to lower the quality threshold as indicated in the example.
 Please do not modify PRINSEQ options -out_format, -out_good, nor -min_len. Instead of using -min_len to define threshold on the read length after trimming, use input => trim_percent_cutoff.
 
 
 
     
  1.  
    Schmieder, R. and Edwards, R. Quality control and preprocessing of metagenomic datasets. Bioinformatics. 2011. ↩
     
    2.  
 
  
 
Example:
 
-ns_max_n 4 -min_qual_mean 20 -trim_qual_left 20 -trim_qual_right 20 -trim_qual_window 10
+
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    fastqc
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    fastqc    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    fastqc    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    fastqc    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/fastqc.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    fastqc    threads
Type: integer Default: 6
 
 
 
 
 
 
 
 
  
 
 
 
 
root    fastqc    no_group
Type: boolean Default: false
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna
Type: object Default: {}
 
NOTE The conda environment for this rule doesn’t work properly. The package on the bioconda channel, mvicuna, is slightly different from VICUNA and it has different command-line arguments. Moreover, VICUNA and mvicuna are no longer maintained. In the future, this rule will be deprecated.
  
 
 
 
  
 
 
 
 
root    initial_vicuna    mem
Type: integer Default: 1000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna    time
Type: integer Default: 600
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/initial_vicuna.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna_msa
Type: object Default: {}
 
NOTE Obtaining a initial reference de novo is implemented for more than one sample.
  
 
 
 
  
 
 
 
 
root    initial_vicuna_msa    mem
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna_msa    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna_msa    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    initial_vicuna_msa    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/initial_vicuna_msa.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    create_vicuna_initial
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    create_vicuna_initial    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    hmm_align
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    hmm_align    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    hmm_align    time
Type: integer Default: 1435
 
 
 
 
 
 
 
 
  
 
 
 
 
root    hmm_align    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    hmm_align    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/hmm_align.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    hmm_align    leave_msa_temp
Type: boolean Default: false
 
This option is useful for debugging purposes.
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    hmm_align    extra
Type: string Default: ""
 
Pass additional options to run ngshmmalign
 
V-pipe uses option -R <path/to/initial_reference>, thus option -r arg is not allowed. Also, instead of passing -l via the property extra, set leave_msa_temp to True. Lastly, please do not modify options -o arg, -w arg, -t arg, and -N arg. These are already managed by V-pipe.
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    sam2bam
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    sam2bam    mem
Type: integer Default: 5000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    sam2bam    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
 
 
 
root    sam2bam    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/sam2bam.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_QA
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    bwa_QA    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_QA    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_QA    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_QA    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/bwa_QA.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_QA    ref_panel
Type: string Default: ""
 
Panel of diverse references against which to align reads as a QA step
 
Note: The virus-specific base configuration specified in general => virus_base_config will most likely change this option’s default.
 You are still free to override that default in your configuration shall the need arise.
  
 
Example:
 
resources/hiv/5-Virus-Mix.fasta
+
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_QA
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    coverage_QA    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_QA    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_QA    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_QA    target
Type: string Default: "HXB2:6614-6812,7109-7217,7376-7478,7601-7634"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    msa
Type: object Default: {}
 
This rule takes all previously aligned reads by hmm_align. Therefore, resources should be allocated accordingly.
  
 
 
 
  
 
 
 
 
root    msa    mem
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    msa    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    msa    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    msa    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/msa.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    convert_to_ref
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    convert_to_ref    mem
Type: integer Default: 8000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    convert_to_ref    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    convert_to_ref    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_bwa_index
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    ref_bwa_index    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_bwa_index    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_bwa_index    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/bwa_align.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_align
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    bwa_align    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_align    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_align    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_align    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/bwa_align.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bwa_align    extra
Type: string Default: ""
 
With property extra, users can pass additional options to run BWA MEM. For more details on BWA MEM configurable options refer to the software documentation.
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_bowtie_index
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    ref_bowtie_index    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_bowtie_index    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_bowtie_index    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/bowtie_align.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    bowtie_align    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/bowtie_align.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    phred
Type: enum (of string) Default: "--phred33"
 
Indicate if qualities are Phred+33 (default) or Phred+64 (--phred64).
 
 
Must be one of:
 
  • "--phred33"
  • "--phred64"
 
 
 
Example:
 
--phred64
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    preset
Type: string Default: "--local --sensitive-local"
 
Specify Bowtie 2 presets.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    maxins
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    bowtie_align    extra
Type: string Default: ""
 
Pass additional options to run Bowtie 2. V-pipe handles the input and output files, as well as the reference sequence. Thus, do not modify these options
 For more details on Bowtie 2 configurable options refer to the software documentation.
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_minimap_index
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    ref_minimap_index    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_minimap_index    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    ref_minimap_index    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/minimap_align.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    minimap_align    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align    preset
Type: enum (of string) Default: "sr"
 
Specify minimap2 preset options. See minimape’s documentation for details about each of the presets.
 
 
Must be one of:
 
  • "map-ont"
  • "map-hifi"
  • "map-pb"
  • "asm5"
  • "asm10"
  • "asm20"
  • "splice"
  • "splice:hq"
  • "sr"
  • "ava-pb"
  • "ava-ont"
 
 
 
Example:
 
map-ont
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align    secondary
Type: boolean Default: false
 
By default V-pipe ignores Minimap2’s secondary alignment(s) and only considers the primary one (A secondary alignment occurs when a given read could align reasonably well to more than one place). This flags turns back on Minimap2’s secondary alignments, and also includes their sequences in the output BAM file.
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/minimap_align.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minimap_align    extra
Type: string Default: ""
 
With property extra, users can pass additional options to run minimap2. For more details on minimap2 configurable options refer to the software documentation.
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    primerstrim
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    primerstrim    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    primerstrim    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    primerstrim    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/primerstrim.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    consensus_sequences    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences    min_coverage
Type: integer Default: 50
 
Minimum read depth for reporting variants per locus.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences    n_coverage
Type: integer Default: 5
 
Read count below which ambiguous base ’n’ is reported.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences    qual_thrd
Type: integer Default: 15
 
Minimum phred quality score for a base to be included.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_sequences    min_freq
Type: number Default: 0.05
 
Minimum frequency for an ambiguous nucleotide.
  
Value must be greater or equal to 0 and lesser or equal to 1
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/bcftools.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    max_coverage
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    mask_coverage_threshold
Type: integer Default: 10
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consensus_bcftools    ambiguous_base_coverage_threshold
Type: number Default: 0.05
 
Value must be greater or equal to 0 and lesser or equal to 1
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consseq_QA
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    consseq_QA    mem
Type: integer Default: 4096
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consseq_QA    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consseq_QA    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    consseq_QA    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/consseq_qa.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    frameshift_deletions_checks
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    frameshift_deletions_checks    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    frameshift_deletions_checks    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
  
 
 
 
 
 
  
 
 
 
 
root    frameshift_deletions_checks    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    paired_end_read_merger
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    paired_end_read_merger    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    paired_end_read_merger    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
 
 
 
root    paired_end_read_merger    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    paired_end_read_merger    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/paired_end_read_merger.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    basecounts
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    basecounts    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    basecounts    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
 
 
 
root    basecounts    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    basecounts    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    coverage    mem
Type: integer Default: 1250
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minor_variants
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    minor_variants    mem
Type: integer Default: 1000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minor_variants    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minor_variants    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minor_variants    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minor_variants    min_coverage
Type: integer Default: 100
 
Minimum read depth for reporting variants per locus.
  
 
Example:
 
50
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    minor_variants    frequencies
Type: boolean Default: false
 
Output a numpy array file containing frequencies of all bases, including gaps and also the most abundant base across samples.
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    coverage_intervals    mem
Type: integer Default: 1000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals    overlap
Type: boolean Default: false
 
Construct intervals based on overlapping windows of the read alignment. By default, regions with high coverage are built based on the position-wise read depth.
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals    coverage
Type: integer Default: 50
 
Minimum read depth. A region spanning the reference genome is returned if coverage is set to 0.
  
 
Example:
 
0
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    coverage_intervals    liberal
Type: boolean Default: true
 
Indicate whether to apply a more liberal shifting on intervals’ right-endpoint.
  
 
Example:
 
False
+
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    snv    mem
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    time
Type: integer Default: 2880
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/snv.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    consensus
Type: boolean Default: true
 
Indicate whether to use the cohort-consensus sequence from the analyzed samples (output from minor_variants rule located in the cohort-wide output results/cohort_onsensus.fasta) or the reference sequence by setting this option to False.
  
 
Example:
 
False
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    alpha
Type: number Default: 0.1
 
Hyperparameter used for instantiating a new cluster.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    ignore_indels
Type: boolean Default: false
 
Ignore SNVs adjacent to indels.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    posterior_threshold
Type: number Default: 0.9
 
Value must be greater or equal to 0 and lesser or equal to 1
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    coverage
Type: integer Default: 0
 
Omit windows with coverage less than this value.
  
 
Example:
 
50
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    shift
Type: integer Default: 3
 
ShoRAH performs local haplotype reconstruction on windows of the read alignment. The overlap between these windows is defined by the window shifts. By default, it is set to 3, i.e., apart from flanking regions each position is covered by 3 windows.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    keep_files
Type: boolean Default: false
 
Indicate whether to move files produced in previous/interrupted runs to subdirectory named old
  
 
Example:
 
True
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    snv    localscratch
Type: string Default: ""
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    samtools_index
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    samtools_index    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    samtools_index    time
Type: integer Default: 20
 
 
 
 
 
 
 
 
  
 
 
 
 
root    samtools_index    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/lofreq.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    viloca
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    viloca    mem
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    viloca    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    viloca    time
Type: integer Default: 2880
 
 
 
 
 
 
 
 
  
 
 
 
 
root    viloca    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/viloca.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    viloca    consensus
Type: boolean Default: false
 
Indicate whether to use the cohort-consensus sequence from the analyzed samples (output from minor_variants rule located in the cohort-wide output results/cohort_onsensus.fasta) or the reference sequence by setting this option to False.
  
 
Example:
 
False
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    viloca    merge_paired_end_reads
Type: boolean Default: false
 
Merge paired-end reads in the preprocessing. This is a preprocessing snakemake rule.
  
 
Example:
 
False
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    viloca    shift
Type: integer Default: 3
 
VILOCA performs local haplotype reconstruction on windows of the read alignment. The overlap between these windows is defined by the window shifts. By default, it is set to 3, i.e., apart from flanking regions each position is covered by 3 windows.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    viloca    insert_bedfile
Type: string Default: "None"
 
VILOCA performs local haplotype reconstruction on windows of the read alignment. In a first step the alignment is tiled into local regions. This can be done uniformly then set this value None, otherwise path to an (optional) insert file (primer tiling strategy)
  
 
 
 
 
 
 
 
  
 
 
 
 
root    viloca    mode
Type: enum (of string) Default: "use_quality_scores"
 
Mode in which to run VILOCA: shorah, learnerrorparams, usequalityscores. If quality scores are available, we recommend this option
 
 
Must be one of:
 
  • "shorah"
  • "learn_error_params"
  • "use_quality_scores"
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    viloca    extra
Type: string Default: ""
 
Pass additional options to run viloca
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    lofreq
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    lofreq    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    lofreq    time
Type: integer Default: 60
 
 
 
 
 
 
 
 
  
 
 
 
 
root    lofreq    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/lofreq.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    lofreq    consensus
Type: boolean Default: true
 
Indicate whether to use the cohort-consensus sequence from the analyzed samples (output from minor_variants rule located in the cohort-wide output results/cohort_onsensus.fasta) or the reference sequence by setting this option to False.
  
 
Example:
 
False
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    lofreq    extra
Type: string Default: ""
 
Pass additional options to run lofreq call
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    alignment_coverage
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    alignment_coverage    mem
Type: integer Default: 1000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    alignment_coverage    time
Type: integer Default: 60
 
 
 
 
 
 
 
 
  
 
 
 
 
root    alignment_coverage    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    alignment_coverage    coverage
Type: integer Default: 5
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    stats
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    stats    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/sam2bam.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    savage
Type: object Default: {}
 
NOTE This rule only works in Linux.
  
 
 
 
  
 
 
 
 
root    savage    mem
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    savage    time
Type: integer Default: 1435
 
 
 
 
 
 
 
 
  
 
 
 
 
root    savage    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    savage    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/savage.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    savage    split
Type: integer Default: 20
 
Size of the batches of reads to be processed by SAVAGE. It is recommended that 500 < coverage/split < 1000.
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    haploclique    mem
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    time
Type: integer Default: 1435
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/haploclique.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    relax
Type: boolean Default: true
 
If set to True (default) a predefined set of parameter values is used for drawing edges between reads in the read graph.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    no_singletons
Type: boolean Default: true
 
Singletons are defined as proposed haplotypes which are supported by a single read. If this property is set to True, singletons are discarded.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    no_prob0
Type: boolean Default: true
 
If set to True (default) probability of the overhangs is ignored.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    clique_size_limit
Type: integer Default: 3
 
Sets a threshold to limit the size of cliques.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    max_num_cliques
Type: integer Default: 10000
 
Indicates the maximum number of clique to be considered in the next iteration.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique    extra_parameters
Type: string Default: ""
 
Additional parameters to be passed to haploclique.
 
Warning: this won’t overwrite the other options (e.g. clique_size_limi and max_num_cliques should still be set via their own respective properties, do not pass parameters --limit_clique_size= nor --max_cliques= via this extra property).
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique_visualization
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    haploclique_visualization    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique_visualization    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique_visualization    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/smallgenomeutilities.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique_visualization    region_start
Type: integer Default: 0
 
Use to specify a region of interest.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique_visualization    region_end
Type: integer Default: -1
 
Use to specify a region of interest.
  
 
Examples:
 
9719
+
 
29836
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    haploclique_visualization    msa
Type: string Default: ""
 
When the ground truth is available (e.g., simulation studies), a multiple sequence alignment of types making up the population can be provided, and additional checks are performed.
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    predicthaplo
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    predicthaplo    mem
Type: integer Default: 10000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    predicthaplo    time
Type: integer Default: 1435
 
 
 
 
 
 
 
 
  
 
 
 
 
root    predicthaplo    threads
Type: integer
 
 
 
 
 
 
 
 
  
 
 
 
 
root    predicthaplo    read_min_length
Type: integer Default: 0
 
 
 
 
 
 
 
 
  
 
 
 
 
root    predicthaplo    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/predicthaplo.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    amplicons
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    amplicons    mem
Type: integer Default: 256
 
 
 
 
 
 
 
 
  
 
 
 
 
root    amplicons    time
Type: integer Default: 10
 
 
 
 
 
 
 
 
  
 
 
 
 
root    amplicons    mincooc
Type: integer Default: 2
 
Minimal number of coocurrences to search for in amplicon. Lowering this property to 1 will make COJAC also look for amplicon with singleton mutations.
  
 
Example:
 
1
+
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    cooc
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    cooc    mem
Type: integer Default: 8192
 
 
 
 
 
 
 
 
  
 
 
 
 
root    cooc    time
Type: integer Default: 45
 
 
 
 
 
 
 
 
  
 
 
 
 
root    cooc    threads
Type: integer Default: 1
 
 
 
 
 
 
 
 
  
 
 
 
 
root    cooc    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/cojac.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    cooc    out_format
Type: enum (of string) Default: "lines"
 
Format of the output CSV.
 
     
  • lines(default) - each amplicon a separate entry on a separate line.
    •  
  • columns - one column per amplicon
    •  
 
 
Must be one of:
 
  • "lines"
  • "columns"
 
 
 
Example:
 
columns
+
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    mutlist
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    mutlist    mem
Type: integer Default: 256
 
 
 
 
 
 
 
 
  
 
 
 
 
root    mutlist    time
Type: integer Default: 10
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    sigmut
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    sigmut    mem
Type: integer Default: 4096
 
 
 
 
 
 
 
 
  
 
 
 
 
root    sigmut    time
Type: integer Default: 15
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline
Type: object Default: {}
 
This section is used to set up a timeline of the samples. Some output, e.g., deconvolution of quasispecies mixture using LolliPop, need to have a time component. By default it calls a script that uses regular expressions and look-up tables to extract this information from the samples’ own names. But by using the properties script and options and adapting the environment provided in property conda, it is possible to heavily customize the actions (e.g. it is possible to query an external database instead). For inspiration, see the default script file_parser.py.
  
 
 
 
  
 
 
 
 
root    timeline    mem
Type: integer Default: 1024
 
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline    time
Type: integer Default: 15
 
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/timeline.yaml"
 
The default environment only provides regular expression functions (python-reges) but depending on your needs you would want to provide a custom environment with additional tools (e.g. drivers to query a databse, etc.)
  
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline    threads
Type: integer Default: 1
 
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline    local
Type: boolean Default: true
 
Don’t dispatch the timeline rule to the cluster for execution, run locally.
  
 
Example:
 
False
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline    script
Type: string Default: "{VPIPE_BASEDIR}/scripts/file_parser.py"
 
Script that sets up a timeline of the samples.
 
Its purpose is to take the V-pipe’s samples TSV file and add two columns:
 
     
  • location: location of the sampes
    •  
  • date: sampling date of the samples
    •  
 
It will receive the following parameters (in addition to what is specified in property options:
 
     
  • –output <OUTPUT>: the output TSV file that must be created by the script.
    •  
  • <SAMPLE_TSV>: the input samples TSV file
    •  
 
For an example, see the default script file_parser.py, it uses regular expressions (regex) to parse the first two columns (sample and batch names) and extract a date, and a location code that is further look-ed up in a table. It takes two additional parameters:
 
     
  • –locations <LOOKUP>: look-up TSV table mapping the code to full location names
    •  
  • –regex_yaml <YAML>: YAML file with regular expressions defining how the date and code are extracted.
    •  
  
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline    options
Type: string Default: " --no-fallback"
 
Additional options to be passed to the script, e.g. for an extra configuration file with database server information.
 
By default, passes an option to the default script to force always using the regex (do not fall back to copy-pasting columns).
  
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline    locations_table
Type: string Default: ""
 
Option for the default script: TSV table that maps location codes (e.g. short alphanumeric codes) used in sample names to full names of locations (e.g. city names).
 
For example:
 
code    location
 10  Zürich (ZH)
 16  Genève (GE)
 Ba  Basel (BS)
-
  
 
Example:
 
wastewater_plants.tsv
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline    regex_yaml
Type: string Default: ""
 
Option for the default script: YAML file the defines how to parse time series information out of the columns of samples.tsv, e.g.:
 
 
sample: (?P<location>\d+)_(?P<year>20\d{2})_(?P<month>[01]?\d)_(?P<day>[0-3]?\d)
-datefmt: "%Y%m%d"
-
 
 
     
  • by default, samples.tsv’s first column (sample names) is used as-is for location codes and the second column (sequencing batch dates) is used for timeline’s date point.
    •  
  • YAML’s optional entry sample defines a regular expression to be applied on the first column (sample names)
    •  
  • YAML’s optional entry batch defines a regular expression to be applied on the second column (sequencing batch dates)
    •  
  • regular expression must define the following named capturing groups: 
       
    • location used for the location codes
      •  
    • year, month, day used for the dates of the timeline
      •  
    • alternatively, if the date doesn’t use a format with year + month + day elements – e.g., date uses week number of the year – group date can be used to capture the whole date string
      •  
    •  
  • YAML’s optional entry datefmt gives a time format string to parse the date capturing group.
    •  
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    tallymut
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    tallymut    mem
Type: integer Default: 4096
 
 
 
 
 
 
 
 
  
 
 
 
 
root    tallymut    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
 
 
 
root    tallymut    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/xsv.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    tallymut    timeline_file
Type: string Default: ""
 
If set, this user-provided TSV file (e.g.: generated with an external tool, prior of running V-pipe) will be used for obtaining locations and dates – as needed by LolliPop – instead of generating results/timeline.tsv with the rule timeline.
 This follows the following format (similar to the output of rule timeline):
 
sample  batch   reads   proto   location_code   date    location
+
  
 
Example:
 
wastewater_plants.tsv
+
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    timeline    regex_yaml
Type: string Default: ""
 
Option for the default script: YAML file the defines how to parse time series information out of the columns of samples.tsv, e.g.:
 
sample: (?P<location>\d+)_(?P<year>20\d{2})_(?P<month>[01]?\d)_(?P<day>[0-3]?\d)
+datefmt: "%Y%m%d"
+
 
     
  • by default, samples.tsv’s first column (sample names) is used as-is for location codes and the second column (sequencing batch dates) is used for timeline’s date point.
    •  
  • YAML’s optional entry sample defines a regular expression to be applied on the first column (sample names)
    •  
  • YAML’s optional entry batch defines a regular expression to be applied on the second column (sequencing batch dates)
    •  
  • regular expression must define the following named capturing groups:
     

       
    • location used for the location codes

      •  
    • year, month, day used for the dates of the timeline

      •  
    • alternatively, if the date doesn’t use a format with year + month + day elements – e.g., date uses week number of the year – group date can be used to capture the whole date string

      •  
    •  
  • YAML’s optional entry datefmt gives a time format string to parse the date capturing group.
    •  
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    tallymut
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    tallymut    mem
Type: integer Default: 4096
 
 
 
 
 
 
 
 
  
 
 
 
 
root    tallymut    time
Type: integer Default: 30
 
 
 
 
 
 
 
 
  
 
 
 
 
root    tallymut    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/xsv.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    tallymut    timeline_file
Type: string Default: ""
 
If set, this user-provided TSV file (e.g.: generated with an external tool, prior of running V-pipe) will be used for obtaining locations and dates – as needed by LolliPop – instead of generating results/timeline.tsv with the rule timeline.
 This follows the following format (similar to the output of rule timeline):
 
sample  batch   reads   proto   location_code   date    location
 A1_05_2023_04_12    20230428_HNG5MDRX2  250 v41 5   2023-04-12  Lugano (TI)
 A2_10_2023_04_13    20230428_HNG5MDRX2  250 v41 10  2023-04-13  Zürich (ZH)
 A3_16_2023_04_14    20230428_HNG5MDRX2  250 v41 16  2023-04-14  Genève (GE)
 …
-
 
     
  • The extra columns location and data are necessary for LolliPop.
    •  
  • Columns sample, batch, reads and proto are simply the fist four columns of samples.tsv 
       
    • V-pipe only needs column sample and batch for now.
      •  
    •  
  
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    deconvolution
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    deconvolution    mem
Type: integer Default: 4096
 
 
 
 
 
 
 
 
  
 
 
 
 
root    deconvolution    time
Type: integer Default: 240
 
 
 
 
 
 
 
 
  
 
 
 
 
root    deconvolution    threads
Type: integer Default: 4
 
 
 
 
 
 
 
 
  
 
 
 
 
root    deconvolution    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/lollipop.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    deconvolution    deconvolution_config
Type: string Default: "{VPIPE_BASEDIR}/../resources/cowwid/deconv_linear_logit_quasi_strat.yaml"
 
Configuration file with parameters for kernel deconvolution
  
 
Examples:
 
/git/lollipop/deconv_linear_logit_quasi_strat.yaml
-
 
/git/lollipop/deconv_linear_wald.yaml
-
 
/git/lollipop/deconv_bootstrap
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    deconvolution    variants_config
Type: string Default: ""
 
Variants configuration used during deconvolution
  
 
Example:
 
var_conf.yaml
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    deconvolution    variants_dates
Type: string Default: ""
 
Variants to scan per periods (as determined with COJAC by leveraging the output of the cooc rule)
  
 
Example:
 
var_dates.yaml
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    deconvolution    out_format
Type: enum (of string) Default: "lines"
 
Format of the output CSV.
 
     
  • lines(default) - each variants a separate entry on a separate line.
    •  
  • columns - one column per variant
    •  
 
 
Must be one of:
 
  • "lines"
  • "columns"
 
 
 
Example:
 
columns
-
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    web_visualization
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    web_visualization    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    web_visualization    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    web_visualization    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/visualization.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    diversity
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    diversity    mem
Type: integer Default: 2000
 
 
 
 
 
 
 
 
  
 
 
 
 
root    diversity    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    diversity    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/diversity_measures.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    dehuman
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    dehuman    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/dehuman.yaml"
 
 
 
 
 
 
 
 
  
 
 
 
 
root    dehuman    mem
Type: integer Default: 4096
 
 
 
 
 
 
 
 
  
 
 
 
 
root    dehuman    time
Type: integer Default: 235
 
 
 
 
 
 
 
 
  
 
 
 
 
root    dehuman    threads
Type: integer Default: 4
 
 
 
 
 
 
 
 
  
 
 
 
 
root    dehuman    ref_host
Type: string Default: "references/human.fa.gz"
 
Host’s genome used to remove reads (e.g. human genome)
 
Note: if this file is absent, it is possible to fetch it from a remote server, see property ref_host_url below.
  
 
Example:
 
/cluster/project/igenomes/Homo_sapiens/NCBI/GRCh38/Sequence/BWAIndex/genome.fa
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    dehuman    ref_host_url
Type: string Default: "http://ftp.ensembl.org/pub/current_fasta/homo_sapiens/dna/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz"
 
If the host’s genome specified in property ref_host isn’t present, fetch it from a remote server.
 
Note remember to set aside enough memory for the indexing rule, see section ref_bwa_index property mem.
  
 
Examples:
 
http://ftp.ensembl.org/pub/release-105/fasta/homo_sapiens/dna/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz
-
 
https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/000/001/405/GCA_000001405.29_GRCh38.p14/GCA_000001405.29_GRCh38.p14_genomic.fna.gz
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    dehuman    keep_host
Type: boolean Default: false
 
Indicate whether to store the host-aligned reads in a CRAM file …/alignments/host_aln.cram.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    dehuman    catchup
Type: boolean Default: false
 
Use this option when generating dehumanized raw reads (dehuman.cram) on old samples that have already been processed in the past — a catch up.
 
Normally, removing host-mapping reads requires analyzing reads which were rejected by V-pipe’s main processing (as specified in section general, property aligner). But this output is considered temporary and will get deleted by Snakemake once the processing of a sample has finished. To generate dehuman.cram V-pipe would need to run the aligner again, which will both regenerate the data necessary for this output but also generate a new alignment which will trigger the whole workflow again.
 Use this property catchup to only generate the input necessary for dehuman.cram, leaving untouched the alignment and everything else that has already been processed.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    checksum
Type: object Default: {}
 
 
 
 
  
 
 
 
 
root    checksum    mem
Type: integer Default: 256
 
 
 
 
 
 
 
 
  
 
 
 
 
root    checksum    time
Type: integer Default: 60
 
 
 
 
 
 
 
 
  
 
 
 
 
root    checksum    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/upload.yaml"
 
 
 
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    upload
Type: object Default: {}
 
This section is used to assist and prepare uploads of the data, e.g. to European Nucleotide Archive. By default it calls a script that creates symlinks making it easy to identify new/updated samples between calls of V-pipe. But by using the properties script and options and adapting the environment provided in property conda, it is possible to heavily customize the actions (e.g. it is possible to upload to an SFTP server by calling sftp from a modified script). For inspiration, see the default script prepare_upload_symlinks.sh.
  
 
 
 
  
 
 
 
 
root    upload    mem
Type: integer Default: 256
 
 
 
 
 
 
 
 
  
 
 
 
 
root    upload    time
Type: integer Default: 60
 
 
 
 
 
 
 
 
  
 
 
 
 
root    upload    conda
Type: string Default: "{VPIPE_BASEDIR}/envs/upload.yaml"
 
The default environment only provides hashing functions (xxhash, linux coreutils’ sha{nnn}sum collection, etc.) but depending on your needs you would want to provide a custom environment with additional tools (e.g. sftp, rsync, curl, lftp, custom specialized cloud uploaders, etc.)
  
 
 
 
 
 
 
 
  
 
 
 
 
root    upload    threads
Type: integer Default: 1
 
 
 
 
 
 
 
 
  
 
 
 
 
root    upload    local
Type: boolean Default: true
 
Don’t dispatch the rule to the cluster for execution, run locally.
  
 
Example:
 
False
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    upload    consensus
Type: enum (of string) Default: "ambig"
 
When preparing data for upload, specifies which consensus sequence should be uploaded.
 
 
Must be one of:
 
  • "ambig"
  • "majority"
 
 
 
Example:
 
majority
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    upload    checksum
Type: boolean Default: false
 
Generate checksum for each individual consensus sequence (if a consensus is regenerated, it will help determine whether the new file has changed content or is virtually the same as the previous).
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    upload    orig_fastq
Type: boolean Default: false
 
Also include the original .fastq.gz sequencing reads files from raw_data/ in the list of files to be uploaded. See property orig_cram below for a compressed version and see output dehumanized_raw_reads and section dehuman for depleting reads from the host.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    upload    orig_cram
Type: boolean Default: false
 
Also include a compressed version of the original sequencing raw reads files from raw_data/. Similar to property orig_fastq above, but with reference-based compression.
  
 
Example:
 
True
-
 
 
 
 
 
 
 
 
 
  
 
 
 
 
root    upload    script
Type: string Default: "{VPIPE_BASEDIR}/scripts/prepare_upload_symlinks.sh"
 
Custom script that assists and prepares uploads.
 
It will receive the following positional parameters:
 
     
  • <OUTPUT>: the output file that must be created by the script.
    •  
  • <SAMPLE_ID>: a string (with no path separator slashes) that can be used as a name, uniquely identifying the sample and the date.
    •  
  • <SAMPLE_DIR>: the base directory of the sample.
    •  
  • <UPLOAD_FILES>…: a list of files to consider for upload
    •  
 
For an example, see the default script prepare_upload_symlinks.sh, it generates symlinks that help tracking which samples are new and/or updated between runs of V-pipe and thus should be considered for upload.
  
 
 
 
 
 
 
 
  
 
 
 
 
root    upload    options
Type: string Default: ""
 
Named options to be passed to the script, before the positional parameters. E.g. for an extra configuration file with SFTP server information.
  
 
 
 
 
 
 
 
  
\ No newline at end of file
+
  • The extra columns location and data are necessary for LolliPop.
  • Columns sample, batch, reads and proto are simply the fist four columns of samples.tsv

    • V-pipe only needs column sample and batch for now.

root deconvolution
Type: object Default: {}

root deconvolution mem
Type: integer Default: 4096

root deconvolution time
Type: integer Default: 240

root deconvolution threads
Type: integer Default: 4

root deconvolution conda
Type: string Default: "{VPIPE_BASEDIR}/envs/lollipop.yaml"

root deconvolution deconvolution_config
Type: string Default: "{VPIPE_BASEDIR}/../resources/cowwid/deconv_linear_logit_quasi_strat.yaml"

Configuration file with parameters for kernel deconvolution
Examples:

/git/lollipop/deconv_linear_logit_quasi_strat.yaml
+
/git/lollipop/deconv_linear_wald.yaml
+
/git/lollipop/deconv_bootstrap
+

root deconvolution variants_config
Type: string Default: ""

Variants configuration used during deconvolution
Example:

var_conf.yaml
+

root deconvolution variants_dates
Type: string Default: ""

Variants to scan per periods (as determined with COJAC by leveraging the output of the cooc rule)
Example:

var_dates.yaml
+

root deconvolution out_format
Type: enum (of string) Default: "lines"

Format of the output CSV.

  • lines(default) - each variants a separate entry on a separate line.
  • columns - one column per variant

Must be one of:

  • "lines"
  • "columns"

Example:

columns
+

root deconvolution filters
Type: string Default: ""

List of filters for removing problematic mutations from tally. Some mutations might be problematic and need to be taken out – e.g. due to drop-outs in the multiplex PCR amplification, they do not show up in the data and this could be misinterpreted by LolliPop as proof of absence of a variant.
Example:

filters_preprint.yaml
+

root web_visualization
Type: object Default: {}

root web_visualization mem
Type: integer Default: 2000

root web_visualization time
Type: integer Default: 235

root web_visualization conda
Type: string Default: "{VPIPE_BASEDIR}/envs/visualization.yaml"

root diversity
Type: object Default: {}

root diversity mem
Type: integer Default: 2000

root diversity time
Type: integer Default: 235

root diversity conda
Type: string Default: "{VPIPE_BASEDIR}/envs/diversity_measures.yaml"

root dehuman
Type: object Default: {}

root dehuman conda
Type: string Default: "{VPIPE_BASEDIR}/envs/dehuman.yaml"

root dehuman mem
Type: integer Default: 4096

root dehuman time
Type: integer Default: 235

root dehuman threads
Type: integer Default: 4

root dehuman ref_host
Type: string Default: "references/human.fa.gz"

Host’s genome used to remove reads (e.g. human genome)

Note: if this file is absent, it is possible to fetch it from a remote server, see property ref_host_url below.
Example:

/cluster/project/igenomes/Homo_sapiens/NCBI/GRCh38/Sequence/BWAIndex/genome.fa
+

root dehuman ref_host_url
Type: string Default: "http://ftp.ensembl.org/pub/current_fasta/homo_sapiens/dna/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz"

If the host’s genome specified in property ref_host isn’t present, fetch it from a remote server.

Note remember to set aside enough memory for the indexing rule, see section ref_bwa_index property mem.
Examples:

http://ftp.ensembl.org/pub/release-105/fasta/homo_sapiens/dna/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz
+
https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/000/001/405/GCA_000001405.29_GRCh38.p14/GCA_000001405.29_GRCh38.p14_genomic.fna.gz
+

root dehuman keep_host
Type: boolean Default: false

Indicate whether to store the host-aligned reads in a CRAM file …/alignments/host_aln.cram.
Example:

True
+

root dehuman catchup
Type: boolean Default: false

Use this option when generating dehumanized raw reads (dehuman.cram) on old samples that have already been processed in the past — a catch up.

Normally, removing host-mapping reads requires analyzing reads which were rejected by V-pipe’s main processing (as specified in section general, property aligner). But this output is considered temporary and will get deleted by Snakemake once the processing of a sample has finished. To generate dehuman.cram V-pipe would need to run the aligner again, which will both regenerate the data necessary for this output but also generate a new alignment which will trigger the whole workflow again.
Use this property catchup to only generate the input necessary for dehuman.cram, leaving untouched the alignment and everything else that has already been processed.
Example:

True
+

root checksum
Type: object Default: {}

root checksum mem
Type: integer Default: 256

root checksum time
Type: integer Default: 60

root checksum conda
Type: string Default: "{VPIPE_BASEDIR}/envs/upload.yaml"

root upload
Type: object Default: {}

This section is used to assist and prepare uploads of the data, e.g. to European Nucleotide Archive. By default it calls a script that creates symlinks making it easy to identify new/updated samples between calls of V-pipe. But by using the properties script and options and adapting the environment provided in property conda, it is possible to heavily customize the actions (e.g. it is possible to upload to an SFTP server by calling sftp from a modified script). For inspiration, see the default script prepare_upload_symlinks.sh.

root upload mem
Type: integer Default: 256

root upload time
Type: integer Default: 60

root upload conda
Type: string Default: "{VPIPE_BASEDIR}/envs/upload.yaml"

The default environment only provides hashing functions (xxhash, linux coreutils’ sha{nnn}sum collection, etc.) but depending on your needs you would want to provide a custom environment with additional tools (e.g. sftp, rsync, curl, lftp, custom specialized cloud uploaders, etc.)

root upload threads
Type: integer Default: 1

root upload local
Type: boolean Default: true

Don’t dispatch the rule to the cluster for execution, run locally.
Example:

False
+

root upload consensus
Type: enum (of string) Default: "ambig"

When preparing data for upload, specifies which consensus sequence should be uploaded.

Must be one of:

  • "ambig"
  • "majority"

Example:

majority
+

root upload checksum
Type: boolean Default: false

Generate checksum for each individual consensus sequence (if a consensus is regenerated, it will help determine whether the new file has changed content or is virtually the same as the previous).
Example:

True
+

root upload orig_fastq
Type: boolean Default: false

Also include the original .fastq.gz sequencing reads files from raw_data/ in the list of files to be uploaded. See property orig_cram below for a compressed version and see output dehumanized_raw_reads and section dehuman for depleting reads from the host.
Example:

True
+

root upload orig_cram
Type: boolean Default: false

Also include a compressed version of the original sequencing raw reads files from raw_data/. Similar to property orig_fastq above, but with reference-based compression.
Example:

True
+

root upload script
Type: string Default: "{VPIPE_BASEDIR}/scripts/prepare_upload_symlinks.sh"

Custom script that assists and prepares uploads.

It will receive the following positional parameters:

  • <OUTPUT>: the output file that must be created by the script.
  • <SAMPLE_ID>: a string (with no path separator slashes) that can be used as a name, uniquely identifying the sample and the date.
  • <SAMPLE_DIR>: the base directory of the sample.
  • <UPLOAD_FILES>…: a list of files to consider for upload

For an example, see the default script prepare_upload_symlinks.sh, it generates symlinks that help tracking which samples are new and/or updated between runs of V-pipe and thus should be considered for upload.

root upload options
Type: string Default: ""

Named options to be passed to the script, before the positional parameters. E.g. for an extra configuration file with SFTP server information.
\ No newline at end of file diff --git a/docs/README.md b/docs/README.md index bc4d9196..bed52a0d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,22 +1,30 @@ # Tutorials -You can find two tutorials in this directory: +We strongly advise our users to start discovering V-pipe by looking at the tutorials -- [tutorial_hiv.md](tutorial_hiv.md): uses HIV test data -- [tutorial_sarscov2.md](tutorial_sarscov2.md): uses SARS-CoV-2 data from a publication +You can find several tutorials in this directory: + +## Getting V-pipe installed + +- [V-pipe Installation](https://github.com/cbg-ethz/V-pipe/blob/master/docs/tutorial_0_install.md) + +## Viruses + +- [V-Pipe HIV Tutorial](https://github.com/cbg-ethz/V-pipe/blob/master/docs/tutorial_hiv.md): uses HIV test data +- [SARS-CoV-2 Tutorial](https://github.com/cbg-ethz/V-pipe/blob/master/docs/tutorial_sarscov2.md): uses SARS-CoV-2 data from a publication ## Note about the tutorials -Due to automated texting, each copy-pastable block begins with a command entering the directory and ends with on leaving the directory: +Due to automated testing, each copy-pastable block begins with a command entering the directory and ends with one leaving the directory: ```bash cd tutorial/work/ # do something cd ../.. ``` -Of course you don't necessarily need to do that. You can simply remain in the working directory. +Of course, you don't necessarily need to do that. You can simply remain in the working directory. -When editing files like `config.yaml`, you can use your favorite editor (`vim`, `emacs`, `nano`, [butterflies](https://xkcd.com/378/), etc.). By default our tutorials use a [_heredoc_](https://en.wikipedia.org/wiki/Here_document) to make it easier to copy-paste the blocks into bash: +When editing files like `config.yaml`, you can use your favorite editor (`vim`, `emacs`, `nano`, [butterflies](https://xkcd.com/378/), etc.). By default, our tutorials use a [_heredoc_](https://en.wikipedia.org/wiki/Here_document) to make it easier to copy-paste the blocks into bash: ```bash cat > config.yaml < # V-Pipe Installation -V-pipe is a workflow designed for the analysis of next generation sequencing (NGS) data from viral pathogens. It produces a number of results in a curated format (e.g., consensus sequences, SNV calls, local/global haplotypes). V-pipe is written using the Snakemake workflow management system. +V-pipe is a workflow designed for the analysis of next-generation sequencing (NGS) data from viral pathogens. It produces a number of results in a curated format (e.g., consensus sequences, SNV calls, local/global haplotypes). V-pipe is written using the Snakemake workflow management system. -The present tutorial will show you how to install V-pipe and the dependencies required to start using it - bioconda, conda-froge mamba and snakemake - before continuing with other tutorials and analyse virus data. +The present tutorial will show you how to install V-pipe and the dependencies required to start using it - bioconda, conda-froge mamba and snakemake - before continuing with other tutorials and analysing virus data. ## Requirements @@ -42,8 +42,8 @@ We will organise our software in the following tree structure, which will be reu - `vp-analysis` is the main directory where we will store everything. - `Miniforge3` is the directory where conda will be installed including the dependencies to start using V-pipe. -- `V-pipe` is the directory where V-pipe's own code will be downloaded from GitHub -- finally, each analysis of virus data will be performed into directory like `work…`, which holds the configuration and the sequencing data for that particular analysis. +- `V-pipe` is the directory where V-pipe's code will be downloaded from GitHub +- finally, each analysis of virus data will be performed in a directory like `work…`, which holds the configuration and the sequencing data for that particular analysis. ## Install V-pipe and conda from scratch @@ -52,7 +52,7 @@ V-pipe uses the [Bioconda](https://bioconda.github.io/) bioinformatics software For advanced users: If your are fluent with these tools, see [below](#fluent-users) -In this present short tutorial you will learn how to setup a workflow for the various examples in the analysis tutorials. +In this short tutorial, you will learn how to setup a workflow for the various examples in the analysis tutorials. To deploy V-pipe, you can use the installation script with the following parameters: @@ -69,7 +69,7 @@ bash quick_install.sh -p vp-analysis -w work If you get `zsh: permission denied: ./quick_install.sh`, run `chmod +x quick_install.sh` this gives the necessary permissions. -**Tip:** To create and populate other new working directories, you can call init_project.sh from within the new directory: +**Tip:** To create and populate other new working directories, you can call `init_project.sh` from within the new directory: ```bash cd vp-analysis/ @@ -85,7 +85,7 @@ cd - ### Analyse data -Now that you have setup the software necessary to start using V-pipe, you can follow with one of the tutorials showing you analysis of viral sequencing data: +Now that you have setup the software necessary to start using V-pipe, you can follow with one of the tutorials showing you the analysis of viral sequencing data: - [tutorial_hiv.md](tutorial_hiv.md): uses HIV test data - [tutorial_sarscov2.md](tutorial_sarscov2.md): uses SARS-CoV-2 data from a publication @@ -95,7 +95,7 @@ Now that you have setup the software necessary to start using V-pipe, you can fo For advanced users: If your are fluent with these tools, you can: -* directly download and install [bioconda](https://bioconda.github.io/user/install.html) and [snakemake](https://snakemake.readthedocs.io/en/stable/getting_started/installation.html#installation-via-conda), +* directly download and install [Miniforge3](https://github.com/conda-forge/miniforge#Download), setup [bioconda](https://bioconda.github.io/index.html#usage) and install [snakemake](https://snakemake.readthedocs.io/en/stable/getting_started/installation.html#installation-via-conda), * specifiy your V-pipe configuration, and start using V-pipe Use `--use-conda` to [automatically download and install](https://snakemake.readthedocs.io/en/stable/snakefiles/deployment.html#integrated-package-management) any further pipeline dependencies. Please refer to the documentation for additional instructions. diff --git a/resources/auxiliary_workflows/benchmark/resources/local_haplotype_setup/README.md b/resources/auxiliary_workflows/benchmark/resources/local_haplotype_setup/README.md index 01461f08..85ae7ec1 100644 --- a/resources/auxiliary_workflows/benchmark/resources/local_haplotype_setup/README.md +++ b/resources/auxiliary_workflows/benchmark/resources/local_haplotype_setup/README.md @@ -1,6 +1,6 @@ # Local haplotype reconstruction benchmark -This repository stores the scripts and notebooks used to conduct the benchmark study presented in the manuscript: XXX +This repository stores the scripts and notebooks used to conduct the benchmark study presented in the manuscript: https://www.biorxiv.org/content/10.1101/2024.06.06.597712v1 To reproduce the benchmark study and create the figures presented in the manuscript, use the following instructions: 1. Clone the repository of V-pipe 3.0 into your working directory: diff --git a/workflow/Snakefile b/workflow/Snakefile index 119ee07a..2bd8af3d 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -1,3 +1,6 @@ +container: "docker://snakemake/snakemake:v7.32.4" + + import logging LOGGER = logging.getLogger("snakemake.logging") diff --git a/workflow/documentation.md b/workflow/documentation.md new file mode 100644 index 00000000..fbf4523a --- /dev/null +++ b/workflow/documentation.md @@ -0,0 +1,20 @@ +# Documentation + +As specified in the [usage section](../README.md#usage) of the main README file, these are the steps you need to perform to use V-pipe. + +To configure V-pipe refer to the documentation present in [config/README.md](../config/README.md). + +V-pipe expects the input samples to be organized in a [two-level](../config/README.md#samples) directory hierarchy, and the sequencing reads must be provided in a sub-folder named `raw_data`. +Check the utils subdirectory for [mass-importers tools](../utils/README.md#samples-mass-importers) that can assist you in generating this hierarchy. + +We provide [virus-specific base configuration files](../config/README.md#virus-base-config) which contain handy defaults for, e.g., HIV and SARS-CoV-2. Set the virus in the general section of the configuration file: +```yaml +general: + virus_base_config: hiv +``` + +## Tutorials + +If you want to test your new V-pipe installation, we strongly encourage you to check our tutorials which provide real example data. + +Tutorials for your first steps with V-pipe for different scenarios are available in the [docs/](../docs/README.md) subdirectory. diff --git a/workflow/envs/bcftools.yaml b/workflow/envs/bcftools.yaml index 4883ab8d..10c494c2 100644 --- a/workflow/envs/bcftools.yaml +++ b/workflow/envs/bcftools.yaml @@ -2,5 +2,5 @@ channels: - conda-forge - bioconda dependencies: - - bcftools = 1.13 - - cyvcf2 = 0.30.11 + - bcftools = 1.20 + - cyvcf2 = 0.31.0 diff --git a/workflow/envs/lollipop.yaml b/workflow/envs/lollipop.yaml index 7f42ad1b..a42daa39 100644 --- a/workflow/envs/lollipop.yaml +++ b/workflow/envs/lollipop.yaml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - lollipop =0.3.0 + - lollipop =0.4.1 diff --git a/workflow/envs/paired_end_read_merger.yaml b/workflow/envs/paired_end_read_merger.yaml new file mode 100644 index 00000000..5535dbd2 --- /dev/null +++ b/workflow/envs/paired_end_read_merger.yaml @@ -0,0 +1,22 @@ +channels: + - conda-forge + - bioconda +dependencies: + - samtools=1.10 + - python >=3 + - progress + - numpy + - pandas + - pyyaml + - biopython + - bcbio-gff + - pysam + - pysamstats + - scikit-learn + - scipy + - matplotlib-base + - mafft + - git-lfs + - pip + - pip: + - git+https://git.bsse.ethz.ch/cbg/viruses/smallgenomeutilities@dev diff --git a/workflow/envs/viloca.yaml b/workflow/envs/viloca.yaml new file mode 100644 index 00000000..18c92cd9 --- /dev/null +++ b/workflow/envs/viloca.yaml @@ -0,0 +1,8 @@ +channels: + - conda-forge + - bioconda +dependencies: + - libshorah + - python=3.10.4 + - viloca=1.0.0 + - coreutils # [not linux] diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index d0b34108..c6724d88 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -685,11 +685,20 @@ for srec in sample_list: # in adition to standard VCF files, ShoRAH2 also produces CSV tables if config.general["snv_caller"] == "shorah": results.append(os.path.join(sdir, "variants/SNVs/snvs.csv")) + elif config.general["snv_caller"] == "viloca": + results.append( + os.path.join(sdir, "variants/SNVs/snv/cooccurring_mutations.csv") + ) # all snv callers ('shorah', 'lofreq') produce standard VCF files results.append(os.path.join(sdir, "variants/SNVs/snvs.vcf")) # local haplotypes if config.output["local"]: - results.append(os.path.join(sdir, "variants/SNVs/snvs.csv")) + if config.general["snv_caller"] == "shorah": + results.append(os.path.join(sdir, "variants/SNVs/snvs.csv")) + elif config.general["snv_caller"] == "viloca": + results.append( + os.path.join(sdir, "variants/SNVs/snv/cooccurring_mutations.csv") + ) # global haplotypes if config.output["global"]: if config.general["haplotype_reconstruction"] == "savage": diff --git a/workflow/rules/signatures.smk b/workflow/rules/signatures.smk index 78b03a16..98ad814e 100644 --- a/workflow/rules/signatures.smk +++ b/workflow/rules/signatures.smk @@ -320,6 +320,9 @@ rule deconvolution: if config.deconvolution["variants_dates"] else [] ), + filters=( + config.deconvolution["filters"] if config.deconvolution["filters"] else [] + ), output: deconvoluted=cohortdir("deconvoluted.tsv.zst"), deconv_json=cohortdir("deconvoluted_upload.json"), @@ -343,7 +346,7 @@ rule deconvolution: threads: config.deconvolution["threads"] shell: """ - {params.LOLLIPOP} deconvolute "--output={output.deconvoluted}" "--out-json={output.deconv_json}" "--var={input.var_conf}" "--vd={input.var_dates}" "--dec={input.deconv_conf}" {params.out_format} {params.seed} "{input.tallymut}" 2> >(tee -a {log.errfile} >&2) > >(tee -a {log.outfile}) + {params.LOLLIPOP} deconvolute "--output={output.deconvoluted}" "--out-json={output.deconv_json}" "--var={input.var_conf}" "--vd={input.var_dates}" "--dec={input.deconv_conf}" "--filters={input.filters}" {params.out_format} {params.seed} "{input.tallymut}" 2> >(tee -a {log.errfile} >&2) > >(tee -a {log.outfile}) """ diff --git a/workflow/rules/snv.smk b/workflow/rules/snv.smk index 0389569c..2c94be8f 100644 --- a/workflow/rules/snv.smk +++ b/workflow/rules/snv.smk @@ -287,10 +287,134 @@ rule lofreq: """ +rule paired_end_read_merger: + input: + fname_bam=alignment_wildcard, + fname_ref=( + cohortdir("cohort_consensus.fasta") + if config.viloca["consensus"] + else reference_file + ), + fname_ref_idx=( + cohortdir("cohort_consensus.fasta") + if config.viloca["consensus"] + else reference_file + ) + + ".fai", + output: + fname_bam_merged="{dataset}/alignment/REF_aln.merged.bam", + fname_sam_merged=temp_with_prefix("{dataset}/alignment/REF_aln.merged.sam"), + fname_sam=temp_with_prefix("{dataset}/alignment/REF_aln.sam"), + fname_sam_nonmerged="{dataset}/alignment/REF_aln.nonmerged.sam", + fname_sam_sort=temp_with_prefix("{dataset}/alignment/REF_aln.sort.sam"), + params: + SAMTOOLS=config.applications["samtools"], + PAIRED_END_READ_MERGER=config.applications["paired_end_read_merger"], + sort_tmp=temp_prefix("{dataset}.tmp"), + log: + outfile="{dataset}/alignment/paired_end_read_merger.out.log", + errfile="{dataset}/alignment/paired_end_read_merger.err.log", + conda: + config.paired_end_read_merger["conda"] + shell: + """ + ## Preparation + {params.SAMTOOLS} view -h -T {input.fname_ref} -t {input.fname_ref_idx} {input.fname_bam} -o {output.fname_sam} > {log.outfile} 2> >(tee {log.errfile} >&2) + ## sort accrording to QNAME + rm -f '{params.sort_tmp}'.[0-9]*.bam + {params.SAMTOOLS} sort -T "{params.sort_tmp}" -O sam -n {output.fname_sam} -o {output.fname_sam_sort} >> {log.outfile} 2> >(tee -a {log.errfile} >&2) + ## run script + {params.PAIRED_END_READ_MERGER} {output.fname_sam_sort} {output.fname_sam_merged} {output.fname_sam_nonmerged} {input.fname_ref} >> {log.outfile} 2> >(tee -a {log.errfile} >&2) + touch {output.fname_sam_nonmerged} + ## sort + rm -f '{params.sort_tmp}'.[0-9]*.bam + {params.SAMTOOLS} sort -T "{params.sort_tmp}" -o "{output.fname_bam_merged}" "{output.fname_sam_merged}" >> {log.outfile} 2> >(tee -a {log.errfile} >&2) + {params.SAMTOOLS} index "{output.fname_bam_merged}" >> {log.outfile} 2> >(tee -a {log.errfile} >&2) + """ + + +rule viloca: + input: + REF=( + cohortdir("cohort_consensus.fasta") + if config.viloca["consensus"] + else reference_file + ), + BAM=( + rules.paired_end_read_merger.output.fname_bam_merged + if config.viloca["merge_paired_end_reads"] + else alignment_wildcard + ), + output: + SNVs="{dataset}/variants/SNVs/snvs.vcf", + CSV="{dataset}/variants/SNVs/snv/cooccurring_mutations.csv", + WORK_DIR=directory("{dataset}/variants/SNVs"), + params: + READ_LEN=read_len, + INSERT_FILE=config.viloca["insert_bedfile"], + MODE=config.viloca["mode"], + SHIFT=config.viloca["shift"], + EXTRA=config.viloca["extra"], + VILOCA=config.applications["viloca"], + log: + outfile="{dataset}/variants/SNVs/viloca.out.log", + errfile="{dataset}/variants/SNVs/viloca.err.log", + conda: + config.viloca["conda"] + benchmark: + "{dataset}/variants/SNVs/viloca.benchmark" + threads: config.viloca["threads"] + resources: + disk_mb=2000, + mem_mb=config.viloca["mem"], + runtime=config.viloca["time"], + shell: + """ + let "WINDOW_SHIFTS=({params.READ_LEN} * 4/5 + {params.SHIFT}) / {params.SHIFT}" + let "WINDOW_LEN=WINDOW_SHIFTS * {params.SHIFT}" + echo "Windows are shifted by: ${{WINDOW_SHIFTS}} bp" > {log.outfile} + echo "The window length is: ${{WINDOW_LEN}} bp" >> {log.outfile} + + # Get absolute path for input files + CWD=${{PWD}} + WORK_DIR="$(realpath -m {output.WORK_DIR})" + BAM="$(realpath {input.BAM})" + REF="$(realpath {input.REF})" + OUTFILE="$(realpath -m {log.outfile})" + ERRFILE="$(realpath -m {log.errfile})" + + # Create directory for running VILOCA + DIR="${{WORK_DIR}}" + if [[ ! -d "${{DIR}}" ]]; then + echo "Creating directory ${{DIR}}" >> $OUTFILE + mkdir -p ${{DIR}} + fi + # Change to the directory where VILOCA is to be executed + cd "${{DIR}}" + + # Run VILOCA + echo "Running VILOCA" >> $OUTFILE + if [[ "{params.INSERT_FILE}" == "None" ]]; then + {params.VILOCA} {params.EXTRA} -t {threads} --mode {params.MODE} -w ${{WINDOW_LEN}} -s {params.SHIFT} -b ${{BAM}} -f ${{REF}} >> $OUTFILE 2> >(tee -a $ERRFILE >&2) + else + INSERTFILE=${{CWD}}/{params.INSERT_FILE} + echo "Insert file used ${{CWD}}/{params.INSERT_FILE}" >> $OUTFILE + {params.VILOCA} {params.EXTRA} -t {threads} --mode {params.MODE} -z ${{INSERTFILE}} -b ${{BAM}} -f ${{REF}} >> $OUTFILE 2> >(tee -a $ERRFILE >&2) + fi + + # rename viloca output snv/SNVs_0.010000_final.vcf --> snvs.vcf + cp "${{WORK_DIR}}/snv/SNVs_0.010000_final.vcf" "${{WORK_DIR}}/snvs.vcf" + """ + + if config.general["snv_caller"] == "shorah": - ruleorder: snv > lofreq + ruleorder: snv > lofreq > viloca elif config.general["snv_caller"] == "lofreq": - ruleorder: lofreq > snv + ruleorder: lofreq > snv > viloca + +elif config.general["snv_caller"] == "viloca": + + ruleorder: viloca > lofreq > snv diff --git a/workflow/schemas/config_schema.json b/workflow/schemas/config_schema.json index 648e5ef6..fed024ea 100644 --- a/workflow/schemas/config_schema.json +++ b/workflow/schemas/config_schema.json @@ -38,8 +38,8 @@ "snv_caller": { "type": "string", "default": "shorah", - "enum": ["shorah","lofreq"], - "description": "There are two options available for calling single nucleotide variants, either using [ShoRAH (`shorah`)](https://github.com/cbg-ethz/shorah) [^5] or [LoFreq (`lofreq`)](https://csb5.github.io/lofreq/) [^6]. ShoRAH is used by default. If you prefer to use LoFreq, then indicate so in the configuration file as in the example\n\n[^5]: Zagordi, O. et al. ShoRAH: estimating the genetic diversity of a mixed sample from next-generation sequencing data. BMC Bioinformatics. 2011.\n[^6]: Wilm, A. et al. LoFreq: A sequence-quality aware, ultra-sensitive variant caller for uncovering cell-population heterogeneity from high-throughput sequencing datasets. Nucleic Acids Res. 2012.", + "enum": ["shorah","lofreq", "viloca"], + "description": "There are three options available for calling single nucleotide variants, either using [ShoRAH (`shorah`)](https://github.com/cbg-ethz/shorah) [^5], [LoFreq (`lofreq`)](https://csb5.github.io/lofreq/) [^6] or [VILOCA (`viloca`)](https://github.com/cbg-ethz/viloca) [^7] . ShoRAH is used by default. If you prefer to use LoFreq, then indicate so in the configuration file as in the example\n\n[^5]: Zagordi, O. et al. ShoRAH: estimating the genetic diversity of a mixed sample from next-generation sequencing data. BMC Bioinformatics. 2011.\n[^6]: Wilm, A. et al. LoFreq: A sequence-quality aware, ultra-sensitive variant caller for uncovering cell-population heterogeneity from high-throughput sequencing datasets. Nucleic Acids Res. 2012.", "examples": ["lofreq"] }, "haplotype_reconstruction": { @@ -228,7 +228,7 @@ "local": { "type": "boolean", "default": false, - "description": "This option activates local haplotype reconstruction (only available when using ShoRAH).", + "description": "This option activates local haplotype reconstruction (only available when using ShoRAH or VILOCA).", "examples": [true] }, "global": { @@ -338,6 +338,10 @@ "type": "string", "default": "frameshift_deletions_checks" }, + "paired_end_read_merger": { + "type": "string", + "default": "paired_end_read_merger" + }, "mafft": { "type": "string", "default": "mafft" @@ -386,6 +390,10 @@ "type": "string", "default": "shorah shotgun" }, + "viloca": { + "type": "string", + "default": "viloca run" + }, "lofreq": { "type": "string", "default": "lofreq" @@ -989,6 +997,27 @@ "default": {}, "type": "object" }, + "paired_end_read_merger": { + "properties": { + "mem": { + "type": "integer", + "default": 1250 + }, + "time": { + "type": "integer", + "default": 30 + }, + "threads": { + "type": "integer" + }, + "conda": { + "type": "string", + "default": "{VPIPE_BASEDIR}/envs/paired_end_read_merger.yaml" + } + }, + "default": {}, + "type": "object" + }, "basecounts": { "properties": { "mem": { @@ -1185,6 +1214,60 @@ "default": {}, "type": "object" }, + "viloca": { + "properties": { + "mem": { + "type": "integer", + "default": 10000 + }, + "threads": { + "type": "integer" + }, + "time": { + "type": "integer", + "default": 2880 + }, + "conda": { + "type": "string", + "default": "{VPIPE_BASEDIR}/envs/viloca.yaml" + }, + "consensus": { + "type": "boolean", + "default": false, + "description": "Indicate whether to use the cohort-consensus sequence from the analyzed samples (output from `minor_variants` rule located in the cohort-wide output `results/cohort_onsensus.fasta`) or the reference sequence by setting this option to False.", + "examples": [false] + }, + "merge_paired_end_reads": { + "type": "boolean", + "default": false, + "description": "Merge paired-end reads in the preprocessing. This is a preprocessing snakemake rule.", + "examples": [false] + }, + "shift": { + "type": "integer", + "default": 3, + "description": "VILOCA performs local haplotype reconstruction on windows of the read alignment. The overlap between these windows is defined by the window shifts. By default, it is set to 3, i.e., apart from flanking regions each position is covered by 3 windows." + }, + "insert_bedfile": { + "type": "string", + "default": "None", + "description": "VILOCA performs local haplotype reconstruction on windows of the read alignment. In a first step the alignment is tiled into local regions. This can be done uniformly then set this value None, otherwise path to an (optional) insert file (primer tiling strategy)" + }, + "mode": { + "type": "string", + "default": "use_quality_scores", + "enum": ["shorah","learn_error_params","use_quality_scores"], + "description": "Mode in which to run VILOCA: shorah, learn_error_params, use_quality_scores. If quality scores are available, we recommend this option" + }, + "extra": { + "type": "string", + "default": "", + "description": "Pass additional options to run `viloca`" + } + }, + "default": {}, + "type": "object" + }, "lofreq": { "properties": { "mem": { @@ -1578,6 +1661,12 @@ "default": "lines", "description": "Format of the output CSV.\n- `lines`(default) - each variants a separate entry on a separate line.\n- `columns` - one column per variant", "examples": ["columns"] + }, + "filters": { + "type": "string", + "default": "", + "description": "List of filters for removing problematic mutations from tally. Some mutations might be problematic and need to be taken out -- e.g. due to drop-outs in the multiplex PCR amplification, they do not show up in the data and this could be misinterpreted by LolliPop as proof of absence of a variant.", + "examples": ["filters_preprint.yaml"] } }, "default": {},