Merge branch 'master' into hotfix/Snakemake-Profiles#45

mbhall88 · Apr 16, 2022 · 1aa7d7b · 1aa7d7b
2 parents 8907607 + d7c0d8a
commit 1aa7d7b
Show file tree

Hide file tree

Showing 8 changed files with 77 additions and 50 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ 3.5, 3.6, 3.7, 3.8 ]
+        python-version: [ 3.6, 3.7, 3.8 ]
     steps:
       - uses: actions/checkout@v2
 
@@ -52,7 +52,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ 3.5, 3.6, 3.7, 3.8 ]
+        python-version: [ 3.6, 3.7, 3.8 ]
     steps:
       - uses: actions/checkout@v2
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,18 @@ This document tracks changes to the `master` branch of the profile.
 
 ## [Unreleased]
 
+### Added
+
+- Default project in cookiecutter
+
+### Removed
+
+- Default threads in cookiecutter
+
+### Changed
+
+- Default project and queue will be removed from the submission command if they are present in the `lsf.yaml`
+
 ### Fixed
 
 - Support quoted jobid from `snakemake>=v7.1.1` [[#45][45]]

diff --git a/README.md b/README.md
@@ -187,16 +187,6 @@ without `mem_mb` set under `resources`.
 See [below](#standard-rule-specific-cluster-resource-settings) for how to overwrite this
 in a `rule`.
 
-#### `default_threads`
-
-**Default**: `1`
-
-This sets the default number of threads for a `rule` being submitted to the cluster
-without the `threads` variable set.
-
-See [below](#standard-rule-specific-cluster-resource-settings) for how to overwrite this
-in a `rule`.
-
 #### `default_cluster_logdir`
 
 **Default**: `"logs/cluster"`
@@ -229,6 +219,15 @@ The default queue on the cluster to submit jobs to. If left unset, then the defa
 your cluster will be used.  
 The `bsub` parameter that this controls is [`-q`][bsub-q].
 
+#### `default_project`
+
+**Default**: None
+
+The default project on the cluster to submit jobs with. If left unset, then the default on
+your cluster will be used.
+
+The `bsub` parameter that this controls is [`-P`][bsub-P].
+
 #### `max_status_checks_per_second`
 
 **Default**: `10`
@@ -314,7 +313,7 @@ rule foo:
     output: "bar.txt"
     shell:
         "grep 'bar' {input} > {output}"
-        
+
 rule bar:
     input: "bar.txt"
     output: "file.out"
@@ -358,7 +357,7 @@ The above is also a valid form of the previous example but **not recommended**.
 
 #### Quote-escaping
 
-Some LSF commands require multiple levels of quote-escaping.
+Some LSF commands require multiple levels of quote-escaping.  
 For example, to exclude a node from job submission which has non-alphabetic characters
 in its name ([docs](https://www.ibm.com/support/knowledgecenter/SSWRJV_10.1.0/lsf_command_ref/bsub.__r.1.html?view=embed)): `bsub -R "select[hname!='node-name']"`.
 
@@ -412,4 +411,3 @@ Please refer to [`CONTRIBUTING.md`](CONTRIBUTING.md).
 [yaml-collections]: https://yaml.org/spec/1.2/spec.html#id2759963
 [leandro]: https://github.com/leoisl
 [snakemake_params]: https://snakemake.readthedocs.io/en/stable/executable.html#all-options
-
diff --git a/cookiecutter.json b/cookiecutter.json
@@ -1,18 +1,32 @@
 {
-  "LSF_UNIT_FOR_LIMITS": ["KB", "MB", "GB", "TB", "PB", "EB", "ZB"],
-  "UNKWN_behaviour": ["wait", "kill"],
-  "ZOMBI_behaviour": ["ignore", "kill"],
+  "LSF_UNIT_FOR_LIMITS": [
+    "KB",
+    "MB",
+    "GB",
+    "TB",
+    "PB",
+    "EB",
+    "ZB"
+  ],
+  "UNKWN_behaviour": [
+    "wait",
+    "kill"
+  ],
+  "ZOMBI_behaviour": [
+    "ignore",
+    "kill"
+  ],
   "latency_wait": 5,
   "use_conda": false,
   "use_singularity": false,
   "restart_times": 0,
   "print_shell_commands": false,
   "jobs": 500,
   "default_mem_mb": 1024,
-  "default_threads": 1,
   "default_cluster_logdir": "logs/cluster",
   "default_queue": "",
+  "default_project": "",
   "max_status_checks_per_second": 10,
   "max_jobs_per_second": 10,
   "profile_name": "lsf"
-}
+}
diff --git a/tests/test_lsf_submit.py b/tests/test_lsf_submit.py
@@ -25,7 +25,7 @@ class TestSubmitter(unittest.TestCase):
         CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000
     )
     @patch.object(
-        CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8
+        CookieCutter, CookieCutter.get_default_project.__name__, return_value="proj"
     )
     @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random")
     def test___several_trivial_getter_methods(self, *mocks):
@@ -72,10 +72,12 @@ def test___several_trivial_getter_methods(self, *mocks):
         ).format(outlog=expected_outlog, errlog=expected_errlog)
         self.assertEqual(lsf_submit.jobinfo_cmd, expected_jobinfo_cmd)
         self.assertEqual(lsf_submit.queue_cmd, "-q q1")
+        self.assertEqual(lsf_submit.proj, "proj")
+        self.assertEqual(lsf_submit.proj_cmd, "-P proj")
         self.assertEqual(
             lsf_submit.submit_cmd,
             "bsub -M {mem} -n 1 -R 'select[mem>{mem}] rusage[mem={mem}] span[hosts=1]' "
-            "{jobinfo} -q q1 cluster_opt_1 cluster_opt_2 cluster_opt_3 "
+            "{jobinfo} -q q1 -P proj cluster_opt_1 cluster_opt_2 cluster_opt_3 "
             "real_jobscript.sh".format(mem=expected_mem, jobinfo=expected_jobinfo_cmd),
         )
 
@@ -96,9 +98,6 @@ def test___several_trivial_getter_methods(self, *mocks):
     @patch.object(
         CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000
     )
-    @patch.object(
-        CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8
-    )
     @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random")
     def test____submit_cmd_and_get_external_job_id___real_output_stream_from_submission(
         self, *mocks
@@ -124,9 +123,6 @@ def test____submit_cmd_and_get_external_job_id___real_output_stream_from_submiss
     @patch.object(
         CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000
     )
-    @patch.object(
-        CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8
-    )
     @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random")
     def test____submit_cmd_and_get_external_job_id___output_stream_has_no_jobid(
         self, *mocks
@@ -150,7 +146,7 @@ def test____submit_cmd_and_get_external_job_id___output_stream_has_no_jobid(
         CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000
     )
     @patch.object(
-        CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8
+        CookieCutter, CookieCutter.get_default_project.__name__, return_value="proj"
     )
     @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random")
     @patch.object(OSLayer, OSLayer.mkdir.__name__)
@@ -203,7 +199,7 @@ def test___submit___successfull_submit(
         expected_mem = "2662"
         run_process_mock.assert_called_once_with(
             "bsub -M {mem} -n 1 -R 'select[mem>{mem}] rusage[mem={mem}] span[hosts=1]' "
-            "{jobinfo} -q q1 cluster_opt_1 cluster_opt_2 cluster_opt_3 "
+            "{jobinfo} -q q1 -P proj cluster_opt_1 cluster_opt_2 cluster_opt_3 "
             "real_jobscript.sh".format(mem=expected_mem, jobinfo=expected_jobinfo_cmd)
         )
         print_mock.assert_called_once_with(
@@ -217,7 +213,7 @@ def test___submit___successfull_submit(
         CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000
     )
     @patch.object(
-        CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8
+        CookieCutter, CookieCutter.get_default_project.__name__, return_value="proj"
     )
     @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random")
     @patch.object(OSLayer, OSLayer.mkdir.__name__)
@@ -262,7 +258,7 @@ def test___submit___failed_submit_bsub_invocation_error(
         expected_mem = "2662"
         run_process_mock.assert_called_once_with(
             "bsub -M {mem} -n 1 -R 'select[mem>{mem}] rusage[mem={mem}] span[hosts=1]' "
-            "{jobinfo} -q q1 cluster_opt_1 cluster_opt_2 cluster_opt_3 "
+            "{jobinfo} -q q1 -P proj cluster_opt_1 cluster_opt_2 cluster_opt_3 "
             "real_jobscript.sh".format(mem=expected_mem, jobinfo=expected_jobinfo_cmd)
         )
         print_mock.assert_not_called()
@@ -297,9 +293,6 @@ def test_get_queue_cmd_returns_cookiecutter_default_if_no_cluster_config(
     @patch.object(
         CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000
     )
-    @patch.object(
-        CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8
-    )
     @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random")
     def test_rule_specific_params_are_submitted(self, *mocks):
         argv = [
@@ -354,7 +347,7 @@ def test_rule_specific_params_are_submitted(self, *mocks):
         CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000
     )
     @patch.object(
-        CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8
+        CookieCutter, CookieCutter.get_default_project.__name__, return_value="proj"
     )
     @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random")
     def test_lsf_mem_unit_is_kb_and_mem_mb_is_converted_accordingly(self, *mocks):
@@ -394,7 +387,7 @@ def test_lsf_mem_unit_is_kb_and_mem_mb_is_converted_accordingly(self, *mocks):
         ).format(outlog=expected_outlog, errlog=expected_errlog)
         expected = (
             "bsub -M {mem} -n 1 -R 'select[mem>{mem}] rusage[mem={mem}] span[hosts=1]' "
-            "{jobinfo} -q q1 cluster_opt_1 cluster_opt_2 cluster_opt_3 "
+            "{jobinfo} cluster_opt_1 cluster_opt_2 cluster_opt_3 "
             "-q queue -gpu - -P project "
             "real_jobscript.sh".format(mem=expected_mem, jobinfo=expected_jobinfo_cmd)
         )
@@ -405,10 +398,10 @@ def test_lsf_mem_unit_is_kb_and_mem_mb_is_converted_accordingly(self, *mocks):
         CookieCutter, CookieCutter.get_log_dir.__name__, return_value="logdir"
     )
     @patch.object(
-        CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000
+        CookieCutter, CookieCutter.get_default_project.__name__, return_value="proj"
     )
     @patch.object(
-        CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8
+        CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000
     )
     @patch.object(OSLayer, OSLayer.get_uuid4_string.__name__, return_value="random")
     def test_lsf_mem_unit_is_tb_and_mem_mb_is_converted_and_rounded_up_to_int(
@@ -450,7 +443,7 @@ def test_lsf_mem_unit_is_tb_and_mem_mb_is_converted_and_rounded_up_to_int(
         ).format(outlog=expected_outlog, errlog=expected_errlog)
         expected = (
             "bsub -M {mem} -n 1 -R 'select[mem>{mem}] rusage[mem={mem}] span[hosts=1]' "
-            "{jobinfo} -q q1 cluster_opt_1 cluster_opt_2 cluster_opt_3 "
+            "{jobinfo} cluster_opt_1 cluster_opt_2 cluster_opt_3 "
             "-q queue -gpu - -P project "
             "real_jobscript.sh".format(mem=expected_mem, jobinfo=expected_jobinfo_cmd)
         )
@@ -585,9 +578,6 @@ def test_jobname_for_group(self):
     @patch.object(
         CookieCutter, CookieCutter.get_default_mem_mb.__name__, return_value=1000
     )
-    @patch.object(
-        CookieCutter, CookieCutter.get_default_threads.__name__, return_value=8
-    )
     def test_time_resource_for_group(self, *mocks):
         for time_str in ("time", "runtime", "walltime", "time_min"):
             jobscript = Path(
@@ -602,7 +592,7 @@ def test_time_resource_for_group(self, *mocks):
 
             actual = lsf_submit.resources_cmd
             expected = (
-                "-M 1000 -n 8 -R 'select[mem>1000] rusage[mem=1000] "
+                "-M 1000 -n 1 -R 'select[mem>1000] rusage[mem=1000] "
                 "span[hosts=1]' -W 1"
             )
 

diff --git a/{{cookiecutter.profile_name}}/CookieCutter.py b/{{cookiecutter.profile_name}}/CookieCutter.py
@@ -3,10 +3,6 @@ class CookieCutter:
     Cookie Cutter wrapper
     """
 
-    @staticmethod
-    def get_default_threads() -> int:
-        return int("{{cookiecutter.default_threads}}")
-
     @staticmethod
     def get_default_mem_mb() -> int:
         return int("{{cookiecutter.default_mem_mb}}")
@@ -19,6 +15,10 @@ def get_log_dir() -> str:
     def get_default_queue() -> str:
         return "{{cookiecutter.default_queue}}"
 
+    @staticmethod
+    def get_default_project() -> str:
+        return "{{cookiecutter.default_project}}"
+
     @staticmethod
     def get_lsf_unit_for_limits() -> str:
         return "{{cookiecutter.LSF_UNIT_FOR_LIMITS}}"

diff --git a/{{cookiecutter.profile_name}}/lsf_submit.py b/{{cookiecutter.profile_name}}/lsf_submit.py
@@ -65,7 +65,7 @@ def cluster(self) -> dict:
 
     @property
     def threads(self) -> int:
-        return self.job_properties.get("threads", CookieCutter.get_default_threads())
+        return self.job_properties.get("threads", 1)
 
     @property
     def resources(self) -> dict:
@@ -165,6 +165,8 @@ def jobinfo_cmd(self) -> str:
 
     @property
     def queue(self) -> str:
+        if re.search(r"-q ", self.rule_specific_params):
+            return ""
         return self.cluster.get("queue", CookieCutter.get_default_queue())
 
     @property
@@ -175,6 +177,16 @@ def queue_cmd(self) -> str:
     def rule_specific_params(self) -> str:
         return self.lsf_config.params_for_rule(self.rule_name)
 
+    @property
+    def proj(self) -> str:
+        if re.search(r"-P ", self.rule_specific_params):
+            return ""
+        return self.cluster.get("project", CookieCutter.get_default_project())
+
+    @property
+    def proj_cmd(self) -> str:
+        return "-P {}".format(self.proj) if self.proj else ""
+
     @property
     def cluster_cmd(self) -> str:
         return self._cluster_cmd
@@ -186,6 +198,7 @@ def submit_cmd(self) -> str:
             self.resources_cmd,
             self.jobinfo_cmd,
             self.queue_cmd,
+            self.proj_cmd,
             self.cluster_cmd,
             self.rule_specific_params,
             self.jobscript,

diff --git a/{{cookiecutter.profile_name}}/memory_units.py b/{{cookiecutter.profile_name}}/memory_units.py
@@ -112,7 +112,7 @@ def _scaling_factor(self, decimal: bool = True) -> int:
 
     def bytes(self, decimal_multiples: bool = True) -> float:
         scaling_factor = self._scaling_factor(decimal_multiples)
-        return float(self.value * (scaling_factor ** self.power))
+        return float(self.value * (scaling_factor**self.power))
 
     def to(self, unit: Unit, decimal_multiples: bool = True) -> "Memory":
         scaling_factor = self._scaling_factor(decimal_multiples) ** unit.power