From 25edbb64d12ba541d58e821c21f374013cc1ec76 Mon Sep 17 00:00:00 2001 From: Athitheya Gobinathan Date: Fri, 21 Jun 2024 14:47:55 -0400 Subject: [PATCH 1/8] feat: add ignore_error to TesExecutor --- pro_tes/ga4gh/tes/models.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pro_tes/ga4gh/tes/models.py b/pro_tes/ga4gh/tes/models.py index 2f0b039..bea8960 100644 --- a/pro_tes/ga4gh/tes/models.py +++ b/pro_tes/ga4gh/tes/models.py @@ -111,6 +111,14 @@ class TesExecutor(CustomBaseModel): ), examples=[{"BLASTDB": "/data/GRC38", "HMMERDB": "/data/hmmer"}], ) + ignore_error: Optional[bool] = Field( + None, + description=( + "Default behavior of running an array of executors is that execution stops" + " on the first error. If `ignore_error` is `True`, then the runner will" + " record error exit codes, but will continue on to the next tesExecutor." + ), + ) class TesExecutorLog(CustomBaseModel): From c04cbab851202f4cc9752561703985880148e526 Mon Sep 17 00:00:00 2001 From: Athitheya Gobinathan Date: Fri, 21 Jun 2024 14:54:12 -0400 Subject: [PATCH 2/8] feat: add backend_parameters and backend_parameters_strict to TesResources --- pro_tes/ga4gh/tes/models.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/pro_tes/ga4gh/tes/models.py b/pro_tes/ga4gh/tes/models.py index bea8960..4131d87 100644 --- a/pro_tes/ga4gh/tes/models.py +++ b/pro_tes/ga4gh/tes/models.py @@ -112,7 +112,7 @@ class TesExecutor(CustomBaseModel): examples=[{"BLASTDB": "/data/GRC38", "HMMERDB": "/data/hmmer"}], ) ignore_error: Optional[bool] = Field( - None, + default=None, description=( "Default behavior of running an array of executors is that execution stops" " on the first error. If `ignore_error` is `True`, then the runner will" @@ -292,6 +292,33 @@ class TesResources(CustomBaseModel): ), examples=["us-west-1"], ) + backend_parameters: Optional[dict[str, str]] = Field( + default=None, + description=( + "Key/value pairs for backend configuration.ServiceInfo shall return a" + " list of keys that a backend supports. Keys are case insensitive.It is" + " expected that clients pass all runtime or hardware requirement" + " key/values that are not mapped to existing tesResources properties to" + " backend_parameters. Backends shall log system warnings if a key is" + " passed that is unsupported. Backends shall not store or return" + " unsupported keys if included in a task. If backend_parameters_strict" + " equals true, backends should fail the task if any key/values are" + " unsupported, otherwise, backends should attempt to run the" + " task Intended uses include VM size selection, coprocessor configuration," + ' etc. \nExample: ```\n{\n "backend_parameters" : {\n "VmSize" :' + ' "Standard_D64_v3"\n }\n}\n```' + ), + example={"VmSize": "Standard_D64_v3"}, + ) + backend_parameters_strict: Optional[bool] = Field( + default=False, + description=( + "If set to true, backends should fail the task if any" + " backend_parameters key/values are unsupported, otherwise, backends" + " should attempt to run the task" + ), + example=False, + ) class Artifact(Enum): From edb2f75404afa52726abbf6406e9e2bfdbd54780 Mon Sep 17 00:00:00 2001 From: Athitheya Gobinathan Date: Fri, 21 Jun 2024 14:55:15 -0400 Subject: [PATCH 3/8] feat: add new tesStates --- pro_tes/ga4gh/tes/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pro_tes/ga4gh/tes/models.py b/pro_tes/ga4gh/tes/models.py index 4131d87..171bf4c 100644 --- a/pro_tes/ga4gh/tes/models.py +++ b/pro_tes/ga4gh/tes/models.py @@ -469,6 +469,8 @@ class TesState(Enum): EXECUTOR_ERROR = "EXECUTOR_ERROR" SYSTEM_ERROR = "SYSTEM_ERROR" CANCELED = "CANCELED" + PREEMPTED = "PREEMPTED" + CANCELING = "CANCELING" class TesNextTes(CustomBaseModel): From d5e059cdd96f1ee065379bce9e91ae3a75263643 Mon Sep 17 00:00:00 2001 From: Athitheya Gobinathan Date: Fri, 21 Jun 2024 16:00:52 -0400 Subject: [PATCH 4/8] refactor: make TesFileType optional --- pro_tes/ga4gh/tes/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pro_tes/ga4gh/tes/models.py b/pro_tes/ga4gh/tes/models.py index 171bf4c..0c1c29c 100644 --- a/pro_tes/ga4gh/tes/models.py +++ b/pro_tes/ga4gh/tes/models.py @@ -189,7 +189,7 @@ class TesInput(CustomBaseModel): ), examples=["/data/file1"], ) - type: TesFileType + type: Optional[TesFileType] = "FILE" content: Optional[str] = Field( default=None, description=( @@ -228,7 +228,7 @@ class TesOutput(CustomBaseModel): " absolute path." ), ) - type: TesFileType + type: Optional[TesFileType] = "FILE" class TesOutputFileLog(CustomBaseModel): From 24c511e93fc313893ff72ca4fde6178a6443c788 Mon Sep 17 00:00:00 2001 From: Athitheya Gobinathan Date: Fri, 21 Jun 2024 16:05:39 -0400 Subject: [PATCH 5/8] feat: add streamable to TesInput --- pro_tes/ga4gh/tes/models.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pro_tes/ga4gh/tes/models.py b/pro_tes/ga4gh/tes/models.py index 0c1c29c..629dda1 100644 --- a/pro_tes/ga4gh/tes/models.py +++ b/pro_tes/ga4gh/tes/models.py @@ -199,6 +199,19 @@ class TesInput(CustomBaseModel): ' "url" must be ignored.' ), ) + streamable: Optional[bool] = Field( + default=None, + description=( + "Indicate that a file resource could be accessed using a" + " streaming interface, ie a FUSE mounted s3 object. This flag indicates" + " that using a streaming mount, as opposed to downloading the whole file" + " to the local scratch space, may be faster despite the latency" + " and overhead. This does not mean that the backend will use a" + " streaming interface, as it may not be provided by the vendor, but if" + " the capacity is avalible it can be used without degrading" + " the performance of the underlying program." + ), + ) class TesOutput(CustomBaseModel): From 8ae101a6474b8286057b8c97fb50a1f1b40bbd5b Mon Sep 17 00:00:00 2001 From: Athitheya Gobinathan Date: Tue, 9 Jul 2024 11:10:29 -0400 Subject: [PATCH 6/8] refactor: avoid string literal --- pro_tes/ga4gh/tes/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pro_tes/ga4gh/tes/models.py b/pro_tes/ga4gh/tes/models.py index 629dda1..1085259 100644 --- a/pro_tes/ga4gh/tes/models.py +++ b/pro_tes/ga4gh/tes/models.py @@ -189,7 +189,7 @@ class TesInput(CustomBaseModel): ), examples=["/data/file1"], ) - type: Optional[TesFileType] = "FILE" + type: Optional[TesFileType] = TesFileType.FILE content: Optional[str] = Field( default=None, description=( From 23c3c6bf3e1cb364e497c8c2c929d0f25c86e73f Mon Sep 17 00:00:00 2001 From: Athitheya Gobinathan Date: Tue, 9 Jul 2024 11:19:29 -0400 Subject: [PATCH 7/8] fix: shorten doc line lengths --- pro_tes/ga4gh/tes/models.py | 51 ++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/pro_tes/ga4gh/tes/models.py b/pro_tes/ga4gh/tes/models.py index 1085259..67b402e 100644 --- a/pro_tes/ga4gh/tes/models.py +++ b/pro_tes/ga4gh/tes/models.py @@ -114,9 +114,10 @@ class TesExecutor(CustomBaseModel): ignore_error: Optional[bool] = Field( default=None, description=( - "Default behavior of running an array of executors is that execution stops" - " on the first error. If `ignore_error` is `True`, then the runner will" - " record error exit codes, but will continue on to the next tesExecutor." + "Default behavior of running an array of executors is that " + "execution stopson the first error. If `ignore_error` is `True`, " + "then the runner will record error exit codes, but will continue " + "on to the next tesExecutor." ), ) @@ -203,13 +204,14 @@ class TesInput(CustomBaseModel): default=None, description=( "Indicate that a file resource could be accessed using a" - " streaming interface, ie a FUSE mounted s3 object. This flag indicates" - " that using a streaming mount, as opposed to downloading the whole file" - " to the local scratch space, may be faster despite the latency" - " and overhead. This does not mean that the backend will use a" - " streaming interface, as it may not be provided by the vendor, but if" - " the capacity is avalible it can be used without degrading" - " the performance of the underlying program." + " streaming interface, ie a FUSE mounted s3 object. This flag" + " indicates that using a streaming mount, as opposed to " + "downloading the whole file to the local scratch space, may be " + "faster despite the latency and overhead. This does not mean that" + " the backend will use a streaming interface, as it may not be " + "provided by the vendor, but if the capacity is avalible it can " + " be used without degrading the performance of the underlying" + " program." ), ) @@ -308,18 +310,19 @@ class TesResources(CustomBaseModel): backend_parameters: Optional[dict[str, str]] = Field( default=None, description=( - "Key/value pairs for backend configuration.ServiceInfo shall return a" - " list of keys that a backend supports. Keys are case insensitive.It is" - " expected that clients pass all runtime or hardware requirement" - " key/values that are not mapped to existing tesResources properties to" - " backend_parameters. Backends shall log system warnings if a key is" - " passed that is unsupported. Backends shall not store or return" - " unsupported keys if included in a task. If backend_parameters_strict" - " equals true, backends should fail the task if any key/values are" - " unsupported, otherwise, backends should attempt to run the" - " task Intended uses include VM size selection, coprocessor configuration," - ' etc. \nExample: ```\n{\n "backend_parameters" : {\n "VmSize" :' - ' "Standard_D64_v3"\n }\n}\n```' + "Key/value pairs for backend configuration.ServiceInfo shall " + "return a list of keys that a backend supports. Keys are case " + "insensitive. It is expected that clients pass all runtime or " + "hardware requirement key/values that are not mapped to existing" + " tesResources properties to backend_parameters. Backends shall" + " log system warnings if a key is passed that is unsupported. " + "Backends shall not store or return unsupported keys if included " + "in a task. If backend_parameters_strict equals true, backends " + "should fail the task if any key/values are unsupported, " + " otherwise, backends should attempt to run the task Intended " + "uses include VM size selection, coprocessor configuration," + ' etc. \nExample: ```\n{\n "backend_parameters" : {\n ' + '"VmSize" :"Standard_D64_v3"\n }\n}\n```' ), example={"VmSize": "Standard_D64_v3"}, ) @@ -327,8 +330,8 @@ class TesResources(CustomBaseModel): default=False, description=( "If set to true, backends should fail the task if any" - " backend_parameters key/values are unsupported, otherwise, backends" - " should attempt to run the task" + " backend_parameters key/values are unsupported, otherwise, " + "backends should attempt to run the task" ), example=False, ) From 64248f2c6a5101584fbdd774fb329fa02ba3a01d Mon Sep 17 00:00:00 2001 From: Athitheya Gobinathan Date: Tue, 9 Jul 2024 11:24:24 -0400 Subject: [PATCH 8/8] fix: change example to examples --- pro_tes/ga4gh/tes/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pro_tes/ga4gh/tes/models.py b/pro_tes/ga4gh/tes/models.py index 67b402e..a9bdb2c 100644 --- a/pro_tes/ga4gh/tes/models.py +++ b/pro_tes/ga4gh/tes/models.py @@ -243,7 +243,7 @@ class TesOutput(CustomBaseModel): " absolute path." ), ) - type: Optional[TesFileType] = "FILE" + type: Optional[TesFileType] = TesFileType.FILE class TesOutputFileLog(CustomBaseModel): @@ -324,7 +324,7 @@ class TesResources(CustomBaseModel): ' etc. \nExample: ```\n{\n "backend_parameters" : {\n ' '"VmSize" :"Standard_D64_v3"\n }\n}\n```' ), - example={"VmSize": "Standard_D64_v3"}, + examples=[{"VmSize": "Standard_D64_v3"}], ) backend_parameters_strict: Optional[bool] = Field( default=False, @@ -333,7 +333,7 @@ class TesResources(CustomBaseModel): " backend_parameters key/values are unsupported, otherwise, " "backends should attempt to run the task" ), - example=False, + examples=[False], )