Merge pull request #452 from dres-dev/dev

V2.0.0-RC2

README.md

@@ -84,8 +84,8 @@ The backend looks for a file named `config.json` at the location from where it i
 
 ## First Steps
 
-Using the DRES CLI, the `help` command lists available commands and their usage. If one would like to know more about a certain command, use the argument `-h`.
-Following the first steps towards a successful installment of a (distributed) retrieval evaluation campagn. A prerequisit is the previous deployment, see [Setup](#setup) and a running DRES instance.
+Using the DRES CLI, the `help` command lists available commands and their usage. If one would like to know more about a certain command `$cmd`, use the argument `-h`: `DRES> $cmd -h`
+Following the first steps towards a successful installment of a (distributed) retrieval evaluation campaign. A prerequisit is the previous deployment, see [Setup](#setup) and a running DRES instance.
 
 ### Create User
 
@@ -130,17 +130,18 @@ Then, navigate to _Evaluation Template Builder_ and create a new competition. Fo
 ### Create Competition Run
 
 An evaluation template serves as the template for one or more _evaluation runs_.
-Please keep in mind, that once a _run_ was created, changes on the template are not reflected in the run.
+Please keep in mind, that once a _run_ was created, changes on the template are not reflected in that run.
 
-Evaluation runs are created from the _Evaluations_ view, where one uses the "+" button to create a new one.
+Evaluation runs are created from the _Evaluation Template Overview_ view, where one uses the "Exit" (a running person) button to create a new one.
 
 In a non distributed setting, it might be desirable, that participants cannot view the actual run from the frontend,
 but require an external source for the query hints (e.g. a large monitor). This could be achieved by unchecking the corresponding option in the dialog.
+A run must be of either type `SYNCHRONOUS` or `ASYNCHRONOUS`. The former has task presentation and task execution syncrhonised for all participants and the latter enables task execution individually per participant.
 
 ### Runnig the evaluation
 
 As evaluation _operator_, one has to first start the run, then switch to a fitting task and ultimately start the task.
-Query hints are displayed as configured to all viewers, once they are all loaded (depending on the setup, this might take a breif moment).
+Query hints are displayed as configured to all viewers, once they are all loaded (depending on the setup, this might take a brief moment).
 Viewers and participants are shown the message "_Waiting for host to start task_". In case this seems to take too long,
 the operator can switch to the admin view and force all participants to be ready, by clicking the red ones.
 
@@ -153,6 +154,9 @@ It is recommended that all programmatic interaction with the DRES server is done
 
 **Notice:** We strongly recommend the usage of the [client OpenApi Specification](doc/oas-client.json) to generate the code to submit (and generally interact with the server)!
 
+**Notice:** With version 2.0.0, we provide a new POST submission endpoint which is more flexible than the legacy GET one.
+That is, for version 2.0.0 the legacy submission endpoint remains in the API version 1 and is deprecated. We highly encourage to move to the new POST-based endpoint.
+
 ---
 
 For legacy reasons, we provide further information below:

backend/src/main/kotlin/dev/dres/api/cli/EvaluationTemplateCommand.kt

@@ -58,7 +58,11 @@ class EvaluationTemplateCommand(private val cache: CacheManager) :
     /**
      * [CliktCommand] to create a new [DbEvaluationTemplate].
      */
-    inner class Create : CliktCommand(name = "create", help = "Creates a new Template") {
+    inner class Create : CliktCommand(
+        name = "create",
+        help = "Creates a new Template",
+        printHelpOnEmptyArgs = true
+    ) {
 
         private val name: String by option("-t", "--template", help = "Name of the new Template")
             .required()
@@ -77,7 +81,11 @@ class EvaluationTemplateCommand(private val cache: CacheManager) :
     /**
      * [CliktCommand] to delete a [DbEvaluationTemplate].
      */
-    inner class Delete : CliktCommand(name = "delete", help = "Deletes a template", printHelpOnEmptyArgs = true) {
+    inner class Delete : CliktCommand(
+        name = "delete",
+        help = "Deletes a template",
+        printHelpOnEmptyArgs = true
+    ) {
 
         private val id: String by option("-i", "--id").required()
             .validate { require(it.isNotEmpty()) { "Id must be non empty." } }
@@ -96,7 +104,11 @@ class EvaluationTemplateCommand(private val cache: CacheManager) :
     /**
      * [CliktCommand] to copy a [DbEvaluationTemplate].
      */
-    inner class Copy : CliktCommand(name = "copy", help = "Copies a Template", printHelpOnEmptyArgs = true) {
+    inner class Copy : CliktCommand(
+        name = "copy",
+        help = "Copies a Template",
+        printHelpOnEmptyArgs = true
+    ) {
 
         private val id: String by option("-i", "--id").required()
             .validate { require(it.isNotEmpty()) { "Id must be non empty." } }
@@ -116,7 +128,11 @@ class EvaluationTemplateCommand(private val cache: CacheManager) :
     /**
      * [CliktCommand] to rename a [DbEvaluationTemplate].
      */
-    inner class Rename : CliktCommand(name = "rename", help = "Renames a Template") {
+    inner class Rename : CliktCommand(
+        name = "rename",
+        help = "Renames a Template",
+        printHelpOnEmptyArgs = true
+    ) {
 
         private val id: String by option("-i", "--id").required()
             .validate { require(it.isNotEmpty()) { "Id must be non empty." } }
@@ -148,6 +164,7 @@ class EvaluationTemplateCommand(private val cache: CacheManager) :
     inner class List : CliktCommand(name = "list", help = "Lists an overview of all Templates") {
         override fun run() {
             var no = 0
+            println()
             println(table {
                 cellStyle {
                     border = true
@@ -202,7 +219,8 @@ class EvaluationTemplateCommand(private val cache: CacheManager) :
      */
     inner class Prepare : CliktCommand(
         name = "prepare",
-        help = "Checks the used media items and generates precomputed previews."
+        help = "Checks the used media items and generates precomputed previews.",
+        printHelpOnEmptyArgs = true
     ) {
 
         private val id: String by option("-i", "--id").required()
@@ -224,7 +242,11 @@ class EvaluationTemplateCommand(private val cache: CacheManager) :
     /**
      * Exports a specific evaluation to a JSON file.
      */
-    inner class Export : CliktCommand(name = "export", help = "Exports a template as JSON.") {
+    inner class Export : CliktCommand(
+        name = "export",
+        help = "Exports a template as JSON.",
+        printHelpOnEmptyArgs = true
+    ) {
 
         /** Path to the file that should be created .*/
         private val path: Path by option("-o", "--out", help = "The destination file for the template.").path()
@@ -266,7 +288,11 @@ class EvaluationTemplateCommand(private val cache: CacheManager) :
     /**
      * Imports a template from a JSON file.
      */
-    inner class Import : CliktCommand(name = "import", help = "Imports a template from JSON.") {
+    inner class Import : CliktCommand(
+        name = "import",
+        help = "Imports a template from JSON.",
+        printHelpOnEmptyArgs = true
+    ) {
 
         /** Path to the file that should be imported.*/
         private val path: Path by option("-i", "--in", help = "The file to import the template from.").path().required()

backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/client/ClientListEvaluationsHandler.kt

@@ -1,12 +1,9 @@
 package dev.dres.api.rest.handler.evaluation.client
 
 import dev.dres.api.rest.handler.GetRestHandler
-import dev.dres.api.rest.types.evaluation.ApiEvaluationType
-import dev.dres.api.rest.types.evaluation.ApiEvaluationInfo
+import dev.dres.api.rest.types.evaluation.ApiClientEvaluationInfo
 import dev.dres.api.rest.types.status.ErrorStatus
 import dev.dres.data.model.run.DbEvaluation
-import dev.dres.run.InteractiveAsynchronousRunManager
-import dev.dres.run.InteractiveSynchronousRunManager
 import io.javalin.http.Context
 import io.javalin.openapi.*
 
@@ -18,7 +15,7 @@ import io.javalin.openapi.*
  * @author Loris Sauter
  * @version 2.0.0
  */
-class ClientListEvaluationsHandler : AbstractEvaluationClientHandler(), GetRestHandler<List<ApiEvaluationInfo>> {
+class ClientListEvaluationsHandler : AbstractEvaluationClientHandler(), GetRestHandler<List<ApiClientEvaluationInfo>> {
 
     override val route = "client/evaluation/list"
 
@@ -28,17 +25,17 @@ class ClientListEvaluationsHandler : AbstractEvaluationClientHandler(), GetRestH
         operationId = OpenApiOperation.AUTO_GENERATE,
         tags = ["Evaluation Client"],
         responses = [
-            OpenApiResponse("200", [OpenApiContent(Array<ApiEvaluationInfo>::class)]),
+            OpenApiResponse("200", [OpenApiContent(Array<ApiClientEvaluationInfo>::class)]),
             OpenApiResponse("401", [OpenApiContent(ErrorStatus::class)])
         ],
         queryParams = [
             OpenApiParam("session", String::class, "Session Token")
         ],
         methods = [HttpMethod.GET]
     )
-    override fun doGet(ctx: Context): List<ApiEvaluationInfo> {
+    override fun doGet(ctx: Context): List<ApiClientEvaluationInfo> {
         return getRelevantManagers(ctx).map {
-            ApiEvaluationInfo(it)
+            ApiClientEvaluationInfo(it)
         }
     }
 }

backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/client/ClientTaskInfoHandler.kt

@@ -22,7 +22,7 @@ import jetbrains.exodus.database.TransientEntityStore
  */
 class ClientTaskInfoHandler : AbstractEvaluationClientHandler(),
     GetRestHandler<ApiTaskTemplateInfo> {
-    override val route = "client/evaluation/currentTask/{runId}"
+    override val route = "client/evaluation/currentTask/{evaluationId}"
 
     @OpenApi(
         summary = "Returns an overview of the currently active task for a run.",

backend/src/main/kotlin/dev/dres/api/rest/types/evaluation/ApiClientEvaluationInfo.kt

@@ -0,0 +1,38 @@
+package dev.dres.api.rest.types.evaluation
+
+import dev.dres.data.model.run.ApiRunProperties
+import dev.dres.run.InteractiveAsynchronousRunManager
+import dev.dres.run.InteractiveSynchronousRunManager
+import dev.dres.run.NonInteractiveRunManager
+import dev.dres.run.RunManager
+
+/**
+ * Contains the basic and most importantly static information about a [RunManager] that is relevant to a participant.
+ *
+ */
+data class ApiClientEvaluationInfo(
+    val id: String,
+    val name: String,
+    val type: ApiEvaluationType,
+    val status: ApiEvaluationStatus,
+    val templateId: String,
+    val templateDescription: String?,
+    val teams: List<String>,
+    val taskTemplates: List<ApiClientTaskTemplateInfo>,
+) {
+    constructor(manager: RunManager): this(
+        manager.id,
+        manager.name,
+        when(manager) {
+            is InteractiveSynchronousRunManager -> ApiEvaluationType.SYNCHRONOUS
+            is InteractiveAsynchronousRunManager -> ApiEvaluationType.ASYNCHRONOUS
+            is NonInteractiveRunManager -> ApiEvaluationType.NON_INTERACTIVE
+            else -> throw IllegalStateException("Incompatible type of run manager.")
+        },
+        manager.status.toApi(),
+        manager.template.id,
+        manager.template.description,
+        manager.template.teams.asSequence().map { team -> team.name ?: "" }.toList(),
+        manager.template.tasks.map { task -> ApiClientTaskTemplateInfo(task) }.toList()
+    )
+}

backend/src/main/kotlin/dev/dres/api/rest/types/evaluation/ApiClientTaskTemplateInfo.kt

@@ -0,0 +1,21 @@
+package dev.dres.api.rest.types.evaluation
+
+import dev.dres.api.rest.types.template.tasks.ApiTaskTemplate
+import dev.dres.data.model.template.task.DbTaskTemplate
+
+/**
+ * Basic and most importantly static information about a [DbTaskTemplate] relevant to the participant.
+ */
+data class ApiClientTaskTemplateInfo(
+    val name: String,
+    val taskGroup: String,
+    val taskType: String,
+    val duration: Long
+) {
+    constructor(task: ApiTaskTemplate) : this(
+        task.name,
+        task.taskGroup,
+        task.taskType,
+        task.duration
+    )
+}

backend/src/main/kotlin/dev/dres/mgmt/TemplateManager.kt

@@ -386,12 +386,16 @@ object TemplateManager {
                 }
             }
         }
+
+        println("start rendering ${segmentTasks.size} videos")
+
         await.all {
             try {
-                it.get(60, TimeUnit.SECONDS)
+                val result = it.get(3, TimeUnit.MINUTES)
+                println("completed rendering of $result")
                 true
             } catch (e: Throwable) {
-                throw IllegalStateException("Required media file could not be prepared.")
+                throw IllegalStateException("Required media file could not be prepared: ${e.message}")
             }
         }
 

doc/oas-client.json

@@ -2,8 +2,8 @@
   "openapi" : "3.0.3",
   "info" : {
     "title" : "DRES Client API",
-    "description" : "Client API for DRES (Distributed Retrieval Evaluation Server), Version 2.0.0",
-    "version" : "2.0.0"
+    "description" : "Client API for DRES (Distributed Retrieval Evaluation Server), Version 2.0.0-RC2",
+    "version" : "2.0.0-RC2"
   },
   "paths" : {
     "/api/v1/submit" : {
@@ -237,7 +237,7 @@
                 "schema" : {
                   "type" : "array",
                   "items" : {
-                    "$ref" : "#/components/schemas/ApiEvaluationInfo"
+                    "$ref" : "#/components/schemas/ApiClientEvaluationInfo"
                   }
                 }
               }
@@ -1228,6 +1228,63 @@
         "type" : "string",
         "enum" : [ "FRAME_NUMBER", "SECONDS", "MILLISECONDS", "TIMECODE" ]
       },
+      "ApiClientEvaluationInfo" : {
+        "type" : "object",
+        "additionalProperties" : false,
+        "properties" : {
+          "id" : {
+            "type" : "string"
+          },
+          "name" : {
+            "type" : "string"
+          },
+          "type" : {
+            "$ref" : "#/components/schemas/ApiEvaluationType"
+          },
+          "status" : {
+            "$ref" : "#/components/schemas/ApiEvaluationStatus"
+          },
+          "templateId" : {
+            "type" : "string"
+          },
+          "templateDescription" : {
+            "type" : "string"
+          },
+          "teams" : {
+            "type" : "array",
+            "items" : {
+              "type" : "string"
+            }
+          },
+          "taskTemplates" : {
+            "type" : "array",
+            "items" : {
+              "$ref" : "#/components/schemas/ApiClientTaskTemplateInfo"
+            }
+          }
+        },
+        "required" : [ "id", "name", "type", "status", "templateId", "teams", "taskTemplates" ]
+      },
+      "ApiClientTaskTemplateInfo" : {
+        "type" : "object",
+        "additionalProperties" : false,
+        "properties" : {
+          "name" : {
+            "type" : "string"
+          },
+          "taskGroup" : {
+            "type" : "string"
+          },
+          "taskType" : {
+            "type" : "string"
+          },
+          "duration" : {
+            "type" : "integer",
+            "format" : "int64"
+          }
+        },
+        "required" : [ "name", "taskGroup", "taskType", "duration" ]
+      },
       "ApiEvaluation" : {
         "type" : "object",
         "additionalProperties" : false,