diff --git a/README.md b/README.md index ae9baa40..37196aed 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,8 @@ The backend looks for a file named `config.json` at the location from where it i ## First Steps -Using the DRES CLI, the `help` command lists available commands and their usage. If one would like to know more about a certain command, use the argument `-h`. -Following the first steps towards a successful installment of a (distributed) retrieval evaluation campagn. A prerequisit is the previous deployment, see [Setup](#setup) and a running DRES instance. +Using the DRES CLI, the `help` command lists available commands and their usage. If one would like to know more about a certain command `$cmd`, use the argument `-h`: `DRES> $cmd -h` +Following the first steps towards a successful installment of a (distributed) retrieval evaluation campaign. A prerequisit is the previous deployment, see [Setup](#setup) and a running DRES instance. ### Create User @@ -130,17 +130,18 @@ Then, navigate to _Evaluation Template Builder_ and create a new competition. Fo ### Create Competition Run An evaluation template serves as the template for one or more _evaluation runs_. -Please keep in mind, that once a _run_ was created, changes on the template are not reflected in the run. +Please keep in mind, that once a _run_ was created, changes on the template are not reflected in that run. -Evaluation runs are created from the _Evaluations_ view, where one uses the "+" button to create a new one. +Evaluation runs are created from the _Evaluation Template Overview_ view, where one uses the "Exit" (a running person) button to create a new one. In a non distributed setting, it might be desirable, that participants cannot view the actual run from the frontend, but require an external source for the query hints (e.g. a large monitor). This could be achieved by unchecking the corresponding option in the dialog. +A run must be of either type `SYNCHRONOUS` or `ASYNCHRONOUS`. The former has task presentation and task execution syncrhonised for all participants and the latter enables task execution individually per participant. ### Runnig the evaluation As evaluation _operator_, one has to first start the run, then switch to a fitting task and ultimately start the task. -Query hints are displayed as configured to all viewers, once they are all loaded (depending on the setup, this might take a breif moment). +Query hints are displayed as configured to all viewers, once they are all loaded (depending on the setup, this might take a brief moment). Viewers and participants are shown the message "_Waiting for host to start task_". In case this seems to take too long, the operator can switch to the admin view and force all participants to be ready, by clicking the red ones. @@ -153,6 +154,9 @@ It is recommended that all programmatic interaction with the DRES server is done **Notice:** We strongly recommend the usage of the [client OpenApi Specification](doc/oas-client.json) to generate the code to submit (and generally interact with the server)! +**Notice:** With version 2.0.0, we provide a new POST submission endpoint which is more flexible than the legacy GET one. +That is, for version 2.0.0 the legacy submission endpoint remains in the API version 1 and is deprecated. We highly encourage to move to the new POST-based endpoint. + --- For legacy reasons, we provide further information below: diff --git a/backend/src/main/kotlin/dev/dres/api/cli/EvaluationTemplateCommand.kt b/backend/src/main/kotlin/dev/dres/api/cli/EvaluationTemplateCommand.kt index f8e56f85..749726d2 100644 --- a/backend/src/main/kotlin/dev/dres/api/cli/EvaluationTemplateCommand.kt +++ b/backend/src/main/kotlin/dev/dres/api/cli/EvaluationTemplateCommand.kt @@ -58,7 +58,11 @@ class EvaluationTemplateCommand(private val cache: CacheManager) : /** * [CliktCommand] to create a new [DbEvaluationTemplate]. */ - inner class Create : CliktCommand(name = "create", help = "Creates a new Template") { + inner class Create : CliktCommand( + name = "create", + help = "Creates a new Template", + printHelpOnEmptyArgs = true + ) { private val name: String by option("-t", "--template", help = "Name of the new Template") .required() @@ -77,7 +81,11 @@ class EvaluationTemplateCommand(private val cache: CacheManager) : /** * [CliktCommand] to delete a [DbEvaluationTemplate]. */ - inner class Delete : CliktCommand(name = "delete", help = "Deletes a template", printHelpOnEmptyArgs = true) { + inner class Delete : CliktCommand( + name = "delete", + help = "Deletes a template", + printHelpOnEmptyArgs = true + ) { private val id: String by option("-i", "--id").required() .validate { require(it.isNotEmpty()) { "Id must be non empty." } } @@ -96,7 +104,11 @@ class EvaluationTemplateCommand(private val cache: CacheManager) : /** * [CliktCommand] to copy a [DbEvaluationTemplate]. */ - inner class Copy : CliktCommand(name = "copy", help = "Copies a Template", printHelpOnEmptyArgs = true) { + inner class Copy : CliktCommand( + name = "copy", + help = "Copies a Template", + printHelpOnEmptyArgs = true + ) { private val id: String by option("-i", "--id").required() .validate { require(it.isNotEmpty()) { "Id must be non empty." } } @@ -116,7 +128,11 @@ class EvaluationTemplateCommand(private val cache: CacheManager) : /** * [CliktCommand] to rename a [DbEvaluationTemplate]. */ - inner class Rename : CliktCommand(name = "rename", help = "Renames a Template") { + inner class Rename : CliktCommand( + name = "rename", + help = "Renames a Template", + printHelpOnEmptyArgs = true + ) { private val id: String by option("-i", "--id").required() .validate { require(it.isNotEmpty()) { "Id must be non empty." } } @@ -148,6 +164,7 @@ class EvaluationTemplateCommand(private val cache: CacheManager) : inner class List : CliktCommand(name = "list", help = "Lists an overview of all Templates") { override fun run() { var no = 0 + println() println(table { cellStyle { border = true @@ -202,7 +219,8 @@ class EvaluationTemplateCommand(private val cache: CacheManager) : */ inner class Prepare : CliktCommand( name = "prepare", - help = "Checks the used media items and generates precomputed previews." + help = "Checks the used media items and generates precomputed previews.", + printHelpOnEmptyArgs = true ) { private val id: String by option("-i", "--id").required() @@ -224,7 +242,11 @@ class EvaluationTemplateCommand(private val cache: CacheManager) : /** * Exports a specific evaluation to a JSON file. */ - inner class Export : CliktCommand(name = "export", help = "Exports a template as JSON.") { + inner class Export : CliktCommand( + name = "export", + help = "Exports a template as JSON.", + printHelpOnEmptyArgs = true + ) { /** Path to the file that should be created .*/ private val path: Path by option("-o", "--out", help = "The destination file for the template.").path() @@ -266,7 +288,11 @@ class EvaluationTemplateCommand(private val cache: CacheManager) : /** * Imports a template from a JSON file. */ - inner class Import : CliktCommand(name = "import", help = "Imports a template from JSON.") { + inner class Import : CliktCommand( + name = "import", + help = "Imports a template from JSON.", + printHelpOnEmptyArgs = true + ) { /** Path to the file that should be imported.*/ private val path: Path by option("-i", "--in", help = "The file to import the template from.").path().required() diff --git a/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/client/ClientListEvaluationsHandler.kt b/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/client/ClientListEvaluationsHandler.kt index f0679b73..6b7a990a 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/client/ClientListEvaluationsHandler.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/client/ClientListEvaluationsHandler.kt @@ -1,12 +1,9 @@ package dev.dres.api.rest.handler.evaluation.client import dev.dres.api.rest.handler.GetRestHandler -import dev.dres.api.rest.types.evaluation.ApiEvaluationType -import dev.dres.api.rest.types.evaluation.ApiEvaluationInfo +import dev.dres.api.rest.types.evaluation.ApiClientEvaluationInfo import dev.dres.api.rest.types.status.ErrorStatus import dev.dres.data.model.run.DbEvaluation -import dev.dres.run.InteractiveAsynchronousRunManager -import dev.dres.run.InteractiveSynchronousRunManager import io.javalin.http.Context import io.javalin.openapi.* @@ -18,7 +15,7 @@ import io.javalin.openapi.* * @author Loris Sauter * @version 2.0.0 */ -class ClientListEvaluationsHandler : AbstractEvaluationClientHandler(), GetRestHandler> { +class ClientListEvaluationsHandler : AbstractEvaluationClientHandler(), GetRestHandler> { override val route = "client/evaluation/list" @@ -28,7 +25,7 @@ class ClientListEvaluationsHandler : AbstractEvaluationClientHandler(), GetRestH operationId = OpenApiOperation.AUTO_GENERATE, tags = ["Evaluation Client"], responses = [ - OpenApiResponse("200", [OpenApiContent(Array::class)]), + OpenApiResponse("200", [OpenApiContent(Array::class)]), OpenApiResponse("401", [OpenApiContent(ErrorStatus::class)]) ], queryParams = [ @@ -36,9 +33,9 @@ class ClientListEvaluationsHandler : AbstractEvaluationClientHandler(), GetRestH ], methods = [HttpMethod.GET] ) - override fun doGet(ctx: Context): List { + override fun doGet(ctx: Context): List { return getRelevantManagers(ctx).map { - ApiEvaluationInfo(it) + ApiClientEvaluationInfo(it) } } } diff --git a/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/client/ClientTaskInfoHandler.kt b/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/client/ClientTaskInfoHandler.kt index 66b1cee5..441a87f8 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/client/ClientTaskInfoHandler.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/client/ClientTaskInfoHandler.kt @@ -22,7 +22,7 @@ import jetbrains.exodus.database.TransientEntityStore */ class ClientTaskInfoHandler : AbstractEvaluationClientHandler(), GetRestHandler { - override val route = "client/evaluation/currentTask/{runId}" + override val route = "client/evaluation/currentTask/{evaluationId}" @OpenApi( summary = "Returns an overview of the currently active task for a run.", diff --git a/backend/src/main/kotlin/dev/dres/api/rest/types/evaluation/ApiClientEvaluationInfo.kt b/backend/src/main/kotlin/dev/dres/api/rest/types/evaluation/ApiClientEvaluationInfo.kt new file mode 100644 index 00000000..4426b343 --- /dev/null +++ b/backend/src/main/kotlin/dev/dres/api/rest/types/evaluation/ApiClientEvaluationInfo.kt @@ -0,0 +1,38 @@ +package dev.dres.api.rest.types.evaluation + +import dev.dres.data.model.run.ApiRunProperties +import dev.dres.run.InteractiveAsynchronousRunManager +import dev.dres.run.InteractiveSynchronousRunManager +import dev.dres.run.NonInteractiveRunManager +import dev.dres.run.RunManager + +/** + * Contains the basic and most importantly static information about a [RunManager] that is relevant to a participant. + * + */ +data class ApiClientEvaluationInfo( + val id: String, + val name: String, + val type: ApiEvaluationType, + val status: ApiEvaluationStatus, + val templateId: String, + val templateDescription: String?, + val teams: List, + val taskTemplates: List, +) { + constructor(manager: RunManager): this( + manager.id, + manager.name, + when(manager) { + is InteractiveSynchronousRunManager -> ApiEvaluationType.SYNCHRONOUS + is InteractiveAsynchronousRunManager -> ApiEvaluationType.ASYNCHRONOUS + is NonInteractiveRunManager -> ApiEvaluationType.NON_INTERACTIVE + else -> throw IllegalStateException("Incompatible type of run manager.") + }, + manager.status.toApi(), + manager.template.id, + manager.template.description, + manager.template.teams.asSequence().map { team -> team.name ?: "" }.toList(), + manager.template.tasks.map { task -> ApiClientTaskTemplateInfo(task) }.toList() + ) +} diff --git a/backend/src/main/kotlin/dev/dres/api/rest/types/evaluation/ApiClientTaskTemplateInfo.kt b/backend/src/main/kotlin/dev/dres/api/rest/types/evaluation/ApiClientTaskTemplateInfo.kt new file mode 100644 index 00000000..7a042a68 --- /dev/null +++ b/backend/src/main/kotlin/dev/dres/api/rest/types/evaluation/ApiClientTaskTemplateInfo.kt @@ -0,0 +1,21 @@ +package dev.dres.api.rest.types.evaluation + +import dev.dres.api.rest.types.template.tasks.ApiTaskTemplate +import dev.dres.data.model.template.task.DbTaskTemplate + +/** + * Basic and most importantly static information about a [DbTaskTemplate] relevant to the participant. + */ +data class ApiClientTaskTemplateInfo( + val name: String, + val taskGroup: String, + val taskType: String, + val duration: Long +) { + constructor(task: ApiTaskTemplate) : this( + task.name, + task.taskGroup, + task.taskType, + task.duration + ) +} diff --git a/backend/src/main/kotlin/dev/dres/mgmt/TemplateManager.kt b/backend/src/main/kotlin/dev/dres/mgmt/TemplateManager.kt index c63b4e3e..c8687291 100644 --- a/backend/src/main/kotlin/dev/dres/mgmt/TemplateManager.kt +++ b/backend/src/main/kotlin/dev/dres/mgmt/TemplateManager.kt @@ -386,12 +386,16 @@ object TemplateManager { } } } + + println("start rendering ${segmentTasks.size} videos") + await.all { try { - it.get(60, TimeUnit.SECONDS) + val result = it.get(3, TimeUnit.MINUTES) + println("completed rendering of $result") true } catch (e: Throwable) { - throw IllegalStateException("Required media file could not be prepared.") + throw IllegalStateException("Required media file could not be prepared: ${e.message}") } } diff --git a/doc/oas-client.json b/doc/oas-client.json index 820ca702..4926aa71 100644 --- a/doc/oas-client.json +++ b/doc/oas-client.json @@ -2,8 +2,8 @@ "openapi" : "3.0.3", "info" : { "title" : "DRES Client API", - "description" : "Client API for DRES (Distributed Retrieval Evaluation Server), Version 2.0.0", - "version" : "2.0.0" + "description" : "Client API for DRES (Distributed Retrieval Evaluation Server), Version 2.0.0-RC2", + "version" : "2.0.0-RC2" }, "paths" : { "/api/v1/submit" : { @@ -237,7 +237,7 @@ "schema" : { "type" : "array", "items" : { - "$ref" : "#/components/schemas/ApiEvaluationInfo" + "$ref" : "#/components/schemas/ApiClientEvaluationInfo" } } } @@ -1228,6 +1228,63 @@ "type" : "string", "enum" : [ "FRAME_NUMBER", "SECONDS", "MILLISECONDS", "TIMECODE" ] }, + "ApiClientEvaluationInfo" : { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "id" : { + "type" : "string" + }, + "name" : { + "type" : "string" + }, + "type" : { + "$ref" : "#/components/schemas/ApiEvaluationType" + }, + "status" : { + "$ref" : "#/components/schemas/ApiEvaluationStatus" + }, + "templateId" : { + "type" : "string" + }, + "templateDescription" : { + "type" : "string" + }, + "teams" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "taskTemplates" : { + "type" : "array", + "items" : { + "$ref" : "#/components/schemas/ApiClientTaskTemplateInfo" + } + } + }, + "required" : [ "id", "name", "type", "status", "templateId", "teams", "taskTemplates" ] + }, + "ApiClientTaskTemplateInfo" : { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "name" : { + "type" : "string" + }, + "taskGroup" : { + "type" : "string" + }, + "taskType" : { + "type" : "string" + }, + "duration" : { + "type" : "integer", + "format" : "int64" + } + }, + "required" : [ "name", "taskGroup", "taskType", "duration" ] + }, "ApiEvaluation" : { "type" : "object", "additionalProperties" : false, diff --git a/doc/oas.json b/doc/oas.json index 54da7718..ca844ba6 100644 --- a/doc/oas.json +++ b/doc/oas.json @@ -2,7 +2,7 @@ "openapi" : "3.0.3", "info" : { "title" : "DRES API", - "description" : "API for DRES (Distributed Retrieval Evaluation Server), Version 2.0.0", + "description" : "API for DRES (Distributed Retrieval Evaluation Server), Version 2.0.0-RC2", "contact" : { "name" : "The DRES Dev Team", "url" : "https://dres.dev" @@ -10,7 +10,7 @@ "license" : { "name" : "MIT" }, - "version" : "2.0.0" + "version" : "2.0.0-RC2" }, "paths" : { "/api/v1/submit" : { @@ -244,7 +244,7 @@ "schema" : { "type" : "array", "items" : { - "$ref" : "#/components/schemas/ApiEvaluationInfo" + "$ref" : "#/components/schemas/ApiClientEvaluationInfo" } } } @@ -5642,6 +5642,63 @@ "type" : "string", "enum" : [ "FRAME_NUMBER", "SECONDS", "MILLISECONDS", "TIMECODE" ] }, + "ApiClientEvaluationInfo" : { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "id" : { + "type" : "string" + }, + "name" : { + "type" : "string" + }, + "type" : { + "$ref" : "#/components/schemas/ApiEvaluationType" + }, + "status" : { + "$ref" : "#/components/schemas/ApiEvaluationStatus" + }, + "templateId" : { + "type" : "string" + }, + "templateDescription" : { + "type" : "string" + }, + "teams" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "taskTemplates" : { + "type" : "array", + "items" : { + "$ref" : "#/components/schemas/ApiClientTaskTemplateInfo" + } + } + }, + "required" : [ "id", "name", "type", "status", "templateId", "teams", "taskTemplates" ] + }, + "ApiClientTaskTemplateInfo" : { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "name" : { + "type" : "string" + }, + "taskGroup" : { + "type" : "string" + }, + "taskType" : { + "type" : "string" + }, + "duration" : { + "type" : "integer", + "format" : "int64" + } + }, + "required" : [ "name", "taskGroup", "taskType", "duration" ] + }, "ApiEvaluation" : { "type" : "object", "additionalProperties" : false, diff --git a/frontend/src/app/template/template-builder/components/task-template-editor/task-template-editor.component.ts b/frontend/src/app/template/template-builder/components/task-template-editor/task-template-editor.component.ts index 89a57209..79112a57 100644 --- a/frontend/src/app/template/template-builder/components/task-template-editor/task-template-editor.component.ts +++ b/frontend/src/app/template/template-builder/components/task-template-editor/task-template-editor.component.ts @@ -106,7 +106,7 @@ export class TaskTemplateEditorComponent implements OnInit, OnDestroy { } public isFormValid(){ - return this.form.valid; + return this.form == undefined || this.form.valid; } public fetchData(){