Merge pull request #244 from dres-dev/dev

Milestone 0.7

.github/workflows/release.yml

@@ -0,0 +1,66 @@
+name: Publish DRES Distribution and Docker Image
+
+on:
+  push:
+    tags:
+      - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+
+env:
+  IMAGE_NAME: dres
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11.0.6
+      - name: Cache Gradle packages
+        uses: actions/cache@v1
+        with:
+          path: ~/.gradle/caches
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
+      - name: Build distribution
+        run: ./gradlew distTar
+      - name: Get the version
+        id: get_version
+        run: echo "VERSION=${GITHUB_REF/refs\/tags\/v/}" >> $GITHUB_ENV
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.VERSION }}
+          release_name: Release ${{ env.VERSION }}
+          draft: false
+          prerelease: false
+      - name: Upload Release Asset
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+          asset_path: ./backend/build/distributions/dres-dist.tar
+          asset_name: dres-dist-${{ env.VERSION }}.tar
+          asset_content_type: application/tar
+      - name: Build Docker Image
+        run: docker build . --file Dockerfile --tag ${{ env.IMAGE_NAME }}
+      - name: Log into Docker Registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+      - name: Push Docker image
+        run: |
+          IMAGE_ID=docker.pkg.github.com/dres-dev/dres/${{ env.IMAGE_NAME }}
+
+          # Use Docker `latest` tag convention
+          [ "$VERSION" == "master" ] && VERSION=latest
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION

.gitignore

@@ -265,3 +265,8 @@ nodejs
 npm-*
 backend/ext/ffmpeg
 events/
+backend/data*
+backend/statistics
+
+## Do not include the generated client bindings
+frontend/openapi/

Dockerfile

@@ -0,0 +1,18 @@
+FROM zenika/kotlin:1.4.20-jdk11 AS build
+
+COPY . /dres-src
+RUN cd /dres-src && \
+  ./gradlew distTar && \
+  mkdir dres-dist && \
+  cd dres-dist && \
+  tar xf ../backend/build/distributions/dres-dist.tar
+
+FROM zenika/kotlin:1.4.20-jdk11-slim
+
+RUN mkdir /dres-data
+COPY config.json /dres-data
+COPY --from=build /dres-src/dres-dist /
+
+EXPOSE 8080
+EXPOSE 8443
+ENTRYPOINT /dres-dist/bin/backend /dres-data/config.json

README.md

@@ -1,6 +1,6 @@
 # DRES
 
-[![swagger-editor](https://img.shields.io/badge/open--API-in--editor-brightgreen.svg?style=flat&label=open-api-v3)](https://editor.swagger.io/?url=https://raw.githubusercontent.com/lucaro/DRES/master/doc/openapi-v3.json)
+[![swagger-editor](https://img.shields.io/badge/open--API-in--editor-brightgreen.svg?style=flat&label=client%20open-api-v3)](https://editor.swagger.io/?url=https://raw.githubusercontent.com/dres-dev/DRES/master/doc/oas-client.json)
 
 The Distributed Retrieval Evaluation Server builds uppon the work of https://github.com/klschoef/vbsserver/ to provide the means to evaluate interactive retrieval approaches in various settings, both on-site and distributed.
 

backend/build.gradle

@@ -1,8 +1,7 @@
 import org.apache.tools.ant.taskdefs.condition.Os
 
 buildscript {
-    ext.kotlinVersion = '1.4.21'
-
+    ext.kotlinVersion = '1.4.31'
     repositories {
         mavenCentral()
     }
@@ -12,20 +11,24 @@ buildscript {
         classpath "org.jetbrains.kotlin:kotlin-serialization:$kotlinVersion"
     }
 }
-plugins {
-    id 'com.github.node-gradle.node' version '2.2.0'
-    id 'de.undercouch.download' version '4.0.4'
-}
+
 apply plugin: 'application'
 apply plugin: 'java'
 apply plugin: 'kotlin'
 apply plugin: 'kotlinx-serialization'
 apply plugin: 'idea'
 
 mainClassName = 'dev.dres.DRES'
-
 sourceCompatibility = 1.8
 
+/* Configuration for frontend classpath files (see dependencies). */
+configurations {
+    frontendClasspath {
+        canBeConsumed = false
+        canBeResolved = true
+    }
+}
+
 repositories {
     mavenCentral()
     maven { url "https://kotlin.bintray.com/kotlinx" }
@@ -41,21 +44,24 @@ compileTestKotlin {
     kotlinOptions.jvmTarget = "1.8"
 }
 
-
 dependencies {
     def jettyVersion = '9.4.25.v20191220'
     def alpnApiVersion = '1.1.3.v20160715'
     def alpnBootVersion = '8.1.12.v20180117'
 
-    ///// MapDB
-    compile group: 'org.mapdb', name: 'mapdb', version: '3.0.8'
+    ///// Frontend files (produced by sub-project).
+    implementation frontendClasspath(project(path: ":frontend", configuration: 'frontendFiles'))
 
+    ///// FFMpeg dependency
+    implementation files("$buildDir/ext") {builtBy 'setupFFMpeg'}
 
-    ////// JLine 3
-    compile group: 'org.jline', name: 'jline', version: '3.15.0'
+    ///// MapDB
+    compile group: 'org.mapdb', name: 'mapdb', version: '3.0.8'
 
     ////// Javalin
-    compile group: 'io.javalin', name: 'javalin-bundle', version: '3.13.0'
+    compile ('io.javalin:javalin-bundle:3.13.0'){
+        exclude group: 'ch.qos.logback', module: 'logback-classic'
+    }
     compile group: 'io.swagger.core.v3', name: 'swagger-core', version: '2.1.5'
     compile group: 'org.webjars', name: 'swagger-ui', version: '3.24.3'
     compile group: 'org.webjars.npm', name: 'js-tokens', version: '3.0.2'
@@ -70,11 +76,13 @@ dependencies {
     ////// bcrypt
     compile group: 'org.mindrot', name: 'jbcrypt', version: '0.4'
 
-    ////// CLIKT
-    compile group: 'com.github.ajalt', name: 'clikt', version: '2.6.0'
+    ////// JLine 3, Clikt & Picnic for optimal terminal experience :-)
+    compile group: 'com.github.ajalt', name: 'clikt', version: '2.8.0'
+    compile group: 'org.jline', name: 'jline', version: '3.19.0'
+    compile group: 'com.jakewharton.picnic', name: 'picnic', version: '0.5.0'
 
-    ///// Picnic
-    compile group: 'com.jakewharton.picnic', name: 'picnic', version: '0.3.1'
+    ///// Fuel
+    compile group: 'com.github.kittinunf.fuel', name: 'fuel', version: '2.3.1'
 
     ////// CSV
     compile group: 'com.github.doyaaaaaken', name: 'kotlin-csv-jvm', version: '0.7.3'
@@ -92,7 +100,6 @@ dependencies {
     compile group: 'org.apache.logging.log4j', name: 'log4j-jul', version: '2.13.1'
 
     ///// JUnit 5
-
     testCompile "org.jetbrains.kotlin:kotlin-test"
     testCompile "org.jetbrains.kotlin:kotlin-test-junit5"
 
@@ -126,50 +133,13 @@ sourceSets {
 distributions {
     main {
         distributionBaseName = 'dres-dist'
-        contents {
-            from 'ext'
-        }
     }
 }
 
-node {
-    version = '12.18.3'
-    npmVersion = '6.14.6'
-
-    download = true
-    workDir = file("${project.projectDir}/../frontend/nodejs")
-    npmWorkDir = file("${project.projectDir}/../frontend")
-    nodeModulesDir = file("${project.projectDir}/../frontend")
-}
-
-/** Custom tasks: Front building (npm / node). */
-
-task cleanFrontendResources(type: Delete) {
-    delete 'src/main/resources/html'
-    doLast {
-        mkdir 'src/main/resources/html'
-        new File('src/main/resources/html/index.html').text = " "
-    }
-}
-
-task buildFrontend(type: NpmTask) {
-    args = ['run', 'pbuild']
-    dependsOn 'npm_install'
-}
-
-task deployFrontend(type: Copy) {
-    dependsOn 'cleanFrontendResources'
-    dependsOn 'buildFrontend'
-    tasks.findByName('buildFrontend').mustRunAfter 'cleanFrontendResources'
-    from('../frontend/dist/dres-frontend')
-    exclude '**/config.json'
-    into('src/main/resources/html')
-}
-
 /** Custom tasks: FFmpeg Download and deployment. */
 
 task downloadFFmpeg(type: Download) {
-    def f = new File('ffmpeg.zip')
+    def f = new File("$buildDir/cache/ffmpeg.zip")
     outputs.upToDateWhen {
         return f.exists()
     }
@@ -188,7 +158,7 @@ task downloadFFmpeg(type: Download) {
 }
 
 task downloadFFprobe(type: Download) {
-    def f = new File('ffprobe.zip')
+    def f = new File("$buildDir/cache/ffprobe.zip")
     outputs.upToDateWhen {
         return f.exists()
     }
@@ -209,20 +179,20 @@ task downloadFFprobe(type: Download) {
 task copyFFmpeg(type: Copy) {
     dependsOn downloadFFmpeg
     outputs.upToDateWhen {
-        return !fileTree('ext/ffmpeg').filter { it.isFile() && it.name.contains("ffmpeg")}.isEmpty()
+        return !fileTree("$buildDir/ext/ffmpeg").filter { it.isFile() && it.name.startsWith('ffmpeg')}.isEmpty()
     }
     from zipTree(downloadFFmpeg.dest)
-    into 'ext/ffmpeg'
+    into "$buildDir/ext/ffmpeg"
     include '*ffmpeg*'
 }
 
 task copyFFprobe(type: Copy) {
     dependsOn downloadFFprobe
     outputs.upToDateWhen {
-        return !fileTree('ext/ffmpeg').filter { it.isFile() && it.name.contains("ffprobe")}.isEmpty()
+        return !fileTree("$buildDir/ext/ffmpeg").filter { it.isFile() && it.name.startsWith('ffprobe') }.isEmpty()
     }
     from zipTree(downloadFFprobe.dest)
-    into 'ext/ffmpeg'
+    into "$buildDir/ext/ffmpeg"
     include '*ffprobe*'
 }
 
@@ -231,13 +201,4 @@ task setupFFMpeg(type: Copy) {
     dependsOn downloadFFprobe
     dependsOn copyFFmpeg
     dependsOn copyFFprobe
-}
-
-/** Jar task depends on frontend deployment. */
-jar.dependsOn deployFrontend
-
-/** Distribution tasks depend on Jar and setupFFMpeg. */
-distZip.dependsOn jar
-distZip.dependsOn setupFFMpeg
-distTar.dependsOn jar
-distTar.dependsOn setupFFMpeg
+}

backend/src/main/kotlin/dev/dres/DRES.kt

@@ -1,6 +1,7 @@
 package dev.dres
 
 import dev.dres.api.cli.Cli
+import dev.dres.api.cli.OpenApiCommand
 import dev.dres.api.rest.RestApi
 import dev.dres.data.dbo.DataAccessLayer
 import dev.dres.data.model.Config
@@ -17,6 +18,9 @@ import java.nio.file.Paths
 
 object DRES {
 
+    /** Application root; shoud pe relative to JAR file or classes path. */
+    val rootPath = File(FFmpegUtil::class.java.protectionDomain.codeSource.location.toURI()).toPath()
+
     init {
         //redirect log of JLine3 from jdk logger to log4j
         System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager")
@@ -30,7 +34,9 @@ object DRES {
             null
         } ?: Config()
 
-        println("initializing...")
+        println("Starting DRES at $rootPath")
+        println("Found FFmpeg at ${FFmpegUtil.ffmpegBin}")
+        println("Initializing...")
 
         /* Initialize data access layer. */
         val dataAccessLayer = DataAccessLayer(Paths.get(config.dataPath))
@@ -53,7 +59,12 @@ object DRES {
 
         println("done")
 
-        Cli.loop(dataAccessLayer, config) //blocks until quit command is given
+        if(args.isNotEmpty() && args.first() == "openapi"){
+            OpenApiCommand().parse(args)
+        }else{
+            Cli.loop(dataAccessLayer, config) //blocks until quit command is given
+        }
+
         RestApi.stop()
         RunExecutor.stop()
         EventStreamProcessor.stop()