From e31051b3cd68a47ae26109c5fc300bc66c3ff3e5 Mon Sep 17 00:00:00 2001 From: Bastiaan Date: Tue, 19 Sep 2023 11:40:20 +0200 Subject: [PATCH 01/17] Java 17 instead of 11 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c50d117c..a2b7b494 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ The app server provides REST endpoints to interact with the entities and data. F 3.2. To use as an embedded in-memory database instance (Not recommended for production deployments), set the `spring.datasource.url=jdbc:hsqldb:mem:/appserver` in `application-.properties`. Also, change the properties in `src/main/resources/application.properties` to dev or prod according to your requirements. -4. Build the project using gradle wrapper and run using spring boot. Note: This project uses JAVA 11, please download and install it before building. +4. Build the project using gradle wrapper and run using spring boot. Note: This project uses JAVA 17, please download and install it before building. 5. The build will need to create a logs directory. The default path is `/usr/local/var/lib/radar/appserver/logs`. Either create the directory there using `sudo mkdir -p /usr/local/var/lib/radar/appserver/logs` followed by `sudo chown $USER /usr/local/var/lib/radar/appserver/logs` or change logs file directory in `src/main/resources/logback-spring.xml` to local log directory like `` From a0aed77831a234ad32738bc5a765041e287c9348 Mon Sep 17 00:00:00 2001 From: Bastiaan Date: Tue, 19 Sep 2023 11:45:33 +0200 Subject: [PATCH 02/17] Command runs on windows as well (powershell). --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a2b7b494..7c951ab6 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ The app server provides REST endpoints to interact with the entities and data. F 6. The appserver uses the Admin SDK to communicate with the Firebase Cloud Messaging. To configure this, please look at the [FCM section](#fcm). -7. To run the build on mac or linux, run the below - +7. To run the build, run the command below - ```bash ./gradlew bootRun ``` From 7c8de2a269414679062e17be1eeac3e2c4650999 Mon Sep 17 00:00:00 2001 From: Bastiaan Date: Tue, 26 Sep 2023 15:00:06 +0200 Subject: [PATCH 03/17] move radar_is.yml to radar-is.yml. This is in line with our file naming convention, and is hardcoded was hardcoded. --- README.md | 6 +++--- build.gradle | 2 +- radar-is.yml | 3 +++ radar_is.yml | 3 --- .../resources/docker/docker-compose.yml | 15 ++++++++------- src/integrationTest/resources/radar_is.yml | 2 +- .../radarbase/appserver/config/AuthConfig.java | 2 +- src/main/resources/radar-is.yml | 3 +++ 8 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 radar-is.yml delete mode 100644 radar_is.yml create mode 100644 src/main/resources/radar-is.yml diff --git a/README.md b/README.md index 7c951ab6..6abe45a6 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ The same can be achieved by running as a docker-compose service. Just specify th ports: - 8080:8080 volumes: - - ./radar_is.yml:/resources/radar_is.yml + - ./radar-is.yml:/resources/radar-is.yml - ./logs/:/var/log/radar/appserver/ - ./etc/google-credentials.json:/etc/google-credentials.json environment: @@ -176,7 +176,7 @@ The same can be achieved by running as a docker-compose service. Just specify th RADAR_ADMIN_USER: "radar" RADAR_ADMIN_PASSWORD: "radar" SPRING_APPLICATION_JSON: '{"spring":{"boot":{"admin":{"client":{"url":"http://spring-boot-admin:1111","username":"radar","password":"appserver"}}}}}' - RADAR_IS_CONFIG_LOCATION: "/resources/radar_is.yml" + RADAR_IS_CONFIG_LOCATION: "/resources/radar-is.yml" SPRING_BOOT_ADMIN_CLIENT_INSTANCE_NAME: radar-appserver ``` @@ -378,7 +378,7 @@ security.radar.managementportal.url= This will instantiate all the classes needed for security using the management portal. Per endpoint level auth is controlled using Pre and Post annotations for each permission. All the classes are located in [/src/main/java/org/radarbase/appserver/auth/managementportal](/src/main/java/org/radarbase/appserver/auth/managementportal). -You can provide the Management Portal specific config in [radar_is.yml](radar_is.yml) file providing the public key endpoint and the resource name. The path to this file should be specified in the env variable `RADAR_IS_CONFIG_LOCATION`. +You can provide the Management Portal specific config in [radar-is.yml](radar-is.yml) file providing the public key endpoint and the resource name. The path to this file should be specified in the env variable `RADAR_IS_CONFIG_LOCATION`. ### Management Portal Clients If security is enabled, please also make sure that the correct resources and scope are set in the OAuth Client configurations in Management Portal. diff --git a/build.gradle b/build.gradle index 23f5e399..8702ff9a 100644 --- a/build.gradle +++ b/build.gradle @@ -165,7 +165,7 @@ task integrationTest(type: Test) { useJUnitPlatform() { excludeEngines 'junit-vintage' } - environment "RADAR_IS_CONFIG_LOCATION", "src/integrationTest/resources/radar_is.yml" + environment "RADAR_IS_CONFIG_LOCATION", "src/integrationTest/resources/radar-is.yml" shouldRunAfter test } diff --git a/radar-is.yml b/radar-is.yml new file mode 100644 index 00000000..a485dc94 --- /dev/null +++ b/radar-is.yml @@ -0,0 +1,3 @@ +resourceName: res_AppServer +publicKeyEndpoints: + - http://localhost:8081/managementportal/oauth/token_key \ No newline at end of file diff --git a/radar_is.yml b/radar_is.yml deleted file mode 100644 index edc6ac67..00000000 --- a/radar_is.yml +++ /dev/null @@ -1,3 +0,0 @@ -resourceName: res_AppServer -publicKeyEndpoints: - - http://localhost:8081/oauth/token_key \ No newline at end of file diff --git a/src/integrationTest/resources/docker/docker-compose.yml b/src/integrationTest/resources/docker/docker-compose.yml index dfb5159e..e7c2293c 100644 --- a/src/integrationTest/resources/docker/docker-compose.yml +++ b/src/integrationTest/resources/docker/docker-compose.yml @@ -19,7 +19,7 @@ services: POSTGRES_DB: radar POSTGRES_PASSWORD: radar ports: - - 5432:5432 + - "5432:5432" appserver: build: ../../../.. @@ -29,12 +29,12 @@ services: - default - admin ports: - - 8080:8080 + - "8080:8080" depends_on: - postgres - spring-boot-admin volumes: - - ../radar_is.yml:/resources/radar_is.yml + - ../radar-is.yml:/resources/radar-is.yml - ../../../../logs/:/var/log/radar/appserver/ environment: JDK_JAVA_OPTIONS: -Xmx4G -Djava.security.egd=file:/dev/./urandom @@ -43,7 +43,7 @@ services: RADAR_ADMIN_USER: "radar" RADAR_ADMIN_PASSWORD: "radar" SPRING_APPLICATION_JSON: '{"spring":{"boot":{"admin":{"client":{"url":"http://spring-boot-admin:1111","username":"radar","password":"appserver"}}}}}' - RADAR_IS_CONFIG_LOCATION: "/resources/radar_is.yml" + RADAR_IS_CONFIG_LOCATION: "/resources/radar-is.yml" SPRING_BOOT_ADMIN_CLIENT_INSTANCE_NAME: radar-appserver spring-boot-admin: @@ -53,7 +53,7 @@ services: - admin - default ports: - - 8888:1111 + - "8888:1111" environment: SPRING_BOOT_ADMIN_USER_NAME: radar SPRING_BOOT_ADMIN_USER_PASSWORD: appserver @@ -71,10 +71,11 @@ services: SPRING_PROFILES_ACTIVE: dev SERVER_PORT: 8081 MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET: "" - MANAGEMENTPORTAL_COMMON_BASE_URL: http://localhost:8081/ - MANAGEMENTPORTAL_COMMON_MANAGEMENT_PORTAL_BASE_URL: http://localhost:8081/ + MANAGEMENTPORTAL_COMMON_BASE_URL: http://localhost:8081/managementportal + MANAGEMENTPORTAL_COMMON_MANAGEMENT_PORTAL_BASE_URL: http://localhost:8081/managementportal MANAGEMENTPORTAL_OAUTH_CLIENTS_FILE: /mp-includes/config/oauth_client_details.csv MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT: 'false' + SERVER_SERVLET_CONTEXT_PATH: /managementportal JAVA_OPTS: -Xmx256m # maximum heap size for the JVM running ManagementPortal, increase this as necessary volumes: - ./etc/:/mp-includes/ diff --git a/src/integrationTest/resources/radar_is.yml b/src/integrationTest/resources/radar_is.yml index edc6ac67..a485dc94 100644 --- a/src/integrationTest/resources/radar_is.yml +++ b/src/integrationTest/resources/radar_is.yml @@ -1,3 +1,3 @@ resourceName: res_AppServer publicKeyEndpoints: - - http://localhost:8081/oauth/token_key \ No newline at end of file + - http://localhost:8081/managementportal/oauth/token_key \ No newline at end of file diff --git a/src/main/java/org/radarbase/appserver/config/AuthConfig.java b/src/main/java/org/radarbase/appserver/config/AuthConfig.java index ad2a2ceb..cde380cd 100644 --- a/src/main/java/org/radarbase/appserver/config/AuthConfig.java +++ b/src/main/java/org/radarbase/appserver/config/AuthConfig.java @@ -36,7 +36,7 @@ public ManagementPortalAuthProperties getAuthProperties() { } /** - * First tries to load config from radar_is.yml config file. If any issues, then uses the default + * First tries to load config from radar-is.yml config file. If any issues, then uses the default * MP oauth token key endpoint. * * @param managementPortalAuthProperties diff --git a/src/main/resources/radar-is.yml b/src/main/resources/radar-is.yml new file mode 100644 index 00000000..a485dc94 --- /dev/null +++ b/src/main/resources/radar-is.yml @@ -0,0 +1,3 @@ +resourceName: res_AppServer +publicKeyEndpoints: + - http://localhost:8081/managementportal/oauth/token_key \ No newline at end of file From 8075438e562d3b512527a2e9f727dbf83c05d17d Mon Sep 17 00:00:00 2001 From: Bastiaan Date: Tue, 26 Sep 2023 15:08:41 +0200 Subject: [PATCH 04/17] minor updates to readme --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6abe45a6..273db0d3 100644 --- a/README.md +++ b/README.md @@ -141,13 +141,12 @@ The same result as stated in [Getting Started](#getting-started) can be achieved ## FCM ### AdminSDK -To configure AdminSDK, follow the official Firebase [documentation](https://firebase.google. -com/docs/admin/setup#initialize-sdk) till you setup the environment variable (`GOOGLE_APPLICATION_CREDENTIALS`). In the properties +To configure AdminSDK, follow the official Firebase [documentation](https://firebase.google.com/docs/admin/setup#initialize-sdk) till you setup the environment variable (`GOOGLE_APPLICATION_CREDENTIALS`). In the properties file, you would need to set `fcmserver.fcmsender` to `org.radarbase.fcm.downstream.AdminSdkFcmSender`. ## Docker/ Docker Compose -The AppServer is also available as a docker container. It's [Dockerfile](/Dockerfile) is provided with the project. It can be run as follows - +The AppServer is also available as a docker container. Its [Dockerfile](/Dockerfile) is provided with the project. It can be run as follows - ```shell docker run -v /logs/:/var/log/radar/appserver/ \ From d7df3bec0cbc6bb32137f36d53d9b5841627a2ac Mon Sep 17 00:00:00 2001 From: Bastiaan Date: Tue, 26 Sep 2023 15:32:11 +0200 Subject: [PATCH 05/17] time in seconds --- src/main/resources/application-dev.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index 694c1546..b5e7f81f 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -100,7 +100,7 @@ security.radar.managementportal.url=http://localhost:8081 #security.oauth2.client.userAuthorizationUri= # Github Authentication security.github.client.token= -security.github.client.timeout=PT10s +security.github.client.timeout=10 # max content size 1 MB security.github.client.maxContentLength=1000000 security.github.cache.size=10000 From 3a2342a4105400cb989d8a5c146a9f9779466ff5 Mon Sep 17 00:00:00 2001 From: Bastiaan Date: Tue, 3 Oct 2023 10:18:28 +0200 Subject: [PATCH 06/17] moved the radar_is.yml file in integrationTest folder --- src/integrationTest/resources/{radar_is.yml => radar-is.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/integrationTest/resources/{radar_is.yml => radar-is.yml} (100%) diff --git a/src/integrationTest/resources/radar_is.yml b/src/integrationTest/resources/radar-is.yml similarity index 100% rename from src/integrationTest/resources/radar_is.yml rename to src/integrationTest/resources/radar-is.yml From 6ffcee39b7aba1ea843842c5c8b90f10128ad17a Mon Sep 17 00:00:00 2001 From: Bastiaan Date: Tue, 3 Oct 2023 10:19:13 +0200 Subject: [PATCH 07/17] update spring docs so the open-api docs get built again. --- build.gradle | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 8702ff9a..69cc1846 100644 --- a/build.gradle +++ b/build.gradle @@ -36,7 +36,7 @@ ext { springBootVersion = '2.6.6' springVersion = '6.0.6' springOauth2Version = "2.5.1.RELEASE" - springDocVersion = '1.6.14' + springDocVersion = '2.2.0' lombokVersion = '1.18.26' junit5Version = '5.9.2' radarSpringAuthVersion = '1.2.0' @@ -68,8 +68,7 @@ dependencies { runtimeOnly("org.hibernate.validator:hibernate-validator:$hibernateValidatorVersion") // Open API spec - implementation(group: 'org.springdoc', name: 'springdoc-openapi-ui', version: springDocVersion) - + implementation(group: 'org.springdoc', name: 'springdoc-openapi-starter-webmvc-ui', version: springDocVersion) //runtimeOnly('org.springframework.boot:spring-boot-devtools') runtimeOnly('org.hsqldb:hsqldb') From 259caeb8ee0783eb47f9d4909386e82fba775f68 Mon Sep 17 00:00:00 2001 From: Bastiaan Date: Mon, 9 Oct 2023 11:48:18 +0200 Subject: [PATCH 08/17] change the authentication endpoint to the previous state --- radar-is.yml | 2 +- src/integrationTest/resources/radar-is.yml | 2 +- src/main/resources/radar-is.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/radar-is.yml b/radar-is.yml index a485dc94..edc6ac67 100644 --- a/radar-is.yml +++ b/radar-is.yml @@ -1,3 +1,3 @@ resourceName: res_AppServer publicKeyEndpoints: - - http://localhost:8081/managementportal/oauth/token_key \ No newline at end of file + - http://localhost:8081/oauth/token_key \ No newline at end of file diff --git a/src/integrationTest/resources/radar-is.yml b/src/integrationTest/resources/radar-is.yml index a485dc94..edc6ac67 100644 --- a/src/integrationTest/resources/radar-is.yml +++ b/src/integrationTest/resources/radar-is.yml @@ -1,3 +1,3 @@ resourceName: res_AppServer publicKeyEndpoints: - - http://localhost:8081/managementportal/oauth/token_key \ No newline at end of file + - http://localhost:8081/oauth/token_key \ No newline at end of file diff --git a/src/main/resources/radar-is.yml b/src/main/resources/radar-is.yml index a485dc94..edc6ac67 100644 --- a/src/main/resources/radar-is.yml +++ b/src/main/resources/radar-is.yml @@ -1,3 +1,3 @@ resourceName: res_AppServer publicKeyEndpoints: - - http://localhost:8081/managementportal/oauth/token_key \ No newline at end of file + - http://localhost:8081/oauth/token_key \ No newline at end of file From 3f772b37252d7e39df2e14063889c251bd9f586f Mon Sep 17 00:00:00 2001 From: Bastiaan Date: Mon, 9 Oct 2023 12:09:08 +0200 Subject: [PATCH 09/17] upload CI build artifacts from --- .github/workflows/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9a42e60d..2ff36ecc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,6 +52,14 @@ jobs: - name: Check run: GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/src/integrationTest/resources/google-credentials.json ./gradlew check + - name: Upload build artifacts + if: always() + uses: actions/upload-artifact@v3 + with: + path: build/reports + if-no-files-found: ignore + retention-days: 5 + # Check that the docker image builds correctly docker: # The type of runner that the job will run on From 6895c3b5b570507109ecdb97f55faf4bf27fb0b2 Mon Sep 17 00:00:00 2001 From: Bastiaan Date: Mon, 9 Oct 2023 12:32:23 +0200 Subject: [PATCH 10/17] add context path to MP_URL url for integration testing --- radar-is.yml | 2 +- .../java/org/radarbase/appserver/auth/common/MPOAuthHelper.java | 2 +- src/integrationTest/resources/radar-is.yml | 2 +- src/main/resources/radar-is.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/radar-is.yml b/radar-is.yml index edc6ac67..a485dc94 100644 --- a/radar-is.yml +++ b/radar-is.yml @@ -1,3 +1,3 @@ resourceName: res_AppServer publicKeyEndpoints: - - http://localhost:8081/oauth/token_key \ No newline at end of file + - http://localhost:8081/managementportal/oauth/token_key \ No newline at end of file diff --git a/src/integrationTest/java/org/radarbase/appserver/auth/common/MPOAuthHelper.java b/src/integrationTest/java/org/radarbase/appserver/auth/common/MPOAuthHelper.java index c90d9ef2..4b57d471 100644 --- a/src/integrationTest/java/org/radarbase/appserver/auth/common/MPOAuthHelper.java +++ b/src/integrationTest/java/org/radarbase/appserver/auth/common/MPOAuthHelper.java @@ -39,7 +39,7 @@ public class MPOAuthHelper implements OAuthHelper { private static final ObjectMapper mapper = new ObjectMapper(); private static final String ACCESS_TOKEN; - private static final String MP_URL = "http://localhost:8081"; + private static final String MP_URL = "http://localhost:8081/managementportal/"; private static final String MP_CLIENT = "ManagementPortalapp"; private static final String REST_CLIENT = "pRMT"; private static final String USER = "sub-1"; diff --git a/src/integrationTest/resources/radar-is.yml b/src/integrationTest/resources/radar-is.yml index edc6ac67..a485dc94 100644 --- a/src/integrationTest/resources/radar-is.yml +++ b/src/integrationTest/resources/radar-is.yml @@ -1,3 +1,3 @@ resourceName: res_AppServer publicKeyEndpoints: - - http://localhost:8081/oauth/token_key \ No newline at end of file + - http://localhost:8081/managementportal/oauth/token_key \ No newline at end of file diff --git a/src/main/resources/radar-is.yml b/src/main/resources/radar-is.yml index edc6ac67..a485dc94 100644 --- a/src/main/resources/radar-is.yml +++ b/src/main/resources/radar-is.yml @@ -1,3 +1,3 @@ resourceName: res_AppServer publicKeyEndpoints: - - http://localhost:8081/oauth/token_key \ No newline at end of file + - http://localhost:8081/managementportal/oauth/token_key \ No newline at end of file From 14c67050575d58e9f50fc6d78a67419a9446db65 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 18 Oct 2023 13:55:35 +0200 Subject: [PATCH 11/17] Fix deadlock in fetching protocols Use the GithubService cache to fetch duplicate protocols. This avoids fetching the same protocol many times, once per user, eventually leading to timeouts --- build.gradle | 10 +-- .../state/MessageStateEventListener.java | 28 ++++---- .../event/state/TaskStateEventListener.java | 23 +++---- .../appserver/service/GithubClient.java | 19 ++--- .../appserver/service/GithubService.java | 17 ++++- .../service/QuestionnaireScheduleService.java | 64 ++++++++--------- .../protocol/DefaultProtocolGenerator.java | 14 ++-- .../GithubProtocolFetcherStrategy.java | 23 +++++-- .../schedule/ScheduleGeneratorService.java | 28 +++++--- .../appserver/util/Base64Deserializer.java | 23 +++---- .../appserver/util/CachedFunction.java | 44 ++++++++++-- .../radarbase/appserver/util/CachedMap.java | 69 ++++++++++--------- 12 files changed, 200 insertions(+), 162 deletions(-) diff --git a/build.gradle b/build.gradle index b782ac09..dce06631 100644 --- a/build.gradle +++ b/build.gradle @@ -112,7 +112,7 @@ javafx { } checkstyle { - configDirectory = file("config/checkstyle") + configDirectory.set(file("config/checkstyle")) toolVersion = "10.8.0" showViolations = false ignoreFailures = true @@ -134,14 +134,14 @@ test { } } -task unpack(type: Copy) { - duplicatesStrategy = 'include' +tasks.register('unpack', Copy) { + duplicatesStrategy = DuplicatesStrategy.INCLUDE dependsOn bootJar from(zipTree(tasks.bootJar.outputs.files.singleFile)) into("build/dependency") } -task loadTest(type: JavaExec) { +tasks.register('loadTest', JavaExec) { dependsOn testClasses description = "Load Test With Gatling" group = "Load Test" @@ -158,7 +158,7 @@ task loadTest(type: JavaExec) { ] } -task integrationTest(type: Test) { +tasks.register('integrationTest', Test) { testClassesDirs = sourceSets.integrationTest.output.classesDirs classpath = sourceSets.integrationTest.runtimeClasspath useJUnitPlatform() { diff --git a/src/main/java/org/radarbase/appserver/event/state/MessageStateEventListener.java b/src/main/java/org/radarbase/appserver/event/state/MessageStateEventListener.java index b3ccd902..c1e68e60 100644 --- a/src/main/java/org/radarbase/appserver/event/state/MessageStateEventListener.java +++ b/src/main/java/org/radarbase/appserver/event/state/MessageStateEventListener.java @@ -24,6 +24,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; +import org.radarbase.appserver.entity.DataMessageStateEvent; +import org.radarbase.appserver.entity.NotificationStateEvent; import org.radarbase.appserver.event.state.dto.DataMessageStateEventDto; import org.radarbase.appserver.event.state.dto.NotificationStateEventDto; import org.radarbase.appserver.service.DataMessageStateEventService; @@ -65,9 +67,8 @@ public MessageStateEventListener(ObjectMapper objectMapper, public void onNotificationStateChange(NotificationStateEventDto event) { String info = convertMapToString(event.getAdditionalInfo()); log.debug("ID: {}, STATE: {}", event.getNotification().getId(), event.getState()); - org.radarbase.appserver.entity.NotificationStateEvent eventEntity = - new org.radarbase.appserver.entity.NotificationStateEvent( - event.getNotification(), event.getState(), event.getTime(), info); + NotificationStateEvent eventEntity = new NotificationStateEvent( + event.getNotification(), event.getState(), event.getTime(), info); notificationStateEventService.addNotificationStateEvent(eventEntity); } @@ -77,22 +78,21 @@ public void onNotificationStateChange(NotificationStateEventDto event) { public void onDataMessageStateChange(DataMessageStateEventDto event) { String info = convertMapToString(event.getAdditionalInfo()); log.debug("ID: {}, STATE: {}", event.getDataMessage().getId(), event.getState()); - org.radarbase.appserver.entity.DataMessageStateEvent eventEntity = - new org.radarbase.appserver.entity.DataMessageStateEvent( - event.getDataMessage(), event.getState(), event.getTime(), info); + DataMessageStateEvent eventEntity = new DataMessageStateEvent( + event.getDataMessage(), event.getState(), event.getTime(), info); dataMessageStateEventService.addDataMessageStateEvent(eventEntity); } public String convertMapToString(Map additionalInfoMap) { - String info = null; - if (additionalInfoMap != null) { - try { - info = objectMapper.writeValueAsString(additionalInfoMap); - } catch (JsonProcessingException exc) { - log.warn("error processing event's additional info: {}", additionalInfoMap); - } + if (additionalInfoMap == null) { + return null; + } + try { + return objectMapper.writeValueAsString(additionalInfoMap); + } catch (JsonProcessingException exc) { + log.warn("error processing event's additional info: {}", additionalInfoMap); + return null; } - return info; } // we can add more event listeners by annotating with @EventListener } diff --git a/src/main/java/org/radarbase/appserver/event/state/TaskStateEventListener.java b/src/main/java/org/radarbase/appserver/event/state/TaskStateEventListener.java index 47b7cfc9..83d099b9 100644 --- a/src/main/java/org/radarbase/appserver/event/state/TaskStateEventListener.java +++ b/src/main/java/org/radarbase/appserver/event/state/TaskStateEventListener.java @@ -24,10 +24,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; -import org.radarbase.appserver.event.state.dto.NotificationStateEventDto; +import org.radarbase.appserver.entity.TaskStateEvent; import org.radarbase.appserver.event.state.dto.TaskStateEventDto; import org.radarbase.appserver.service.TaskStateEventService; -import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Propagation; @@ -50,7 +49,6 @@ public TaskStateEventListener(ObjectMapper objectMapper, this.taskStateEventService = taskStateEventService; } - /** * Handle an application event. * @@ -62,22 +60,21 @@ public TaskStateEventListener(ObjectMapper objectMapper, public void onTaskStateChange(TaskStateEventDto event) { String info = convertMapToString(event.getAdditionalInfo()); log.debug("ID: {}, STATE: {}", event.getTask().getId(), event.getState()); - org.radarbase.appserver.entity.TaskStateEvent eventEntity = - new org.radarbase.appserver.entity.TaskStateEvent( + TaskStateEvent eventEntity = new TaskStateEvent( event.getTask(), event.getState(), event.getTime(), info); taskStateEventService.addTaskStateEvent(eventEntity); } public String convertMapToString(Map additionalInfoMap) { - String info = null; - if (additionalInfoMap != null) { - try { - info = objectMapper.writeValueAsString(additionalInfoMap); - } catch (JsonProcessingException exc) { - log.warn("error processing event's additional info: {}", additionalInfoMap); - } + if (additionalInfoMap == null) { + return null; + } + try { + return objectMapper.writeValueAsString(additionalInfoMap); + } catch (JsonProcessingException exc) { + log.warn("error processing event's additional info: {}", additionalInfoMap); + return null; } - return info; } // we can add more event listeners by annotating with @EventListener } diff --git a/src/main/java/org/radarbase/appserver/service/GithubClient.java b/src/main/java/org/radarbase/appserver/service/GithubClient.java index 78c2603b..c9af99be 100644 --- a/src/main/java/org/radarbase/appserver/service/GithubClient.java +++ b/src/main/java/org/radarbase/appserver/service/GithubClient.java @@ -57,7 +57,7 @@ public class GithubClient { private final transient String authorizationHeader; private transient final Duration httpTimeout; - private transient final Executor executor; + private transient final HttpClient client; @Value("${security.github.client.maxContentLength:1000000}") private transient int maxContentLength; @@ -69,14 +69,15 @@ public GithubClient( @Value("${security.github.client.token:}") String githubToken) { this.authorizationHeader = githubToken != null ? "Bearer " + githubToken.trim() : ""; this.httpTimeout = Duration.ofSeconds(httpTimeout); - this.executor = new ThreadPoolExecutor(0, - 8, - 30, - TimeUnit.SECONDS, - new SynchronousQueue<>()); + this.client = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) + .followRedirects(HttpClient.Redirect.NORMAL) + .connectTimeout(this.httpTimeout) + .build(); } public String getGithubContent(String url) throws IOException, InterruptedException { + log.debug("Fetching Github URL {}", url); URI uri = URI.create(url); HttpResponse response = makeRequest(uri); @@ -96,12 +97,6 @@ public String getGithubContent(String url) throws IOException, InterruptedExcept } private HttpResponse makeRequest(URI uri) throws InterruptedException { - HttpClient client = HttpClient.newBuilder() - .executor(executor) - .version(HttpClient.Version.HTTP_1_1) - .followRedirects(HttpClient.Redirect.NORMAL) - .connectTimeout(this.httpTimeout) - .build(); try { return client.send(getRequest(uri), HttpResponse.BodyHandlers.ofInputStream()); } catch (IOException ex) { diff --git a/src/main/java/org/radarbase/appserver/service/GithubService.java b/src/main/java/org/radarbase/appserver/service/GithubService.java index 644e9496..46e8edc0 100644 --- a/src/main/java/org/radarbase/appserver/service/GithubService.java +++ b/src/main/java/org/radarbase/appserver/service/GithubService.java @@ -7,6 +7,7 @@ import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; +import java.io.IOException; import java.time.Duration; @Component @@ -14,6 +15,7 @@ public class GithubService { private final transient CachedFunction cachedGetContent; + private final transient GithubClient githubClient; @Autowired public GithubService( @@ -24,13 +26,24 @@ public GithubService( int retryTime, @Value("${security.github.cache.size:10000}") int maxSize) { + this.githubClient = githubClient; this.cachedGetContent = new CachedFunction<>(githubClient::getGithubContent, Duration.ofSeconds(cacheTime), Duration.ofSeconds(retryTime), maxSize); } - public String getGithubContent(String url) throws Exception { - return this.cachedGetContent.applyWithException(url); + public String getGithubContent(String url) throws IOException, InterruptedException { + try { + return this.cachedGetContent.applyWithException(url); + } catch (IOException | InterruptedException ex) { + throw ex; + } catch (Exception ex) { + throw new IllegalStateException("Unknown exception " + ex, ex); + } + } + + public String getGithubContentWithoutCache(String url) throws IOException, InterruptedException { + return githubClient.getGithubContent(url); } } diff --git a/src/main/java/org/radarbase/appserver/service/QuestionnaireScheduleService.java b/src/main/java/org/radarbase/appserver/service/QuestionnaireScheduleService.java index 9860b79e..5b37b629 100644 --- a/src/main/java/org/radarbase/appserver/service/QuestionnaireScheduleService.java +++ b/src/main/java/org/radarbase/appserver/service/QuestionnaireScheduleService.java @@ -21,14 +21,12 @@ package org.radarbase.appserver.service; -import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.radarbase.appserver.dto.protocol.Assessment; import org.radarbase.appserver.dto.protocol.AssessmentType; import org.radarbase.appserver.dto.protocol.Protocol; import org.radarbase.appserver.dto.questionnaire.AssessmentSchedule; import org.radarbase.appserver.dto.questionnaire.Schedule; -import org.radarbase.appserver.entity.Notification; import org.radarbase.appserver.entity.Project; import org.radarbase.appserver.entity.Task; import org.radarbase.appserver.entity.User; @@ -47,11 +45,13 @@ import org.springframework.transaction.annotation.Transactional; import java.time.Instant; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; @Slf4j @Service @@ -59,11 +59,11 @@ @SuppressWarnings("PMD.ConstructorCallsOverridableMethod") public class QuestionnaireScheduleService { - private static final String TASK_SEARCH_PATTERN = "(\\w+?)(:|<|>)(\\w+?),"; + private static final Pattern TASK_SEARCH_PATTERN = Pattern.compile("(\\w+?)([:<>])(\\w+?),"); private final transient ProtocolGenerator protocolGenerator; - private transient HashMap subjectScheduleMap = new HashMap(); + private final transient HashMap subjectScheduleMap = new HashMap<>(); private final transient UserRepository userRepository; @@ -136,28 +136,31 @@ public Schedule generateScheduleUsingProjectIdAndSubjectId(String projectId, Str @Transactional public Schedule generateScheduleForUser(User user) { Protocol protocol = protocolGenerator.getProtocolForSubject(user.getSubjectId()); + Schedule newSchedule; + if (protocol == null) { - Schedule emptySchedule = new Schedule(); - subjectScheduleMap.put(user.getSubjectId(), emptySchedule); - return emptySchedule; - } - Schedule prevSchedule = getScheduleForSubject(user.getSubjectId()); - String prevTimezone = prevSchedule.getTimezone() != null ? prevSchedule.getTimezone() : user.getTimezone(); - if (!Objects.equals(prevSchedule.getVersion(), protocol.getVersion()) || !prevTimezone.equals(user.getTimezone())) { - this.removeScheduleForUser(user); + newSchedule = new Schedule(); + } else { + Schedule prevSchedule = getScheduleForSubject(user.getSubjectId()); + String prevTimezone = prevSchedule.getTimezone() != null + ? prevSchedule.getTimezone() + : user.getTimezone(); + if (!Objects.equals(prevSchedule.getVersion(), protocol.getVersion()) || !prevTimezone.equals(user.getTimezone())) { + this.removeScheduleForUser(user); + } + newSchedule = this.scheduleGeneratorService.generateScheduleForUser(user, protocol, prevSchedule); } - Schedule newSchedule = this.scheduleGeneratorService.generateScheduleForUser(user, protocol, prevSchedule); subjectScheduleMap.put(user.getSubjectId(), newSchedule); - this.saveTasksAndNotifications(newSchedule.getAssessmentSchedules(), user); + return newSchedule; } private void saveTasksAndNotifications(List assessmentSchedules, User user) { assessmentSchedules.stream() .filter(Objects::nonNull) - .filter(s -> s.hasTasks()) + .filter(AssessmentSchedule::hasTasks) .forEach(a -> { this.taskService.addTasks(a.getTasks(), user); this.notificationService.addNotifications(a.getNotifications(), user); @@ -175,8 +178,9 @@ public Schedule generateScheduleUsingProjectIdAndSubjectIdAndAssessment(String p throw new NotFoundException("Assessment not found in protocol. Add assessment to protocol first."); } - Schedule schedule = this.getScheduleForSubject(user.getSubjectId()); - AssessmentSchedule a = this.scheduleGeneratorService.generateSingleAssessmentSchedule(assessment, user, Collections.emptyList(), user.getTimezone()); + Schedule schedule = getScheduleForSubject(user.getSubjectId()); + AssessmentSchedule a = scheduleGeneratorService.generateSingleAssessmentSchedule( + assessment, user, Collections.emptyList(), user.getTimezone()); schedule.addAssessmentSchedule(a); this.saveTasksAndNotifications(List.of(a), user); @@ -184,26 +188,15 @@ public Schedule generateScheduleUsingProjectIdAndSubjectIdAndAssessment(String p } @Scheduled(fixedRate = 3_600_000) - @SuppressWarnings("PMD.DataflowAnomalyAnalysis") public void generateAllSchedules() { List users = this.userRepository.findAll(); log.info("Generating all schedules.."); - - List schedules = users.stream() - .map(u -> { - Schedule schedule = this.generateScheduleForUser(u); - return schedule; - }).collect(Collectors.toList()); + users.forEach(this::generateScheduleForUser); } public Schedule getScheduleForSubject(String subjectId) { - try { - Schedule schedule = subjectScheduleMap.get(subjectId); - return schedule != null ? schedule : new Schedule(); - } catch (NoSuchElementException ex) { - log.warn("Subject does not exist in map."); - } - return new Schedule(); + Schedule schedule = subjectScheduleMap.get(subjectId); + return schedule != null ? schedule : new Schedule(); } @Transactional @@ -252,8 +245,7 @@ private TaskSpecificationsBuilder getSearchBuilder(String projectId, builder.with("type", ":", type); } - Pattern pattern = Pattern.compile(TASK_SEARCH_PATTERN); - Matcher matcher = pattern.matcher(search + ","); + Matcher matcher = TASK_SEARCH_PATTERN.matcher(search + ","); while (matcher.find()) { builder.with(matcher.group(1), matcher.group(2), matcher.group(3)); } diff --git a/src/main/java/org/radarbase/appserver/service/questionnaire/protocol/DefaultProtocolGenerator.java b/src/main/java/org/radarbase/appserver/service/questionnaire/protocol/DefaultProtocolGenerator.java index 57584cb0..e7b0a37a 100644 --- a/src/main/java/org/radarbase/appserver/service/questionnaire/protocol/DefaultProtocolGenerator.java +++ b/src/main/java/org/radarbase/appserver/service/questionnaire/protocol/DefaultProtocolGenerator.java @@ -66,8 +66,7 @@ public void init() { new CachedMap<>( protocolFetcherStrategy::fetchProtocols, CACHE_INVALIDATE_DEFAULT, CACHE_RETRY_DEFAULT); cachedProjectProtocolMap = - new CachedMap<>( - protocolFetcherStrategy::fetchProtocolsPerProject, CACHE_INVALIDATE_DEFAULT, CACHE_RETRY_DEFAULT); + new CachedMap<>(protocolFetcherStrategy::fetchProtocolsPerProject, CACHE_INVALIDATE_DEFAULT, CACHE_RETRY_DEFAULT); log.debug("initialized Github Protocol generator"); } @@ -87,16 +86,14 @@ public Protocol getProtocol(String projectId) throws IOException { try { return cachedProjectProtocolMap.get(projectId); } catch (IOException ex) { - log.warn( - "Cannot retrieve Protocols for project {} : {}, Using cached values.", projectId, ex.toString()); - return cachedProjectProtocolMap.get(true).get(projectId); + log.warn("Cannot retrieve Protocols for project {} : {}, Using cached values.", projectId, ex.toString()); + return cachedProjectProtocolMap.getCache().get(projectId); } } private @NonNull Protocol forceGetProtocolForSubject(String subjectId) { try { - cachedProtocolMap.get(true); - return cachedProtocolMap.get(subjectId); + return cachedProtocolMap.get(true).get(subjectId); } catch (IOException ex) { log.warn("Cannot retrieve Protocols, using cached values if available.", ex); return cachedProtocolMap.getCache().get(subjectId); @@ -112,8 +109,7 @@ public Protocol getProtocolForSubject(String subjectId) { } return protocol; } catch (IOException ex) { - log.warn( - "Cannot retrieve Protocols for subject {} : {}, Using cached values.", subjectId, ex.toString()); + log.warn("Cannot retrieve Protocols for subject {} : {}, Using cached values.", subjectId, ex.toString()); return cachedProtocolMap.getCache().get(subjectId); } catch(NoSuchElementException ex) { log.warn("Subject does not exist in map. Fetching.."); diff --git a/src/main/java/org/radarbase/appserver/service/questionnaire/protocol/GithubProtocolFetcherStrategy.java b/src/main/java/org/radarbase/appserver/service/questionnaire/protocol/GithubProtocolFetcherStrategy.java index 42030b58..22777e9f 100644 --- a/src/main/java/org/radarbase/appserver/service/questionnaire/protocol/GithubProtocolFetcherStrategy.java +++ b/src/main/java/org/radarbase/appserver/service/questionnaire/protocol/GithubProtocolFetcherStrategy.java @@ -33,10 +33,12 @@ import org.radarbase.appserver.dto.protocol.GithubContent; import org.radarbase.appserver.dto.protocol.Protocol; import org.radarbase.appserver.dto.protocol.ProtocolCacheEntry; +import org.radarbase.appserver.entity.Project; import org.radarbase.appserver.entity.User; import org.radarbase.appserver.repository.ProjectRepository; import org.radarbase.appserver.repository.UserRepository; import org.radarbase.appserver.service.GithubClient; +import org.radarbase.appserver.service.GithubService; import org.radarbase.appserver.util.CachedMap; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -47,13 +49,19 @@ import java.io.IOException; import java.net.URI; +import java.net.URL; import java.time.Duration; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; +import java.util.function.Function; import java.util.stream.Collectors; @Slf4j @@ -72,7 +80,8 @@ public class GithubProtocolFetcherStrategy implements ProtocolFetcherStrategy { private final transient ObjectMapper localMapper; // Keeps a cache of github URI's associated with protocol for each project private final transient CachedMap projectProtocolUriMap; - private final transient GithubClient githubClient; + + private final transient GithubService githubService; @SneakyThrows @Autowired @@ -83,7 +92,7 @@ public GithubProtocolFetcherStrategy( ObjectMapper objectMapper, UserRepository userRepository, ProjectRepository projectRepository, - GithubClient githubClient) { + GithubService githubService) { if (protocolRepo == null || protocolRepo.isEmpty() || protocolFileName == null @@ -101,7 +110,7 @@ public GithubProtocolFetcherStrategy( this.localMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); this.userRepository = userRepository; this.projectRepository = projectRepository; - this.githubClient = githubClient; + this.githubService = githubService; } @Override @@ -147,7 +156,7 @@ private ProtocolCacheEntry fetchProtocolForSingleUser(User u, String projectId, } @Override - public synchronized Map fetchProtocolsPerProject() { + public Map fetchProtocolsPerProject() { Set protocolPaths = getProtocolPaths(); if (protocolPaths == null) { @@ -221,10 +230,10 @@ private Map getProtocolDirectories() throws IOException { Map protocolUriMap = new HashMap<>(); try { - String content = githubClient.getGithubContent(GITHUB_API_URI + protocolRepo + "/branches/" + protocolBranch); + String content = githubService.getGithubContentWithoutCache(GITHUB_API_URI + protocolRepo + "/branches/" + protocolBranch); ObjectNode result = getArrayNode(content); String treeSha = result.findValue("tree").findValue("sha").asText(); - String treeContent = githubClient.getGithubContent(GITHUB_API_URI + protocolRepo + "/git/trees/" + treeSha + "?recursive=true"); + String treeContent = githubService.getGithubContent(GITHUB_API_URI + protocolRepo + "/git/trees/" + treeSha + "?recursive=true"); JsonNode tree = getArrayNode(treeContent).get("tree"); for (JsonNode jsonNode : tree) { @@ -242,7 +251,7 @@ private Map getProtocolDirectories() throws IOException { } private Protocol getProtocolFromUrl(URI uri) throws IOException, InterruptedException { - String contentString = githubClient.getGithubContent(uri.toString()); + String contentString = githubService.getGithubContent(uri.toString()); GithubContent content = localMapper.readValue(contentString, GithubContent.class); return localMapper.readValue(content.getContent(), Protocol.class); } diff --git a/src/main/java/org/radarbase/appserver/service/questionnaire/schedule/ScheduleGeneratorService.java b/src/main/java/org/radarbase/appserver/service/questionnaire/schedule/ScheduleGeneratorService.java index d9ae7263..7081707c 100644 --- a/src/main/java/org/radarbase/appserver/service/questionnaire/schedule/ScheduleGeneratorService.java +++ b/src/main/java/org/radarbase/appserver/service/questionnaire/schedule/ScheduleGeneratorService.java @@ -9,6 +9,7 @@ import org.radarbase.appserver.service.questionnaire.protocol.ProtocolHandler; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -35,28 +36,33 @@ default Schedule generateScheduleForUser(User user, Protocol protocol, Schedule return new Schedule(); } List prevAssessmentSchedules = prevSchedule.getAssessmentSchedules(); - String prevTimezone = prevSchedule.getTimezone() != null ? prevSchedule.getTimezone() : user.getTimezone(); + String prevTimezone = prevSchedule.getTimezone() != null + ? prevSchedule.getTimezone() + : user.getTimezone(); - List assessmentSchedules = assessments.parallelStream().map( - assessment -> { - Optional prevAssessmentSchedule = - prevAssessmentSchedules.stream().filter(a -> Objects.equals(a.getName(), assessment.getName())).findFirst(); - return this.generateSingleAssessmentSchedule(assessment, user, prevAssessmentSchedule.isPresent() ? prevAssessmentSchedule.get().getTasks() : new ArrayList<>(), prevTimezone); - } - ).collect(Collectors.toList()); + List assessmentSchedules = assessments.parallelStream() + .map(assessment -> { + List tasks = prevAssessmentSchedules.stream() + .filter(a -> Objects.equals(a.getName(), assessment.getName())) + .findFirst() + .map(AssessmentSchedule::getTasks) + .orElse(Collections.emptyList()); + return generateSingleAssessmentSchedule(assessment, user, tasks, prevTimezone); + }) + .collect(Collectors.toList()); return new Schedule(assessmentSchedules, user, protocol.getVersion()); } default AssessmentSchedule generateSingleAssessmentSchedule(Assessment assessment, User user, List previousTasks, String prevTimezone) { - ProtocolHandlerRunner protocolHandlerRunner = - new ProtocolHandlerRunner(); + ProtocolHandlerRunner protocolHandlerRunner = new ProtocolHandlerRunner(); protocolHandlerRunner.addProtocolHandler(this.getProtocolHandler(assessment)); protocolHandlerRunner.addProtocolHandler(this.getRepeatProtocolHandler(assessment)); protocolHandlerRunner.addProtocolHandler(this.getRepeatQuestionnaireHandler(assessment)); protocolHandlerRunner.addProtocolHandler(this.getNotificationHandler(assessment)); protocolHandlerRunner.addProtocolHandler(this.getReminderHandler(assessment)); - protocolHandlerRunner.addProtocolHandler(this.getCompletedQuestionnaireHandler(assessment, previousTasks, prevTimezone)); + protocolHandlerRunner.addProtocolHandler( + this.getCompletedQuestionnaireHandler(assessment, previousTasks, prevTimezone)); return protocolHandlerRunner.runProtocolHandlers(assessment, user); } diff --git a/src/main/java/org/radarbase/appserver/util/Base64Deserializer.java b/src/main/java/org/radarbase/appserver/util/Base64Deserializer.java index 5306e019..fb71729e 100644 --- a/src/main/java/org/radarbase/appserver/util/Base64Deserializer.java +++ b/src/main/java/org/radarbase/appserver/util/Base64Deserializer.java @@ -21,34 +21,33 @@ package org.radarbase.appserver.util; -import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.*; +import com.fasterxml.jackson.databind.BeanProperty; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.deser.ContextualDeserializer; import com.fasterxml.jackson.databind.exc.InvalidFormatException; import java.io.IOException; import java.util.Base64; -public class Base64Deserializer extends JsonDeserializer implements ContextualDeserializer { - - private transient Class resultClass; - +public class Base64Deserializer extends JsonDeserializer implements ContextualDeserializer { @Override public JsonDeserializer createContextual(DeserializationContext context, BeanProperty property) throws JsonMappingException { - this.resultClass = property.getType().getRawClass(); + if (!String.class.isAssignableFrom(property.getType().getRawClass())) { + throw context.invalidTypeIdException(property.getType(), "String", "Base64 decoding is only applied to String fields."); + } return this; } @Override - public String deserialize(JsonParser parser, DeserializationContext context) throws IOException, JsonProcessingException { + public String deserialize(JsonParser parser, DeserializationContext context) throws IOException { String value = clean(parser.getValueAsString()); Base64.Decoder decoder = Base64.getDecoder(); try { byte[] decodedValue = decoder.decode(value); - return new String(decodedValue); } catch (IllegalArgumentException e) { String fieldName = parser.getParsingContext().getCurrentName(); @@ -64,6 +63,6 @@ public String deserialize(JsonParser parser, DeserializationContext context) thr } public String clean(String value) { - return value.replace("\n", "").replace("\r", ""); + return value.replaceAll("[\n\r]", ""); } -} \ No newline at end of file +} diff --git a/src/main/java/org/radarbase/appserver/util/CachedFunction.java b/src/main/java/org/radarbase/appserver/util/CachedFunction.java index 943b9cf1..192ef34b 100644 --- a/src/main/java/org/radarbase/appserver/util/CachedFunction.java +++ b/src/main/java/org/radarbase/appserver/util/CachedFunction.java @@ -1,5 +1,6 @@ package org.radarbase.appserver.util; +import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import org.springframework.util.function.ThrowingFunction; @@ -9,7 +10,9 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.Semaphore; +@Slf4j public class CachedFunction implements ThrowingFunction { private transient final Duration cacheTime; @@ -20,6 +23,8 @@ public class CachedFunction implements ThrowingFunction { private transient final Map>> cachedMap; private transient final ThrowingFunction function; + private transient final Semaphore semaphore; + public CachedFunction(ThrowingFunction function, Duration cacheTime, Duration retryTime, @@ -29,20 +34,23 @@ public CachedFunction(ThrowingFunction function, this.maxSize = maxSize; this.cachedMap = new LinkedHashMap<>(16, 0.75f, false); this.function = function; + this.semaphore = new Semaphore(2); } @NotNull public V applyWithException(@NotNull K input) throws Exception { - SoftReference> localRef; - synchronized (cachedMap) { - localRef = cachedMap.get(input); - } - Result result = localRef != null ? localRef.get() : null; - if (result != null && !result.isExpired()) { - return result.getOrThrow(); + V result = tryGet(input); + if (result != null) { + return result; } + semaphore.acquire(); try { + result = tryGet(input); + if (result != null) { + return result; + } + log.debug("Recomputing {} in cache", input); V content = function.applyWithException(input); putCache(input, new Result<>(cacheTime, content, null)); return content; @@ -57,12 +65,34 @@ public V applyWithException(@NotNull K input) throws Exception { return exResult.getOrThrow(); } } + } finally { + semaphore.release(); + } + } + + private V tryGet(@NotNull K input) throws Exception { + SoftReference> localRef; + synchronized (cachedMap) { + localRef = cachedMap.get(input); + } + Result result = localRef != null ? localRef.get() : null; + if (result != null && !result.isExpired()) { + log.debug("Retrieved {} from cache", input); + return result.getOrThrow(); + } else { + log.debug("Value for {} not in cache", input); + return null; } } @SuppressWarnings("PMD.DataflowAnomalyAnalysis") private void putCache(K input, Result result) { synchronized (cachedMap) { + if (result.exception != null) { + log.debug("Put {} in cache with exception {}", input, result.exception.toString()); + } else { + log.debug("Put {} in cache with value", input); + } cachedMap.put(input, new SoftReference<>(result)); int toRemove = cachedMap.size() - maxSize; if (toRemove > 0) { diff --git a/src/main/java/org/radarbase/appserver/util/CachedMap.java b/src/main/java/org/radarbase/appserver/util/CachedMap.java index 4dcfa5bd..e18dce8b 100644 --- a/src/main/java/org/radarbase/appserver/util/CachedMap.java +++ b/src/main/java/org/radarbase/appserver/util/CachedMap.java @@ -27,6 +27,7 @@ import java.time.temporal.Temporal; import java.util.Map; import java.util.NoSuchElementException; +import java.util.concurrent.atomic.AtomicReference; /** * Map that caches the result of a list for a limited time. @@ -38,8 +39,8 @@ public class CachedMap { private final transient ThrowingSupplier> retriever; private final transient Duration invalidateAfter; private final transient Duration retryAfter; - private transient Temporal lastFetch; - private transient Map cache; + + private final transient AtomicReference> cache; /** * Map that retrieves data from a supplier and converts that to a map with given key extractor. @@ -56,7 +57,7 @@ public CachedMap( this.retriever = retriever; this.invalidateAfter = invalidateAfter; this.retryAfter = retryAfter; - this.lastFetch = Instant.MIN; + this.cache = new AtomicReference<>(new Result<>(Map.of(), Instant.MIN)); } /** @@ -78,19 +79,14 @@ public Map get() throws IOException { */ public Map get(boolean forceRefresh) throws IOException { if (!forceRefresh) { - synchronized (this) { - if (cache != null && !isThresholdPassed(lastFetch, invalidateAfter)) { - return cache; - } + Result existingResult = cache.get(); + if (!existingResult.isStale(invalidateAfter)) { + return existingResult.result; } } Map result = retriever.get(); - - synchronized (this) { - cache = result; - lastFetch = Instant.now(); - return cache; - } + cache.set(new Result<>(result)); + return result; } /** @@ -99,33 +95,25 @@ public Map get(boolean forceRefresh) throws IOException { * @return map of data */ public Map getCache() { - synchronized (this) { - return cache; - } + return cache.get().result; } /** - * Get a key from the map. If the key is missing, it will check with {@link #mayRetry()} whether + * Get a key from the map. If the key is missing, it will check whether * the cache may be updated. If so, it will fetch the cache again and look the key up. * * @param key key of the value to find. - * @return element + * @return element or null if it is not found * @throws IOException if the cache cannot be refreshed. - * @throws NoSuchElementException if the element is not found. */ - public T get(S key) throws IOException, NoSuchElementException { - T value = get().get(key); - if (value == null) { - if (mayRetry()) { - value = get(true).get(key); - } + public T get(S key) throws IOException { + Result result = cache.get(); + T value = result.result.get(key); + if (value == null && result.isStale(retryAfter)) { + return get(true).get(key); + } else { + return value; } - return value; - } - - /** Whether the cache may be refreshed. */ - public synchronized boolean mayRetry() { - return isThresholdPassed(lastFetch, retryAfter); } /** @@ -138,8 +126,21 @@ public interface ThrowingSupplier { T get() throws IOException; } - /** Whether a given temporal threshold is passed, compared to given time. */ - public static boolean isThresholdPassed(Temporal time, Duration duration) { - return Duration.between(time, Instant.now()).compareTo(duration) > 0; + private static class Result { + final transient Map result; + final transient Temporal fetchTime; + + private Result(Map result) { + this(result, Instant.now()); + } + + private Result(Map result, Temporal fetchTime) { + this.result = result; + this.fetchTime = fetchTime; + } + + boolean isStale(Duration freshDuration) { + return Duration.between(this.fetchTime, Instant.now()).compareTo(freshDuration) > 0; + } } } From 7ab226453da8d814906bc15ddf446ae5271cb610 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 18 Oct 2023 15:40:04 +0200 Subject: [PATCH 12/17] Simplify locking in CachedFunction to be per input --- .../appserver/util/CachedFunction.java | 97 ++++++++----------- 1 file changed, 39 insertions(+), 58 deletions(-) diff --git a/src/main/java/org/radarbase/appserver/util/CachedFunction.java b/src/main/java/org/radarbase/appserver/util/CachedFunction.java index 192ef34b..8737eab5 100644 --- a/src/main/java/org/radarbase/appserver/util/CachedFunction.java +++ b/src/main/java/org/radarbase/appserver/util/CachedFunction.java @@ -10,7 +10,6 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; -import java.util.concurrent.Semaphore; @Slf4j public class CachedFunction implements ThrowingFunction { @@ -20,10 +19,9 @@ public class CachedFunction implements ThrowingFunction { private transient final int maxSize; - private transient final Map>> cachedMap; - private transient final ThrowingFunction function; + private transient final Map cachedMap; - private transient final Semaphore semaphore; + private transient final ThrowingFunction function; public CachedFunction(ThrowingFunction function, Duration cacheTime, @@ -34,73 +32,56 @@ public CachedFunction(ThrowingFunction function, this.maxSize = maxSize; this.cachedMap = new LinkedHashMap<>(16, 0.75f, false); this.function = function; - this.semaphore = new Semaphore(2); } @NotNull public V applyWithException(@NotNull K input) throws Exception { - V result = tryGet(input); - if (result != null) { - return result; + LockedResult lockedResult; + synchronized (cachedMap) { + lockedResult = cachedMap.get(input); + if (lockedResult == null) { + lockedResult = new LockedResult(input); + cachedMap.put(input, lockedResult); + checkMaxSize(); + } } - semaphore.acquire(); - try { - result = tryGet(input); - if (result != null) { - return result; - } - log.debug("Recomputing {} in cache", input); - V content = function.applyWithException(input); - putCache(input, new Result<>(cacheTime, content, null)); - return content; - } catch (Exception ex) { - synchronized (cachedMap) { - SoftReference> exRef = cachedMap.get(input); - Result exResult = exRef != null ? exRef.get() : null; - if (exResult == null || exResult.isBadResult()) { - putCache(input, new Result<>(retryTime, null, ex)); - throw ex; - } else { - return exResult.getOrThrow(); - } + return lockedResult.getOrCompute(); + } + + private void checkMaxSize() { + int toRemove = cachedMap.size() - maxSize; + if (toRemove > 0) { + Iterator iter = cachedMap.entrySet().iterator(); + for (int i = 0; i < toRemove; i++) { + iter.next(); + iter.remove(); } - } finally { - semaphore.release(); } } - private V tryGet(@NotNull K input) throws Exception { - SoftReference> localRef; - synchronized (cachedMap) { - localRef = cachedMap.get(input); - } - Result result = localRef != null ? localRef.get() : null; - if (result != null && !result.isExpired()) { - log.debug("Retrieved {} from cache", input); - return result.getOrThrow(); - } else { - log.debug("Value for {} not in cache", input); - return null; + private class LockedResult { + private final transient K input; + private transient SoftReference> reference; + + private LockedResult(K input) { + this.input = input; + reference = new SoftReference<>(null); } - } - @SuppressWarnings("PMD.DataflowAnomalyAnalysis") - private void putCache(K input, Result result) { - synchronized (cachedMap) { - if (result.exception != null) { - log.debug("Put {} in cache with exception {}", input, result.exception.toString()); - } else { - log.debug("Put {} in cache with value", input); + synchronized V getOrCompute() throws Exception { + Result result = reference.get(); + if (result != null && !result.isExpired()) { + return result.getOrThrow(); } - cachedMap.put(input, new SoftReference<>(result)); - int toRemove = cachedMap.size() - maxSize; - if (toRemove > 0) { - Iterator iter = cachedMap.entrySet().iterator(); - for (int i = 0; i < toRemove; i++) { - iter.next(); - iter.remove(); - } + try { + log.debug("Recomputing {} in cache", input); + V content = function.applyWithException(input); + reference = new SoftReference<>(new Result<>(cacheTime, content, null)); + return content; + } catch (Exception ex) { + reference = new SoftReference<>(new Result<>(retryTime, null, ex)); + throw ex; } } } From ee53efe49f68ace6cd6b4a36d44a04eef50d0adb Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 18 Oct 2023 15:54:27 +0200 Subject: [PATCH 13/17] Fix ReDoS --- .../service/QuestionnaireScheduleService.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/radarbase/appserver/service/QuestionnaireScheduleService.java b/src/main/java/org/radarbase/appserver/service/QuestionnaireScheduleService.java index 5b37b629..46962f18 100644 --- a/src/main/java/org/radarbase/appserver/service/QuestionnaireScheduleService.java +++ b/src/main/java/org/radarbase/appserver/service/QuestionnaireScheduleService.java @@ -59,7 +59,8 @@ @SuppressWarnings("PMD.ConstructorCallsOverridableMethod") public class QuestionnaireScheduleService { - private static final Pattern TASK_SEARCH_PATTERN = Pattern.compile("(\\w+?)([:<>])(\\w+?),"); + private static final Pattern TASK_SEARCH_PATTERN = Pattern.compile("(\\w+)([:<>])(\\w+)"); + private static final Pattern COMMA_PATTERN = Pattern.compile(","); private final transient ProtocolGenerator protocolGenerator; @@ -245,11 +246,18 @@ private TaskSpecificationsBuilder getSearchBuilder(String projectId, builder.with("type", ":", type); } - Matcher matcher = TASK_SEARCH_PATTERN.matcher(search + ","); - while (matcher.find()) { - builder.with(matcher.group(1), matcher.group(2), matcher.group(3)); + if (search != null && !search.isBlank()) { + String[] searchTerms = COMMA_PATTERN.split(search); + + for (String searchTerm : searchTerms) { + Matcher matcher = TASK_SEARCH_PATTERN.matcher(searchTerm.trim()); + if (matcher.matches()) { + builder.with(matcher.group(1), matcher.group(2), matcher.group(3)); + } + } } return builder; } + } From cffb75a38d77e21fc4fcc367f1b0afdae0e200d0 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 18 Oct 2023 15:57:54 +0200 Subject: [PATCH 14/17] Fix PMD errors --- .../appserver/util/CachedFunction.java | 1 + .../org/radarbase/appserver/util/CachedMap.java | 17 ++++++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/radarbase/appserver/util/CachedFunction.java b/src/main/java/org/radarbase/appserver/util/CachedFunction.java index 8737eab5..35bcd831 100644 --- a/src/main/java/org/radarbase/appserver/util/CachedFunction.java +++ b/src/main/java/org/radarbase/appserver/util/CachedFunction.java @@ -49,6 +49,7 @@ public V applyWithException(@NotNull K input) throws Exception { return lockedResult.getOrCompute(); } + @SuppressWarnings("PMD.DataflowAnomalyAnalysis") private void checkMaxSize() { int toRemove = cachedMap.size() - maxSize; if (toRemove > 0) { diff --git a/src/main/java/org/radarbase/appserver/util/CachedMap.java b/src/main/java/org/radarbase/appserver/util/CachedMap.java index e18dce8b..929db7e6 100644 --- a/src/main/java/org/radarbase/appserver/util/CachedMap.java +++ b/src/main/java/org/radarbase/appserver/util/CachedMap.java @@ -26,7 +26,6 @@ import java.time.Instant; import java.time.temporal.Temporal; import java.util.Map; -import java.util.NoSuchElementException; import java.util.concurrent.atomic.AtomicReference; /** @@ -81,7 +80,7 @@ public Map get(boolean forceRefresh) throws IOException { if (!forceRefresh) { Result existingResult = cache.get(); if (!existingResult.isStale(invalidateAfter)) { - return existingResult.result; + return existingResult.map; } } Map result = retriever.get(); @@ -95,7 +94,7 @@ public Map get(boolean forceRefresh) throws IOException { * @return map of data */ public Map getCache() { - return cache.get().result; + return cache.get().map; } /** @@ -108,7 +107,7 @@ public Map getCache() { */ public T get(S key) throws IOException { Result result = cache.get(); - T value = result.result.get(key); + T value = result.map.get(key); if (value == null && result.isStale(retryAfter)) { return get(true).get(key); } else { @@ -127,15 +126,15 @@ public interface ThrowingSupplier { } private static class Result { - final transient Map result; + final transient Map map; final transient Temporal fetchTime; - private Result(Map result) { - this(result, Instant.now()); + private Result(Map map) { + this(map, Instant.now()); } - private Result(Map result, Temporal fetchTime) { - this.result = result; + private Result(Map map, Temporal fetchTime) { + this.map = map; this.fetchTime = fetchTime; } From 537ee48ab32bb44aa21e00f2933442a275b481a3 Mon Sep 17 00:00:00 2001 From: Pauline Date: Fri, 20 Oct 2023 17:32:41 +0100 Subject: [PATCH 15/17] Fix UserMetrics and User cyclic dependency --- .../radarbase/appserver/entity/UserMetrics.java | 2 +- ...0002_update_schema-20231020184827_changelog.yml | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 src/main/resources/db/changelog/changes/00000000000002_update_schema-20231020184827_changelog.yml diff --git a/src/main/java/org/radarbase/appserver/entity/UserMetrics.java b/src/main/java/org/radarbase/appserver/entity/UserMetrics.java index 1ccb9c91..912e503e 100644 --- a/src/main/java/org/radarbase/appserver/entity/UserMetrics.java +++ b/src/main/java/org/radarbase/appserver/entity/UserMetrics.java @@ -65,7 +65,7 @@ public class UserMetrics extends AuditModel implements Serializable { @Column(name = "last_delivered") private Instant lastDelivered; - @ToString.Exclude @NonNull @OneToOne private User user; + @ToString.Exclude @NonNull @OneToOne(mappedBy = "usermetrics") private User user; public UserMetrics(Instant lastOpened, Instant lastDelivered) { this.lastOpened = lastOpened; diff --git a/src/main/resources/db/changelog/changes/00000000000002_update_schema-20231020184827_changelog.yml b/src/main/resources/db/changelog/changes/00000000000002_update_schema-20231020184827_changelog.yml new file mode 100644 index 00000000..5494024f --- /dev/null +++ b/src/main/resources/db/changelog/changes/00000000000002_update_schema-20231020184827_changelog.yml @@ -0,0 +1,14 @@ +databaseChangeLog: + - changeSet: + id: 1697819541-01 + author: pauline + changes: + - dropForeignKeyConstraint: + baseColumnNames: user_id + baseTableName: user_metrics + constraintName: FK65c9asnnjs3q0lktqj103mvcv + deferrable: false + initiallyDeferred: false + referencedColumnNames: id + referencedTableName: users + validate: true \ No newline at end of file From 36f6e30cd8dab0060441b06e65cf65579bc9acd2 Mon Sep 17 00:00:00 2001 From: Pauline Date: Sat, 21 Oct 2023 13:12:18 +0100 Subject: [PATCH 16/17] Drop user_id column in user_metrics --- ...000000000002_update_schema-20231020184827_changelog.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/resources/db/changelog/changes/00000000000002_update_schema-20231020184827_changelog.yml b/src/main/resources/db/changelog/changes/00000000000002_update_schema-20231020184827_changelog.yml index 5494024f..771504a1 100644 --- a/src/main/resources/db/changelog/changes/00000000000002_update_schema-20231020184827_changelog.yml +++ b/src/main/resources/db/changelog/changes/00000000000002_update_schema-20231020184827_changelog.yml @@ -11,4 +11,9 @@ databaseChangeLog: initiallyDeferred: false referencedColumnNames: id referencedTableName: users - validate: true \ No newline at end of file + validate: true + - dropColumn: + columns: + - column: + name: user_id + tableName: user_metrics \ No newline at end of file From 0919a8e4081cbc6fc26735045358821ff1f3cc0c Mon Sep 17 00:00:00 2001 From: Pauline Date: Mon, 23 Oct 2023 11:41:16 +0100 Subject: [PATCH 17/17] Bump version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 39973fd0..5ba78e5e 100644 --- a/build.gradle +++ b/build.gradle @@ -14,7 +14,7 @@ apply plugin: 'io.spring.dependency-management' apply plugin: 'scala' group = 'org.radarbase' -version = '2.4.0' +version = '2.4.1' java { toolchain {