Add statistics control table, fix cellName in some statistics, improv…

…e docker support, remove test web server (#125)

* Recreate docker compose so it handles the database

* Update other instructions according to the new docker approach

* Add statistics control table

* Save error message when generating statistics

* Save statistics control status

* Update docker compose version to the next one (make github actions happy)

* Fix oldest standing record query

* Run tests in random port

* Do not rely on the test web server anymore, mock it instead

* Fix tests

* Avoid downloading database again if not needed

.github/workflows/backtest.yaml

@@ -32,7 +32,7 @@ jobs:
       - name: Validate Gradle wrapper
         uses: gradle/wrapper-validation-action@v1
       - name: Run containers
-        run: docker-compose -f server/docker-compose-test.yaml up -d
+        run: docker compose -f server/docker-compose-test.yaml up -d
       - name: Wait for database to start
         run: |
           for i in `seq 1 60`;

README.md

@@ -30,7 +30,7 @@ source server/get_db_export.sh
 
 - Run docker
 
-`docker-compose up`
+`docker compose up`
 
 ## Generate all statistics
 

docker-compose.yml

@@ -1,16 +1,41 @@
-version: "3.7"
+version: "3.8"
 
 services:
-  server:
-    build:
-      context: server
-      dockerfile: Dockerfile
-    network_mode: host
+  mysql:
+    image: mysql:8.0
+    environment:
+      MYSQL_ALLOW_EMPTY_PASSWORD: "true"
+      MYSQL_DATABASE: wca_development
+    ports:
+      - "3306:3306"
+    volumes:
+      - mysql-data:/var/lib/mysql
 
-  client:
-    build:
-      context: client
-      dockerfile: Dockerfile
+  db_restore:
+    image: ubuntu:20.04
     depends_on:
-      - server
-    network_mode: host
+      - mysql
+    entrypoint: >
+      sh -c "
+      apt update && apt install curl unzip mysql-client wget -y &&
+      result=$(mysql -h mysql -sN -u root -e 'use wca_development; select datediff(now(), max(results_posted_at)) from Competitions;'  || echo 999) &&
+      echo 'Days since last update: ' $result &&
+      if [ -n \"$result\" ] && [ $result -lt 7 ]; then
+        echo 'Database is already up to date, skipping restoration.'
+        exit 0
+      else
+        echo 'Database is outdated, restoring...'
+      fi &&
+      echo 'Downloading WCA database dump, this may take a while...' &&
+      wget -q https://www.worldcubeassociation.org/wst/wca-developer-database-dump.zip &&
+      rm -rf wca-developer-database-dump.sql &&
+      unzip wca-developer-database-dump.zip &&
+      echo 'Restoring WCA database. This also may take a while...' &&
+      mysql -h mysql -u root -e 'drop database if exists wca_development; create database wca_development; use wca_development; source wca-developer-database-dump.sql;' &&
+      echo 'Restoration complete!'
+      "
+    volumes:
+      - mysql-data:/var/lib/mysql
+
+volumes:
+  mysql-data:

server/README.md

@@ -10,34 +10,11 @@
 
 ### Using docker
 
-If you are in the `server` folder, you can just run `docker-compose up -d`. This will spin up an empty MySQL database running on port 3306.
+If you are in the root folder, you can just run `docker compose up -d`. This will spin up an empty MySQL database running on port 3306, it will also download the latest WCA dump and apply it for you.
 
 ### Your own local copy of WCA's database
 
-You can also use an internal database for handling WCA data.
-
-In case you do not have it installed yet, you will need to get MySQL.
-
-- Install [MySQL 8.0](https://dev.mysql.com/doc/refman/8.0/en/linux-installation.html), and set it up with a user with username "root" with an empty password.
-
-```
-sudo mysql -u root
-ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY '';
-
-create database wca_development;
-```
-
-The database `wca_development` will be populated with WCA data. If you want to change password, username or others, make sure to also change on `application-local.properties`.
-
-## Before you run this
-
-You need your copy of the database from WCA. If you already have it (with a user 'root' with no password), you can skip this.
-
-Download [the latest export](https://www.worldcubeassociation.org/wst/wca-developer-database-dump.zip) and execute the sql (as stated in the last step). If you wish, you can execute the file `get_db_export.sh` in the scripts folder.
-
-From the root folder, use
-
-    source scripts/get_db_export.sh
+If you are familiar with the development for the WCA ecosystem, it is likely that you have your own database running. You can reuse that. Just make sure that the connection string in `server/src/main/resources/application-local.yaml` fits your case.
 
 ## How to run it
 
@@ -79,13 +56,13 @@ The `-d` part means "detached", so you'll have to stop by killing the process ru
 
 ## Tests
 
-This backend project uses integration tests so we need to actually connect to a database and fetch data from somewhere.
+This backend project uses integration tests so we need to actually connect to a database. We mock WCA profile response.
 
 - Preparing the database for tests (from the repository root)
 
-  `docker-compose -f server/docker-compose-test.yaml up -d`
+  `docker compose -f server/docker-compose-test.yaml up -d`
 
-This will start the database (port 3307) and also a mocked version of the WCA's api (for getting user info) on port 3500.
+This will start the database (port 3307).
 
 - Run the tests
 
@@ -96,8 +73,8 @@ In a new terminal, from the repository root, run
 - If you need to change migrations, run
 
   ```
-  docker-compose -f server/docker-compose-test.yaml down --volumes
-  docker-compose -f server/docker-compose-test.yaml up -d
+  docker compose -f server/docker-compose-test.yaml down --volumes
+  docker compose -f server/docker-compose-test.yaml up -d
   ```
 
 ## Code format

server/docker-compose-test.yaml

@@ -1,19 +1,15 @@
 version: "3.1"
 services:
   wca-test-db:
-    image: "mysql:8.0.26"
+    image: mysql:8.0
     environment:
       MYSQL_ALLOW_EMPTY_PASSWORD: "true"
       MYSQL_DATABASE: wca_development
     ports:
       - "3307:3306"
     volumes:
       - "./db/migration:/docker-entrypoint-initdb.d/:ro"
-  wca-mock-server:
-    image: "node:16-bullseye-slim"
-    volumes:
-      - "./test-web-server/data.js/:/data.js"
-      - "./test-web-server/package.json/:/package.json"
-      - "./test-web-server/server.js/:/server.js"
-    entrypoint: sh -c "npm install;npm start"
-    network_mode: host
+      - wca-test-db-data:/var/lib/mysql
+
+volumes:
+  wca-test-db-data:

server/src/main/java/org/worldcubeassociation/statistics/api/WCAApi.java

@@ -1,6 +1,7 @@
 package org.worldcubeassociation.statistics.api;
 
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
@@ -14,7 +15,9 @@
 @Slf4j
 @Component
 public class WCAApi {
-    private static final RestTemplate REST_TEMPLATE = new RestTemplate();
+
+    @Autowired
+    private RestTemplate restTemplate;
 
     @Value("${api.wca.baseurl}")
     private String wcaBaseUrl;
@@ -30,7 +33,7 @@ public UserInfoDTO getUserInfo(String token) {
         headers.set("Content-Type", "application/json");
         HttpEntity<String> entity = new HttpEntity<>("body", headers);
         ResponseEntity<UserInfoWrapperDTO> response =
-                REST_TEMPLATE.exchange(url, HttpMethod.GET, entity, UserInfoWrapperDTO.class);
+            restTemplate.exchange(url, HttpMethod.GET, entity, UserInfoWrapperDTO.class);
         return response.getBody().getMe();
     }
 }

server/src/main/java/org/worldcubeassociation/statistics/config/RestTemplateConfig.java

@@ -0,0 +1,14 @@
+package org.worldcubeassociation.statistics.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.client.RestTemplate;
+
+@Configuration
+public class RestTemplateConfig {
+
+    @Bean
+    public RestTemplate restTemplate() {
+        return new RestTemplate();
+    }
+}

server/src/main/java/org/worldcubeassociation/statistics/enums/StatisticsControlStatus.java

@@ -0,0 +1,7 @@
+package org.worldcubeassociation.statistics.enums;
+
+public enum StatisticsControlStatus {
+    STARTED,
+    COMPLETED,
+    FAILED
+}

server/src/main/java/org/worldcubeassociation/statistics/model/StatisticsControl.java

@@ -0,0 +1,37 @@
+package org.worldcubeassociation.statistics.model;
+
+import java.time.LocalDateTime;
+import java.util.List;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
+import org.hibernate.annotations.Type;
+import org.worldcubeassociation.statistics.dto.StatisticsGroupResponseDTO;
+import org.worldcubeassociation.statistics.enums.DisplayModeEnum;
+
+@Getter
+@Setter
+@Entity
+@EqualsAndHashCode(callSuper = true)
+@Table(name = "statistics_control")
+public class StatisticsControl extends BaseEntity {
+    @Id
+    private String path;
+
+    private String message;
+
+    private String status;
+
+    @Column(name = "created_at")
+    private LocalDateTime createdAt;
+
+    @Column(name = "completed_at")
+    private LocalDateTime completedAt;
+}

server/src/main/java/org/worldcubeassociation/statistics/repository/StatisticsControlRepository.java

@@ -0,0 +1,10 @@
+package org.worldcubeassociation.statistics.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+import org.worldcubeassociation.statistics.model.StatisticsControl;
+
+@Repository
+public interface StatisticsControlRepository extends JpaRepository<StatisticsControl, String> {
+
+}

server/src/main/java/org/worldcubeassociation/statistics/service/impl/StatisticsServiceImpl.java

@@ -18,7 +18,6 @@
 import javax.validation.Valid;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.ResourceLoader;
 import org.springframework.core.io.support.ResourcePatternUtils;
@@ -34,8 +33,11 @@
 import org.worldcubeassociation.statistics.dto.StatisticsRequestDTO;
 import org.worldcubeassociation.statistics.dto.StatisticsResponseDTO;
 import org.worldcubeassociation.statistics.enums.DisplayModeEnum;
+import org.worldcubeassociation.statistics.enums.StatisticsControlStatus;
 import org.worldcubeassociation.statistics.exception.NotFoundException;
 import org.worldcubeassociation.statistics.model.Statistics;
+import org.worldcubeassociation.statistics.model.StatisticsControl;
+import org.worldcubeassociation.statistics.repository.StatisticsControlRepository;
 import org.worldcubeassociation.statistics.repository.StatisticsRepository;
 import org.worldcubeassociation.statistics.service.DatabaseQueryService;
 import org.worldcubeassociation.statistics.service.StatisticsService;
@@ -45,17 +47,11 @@
 @Service
 public class StatisticsServiceImpl implements StatisticsService {
 
-    @Autowired
-    private DatabaseQueryService databaseQueryService;
-
-    @Autowired
-    private ResourceLoader resourceLoader;
-
-    @Autowired
-    private StatisticsRepository statisticsRepository;
-
-    @Autowired
-    private ObjectMapper objectMapper;
+    private final DatabaseQueryService databaseQueryService;
+    private final ResourceLoader resourceLoader;
+    private final StatisticsRepository statisticsRepository;
+    private final ObjectMapper objectMapper;
+    private final StatisticsControlRepository statisticsControlRepository;
 
     private static final Yaml YAML = new Yaml();
 
@@ -65,6 +61,16 @@ public class StatisticsServiceImpl implements StatisticsService {
     private static final String MAIN_LIST_CACHE = "MAIN_LIST_CACHE";
     private static final int CACHING_TIME = 6;
 
+    public StatisticsServiceImpl(DatabaseQueryService databaseQueryService,
+        ResourceLoader resourceLoader, StatisticsRepository statisticsRepository,
+        ObjectMapper objectMapper, StatisticsControlRepository statisticsControlRepository) {
+        this.databaseQueryService = databaseQueryService;
+        this.resourceLoader = resourceLoader;
+        this.statisticsRepository = statisticsRepository;
+        this.objectMapper = objectMapper;
+        this.statisticsControlRepository = statisticsControlRepository;
+    }
+
     @Override
     public StatisticsResponseDTO sqlToStatistics(StatisticsRequestDTO statisticsRequestDTO) {
         log.info("SQL to statistics for {}", statisticsRequestDTO);
@@ -172,11 +178,28 @@ private void resourcesToStatistics(List<Resource> resources) throws IOException
 
             log.info("Statistic {}", resource.getDescription());
 
-            InputStream inputStream = resource.getInputStream();
+            var statisticsControl = new StatisticsControl();
+            statisticsControl.setPath(resource.getFilename());
+            statisticsControl.setCreatedAt(LocalDateTime.now());
+            statisticsControl.setStatus(StatisticsControlStatus.STARTED.name());
+            statisticsControlRepository.save(statisticsControl);
+
+            try {
+                InputStream inputStream = resource.getInputStream();
 
-            StatisticsRequestDTO request = YAML.loadAs(inputStream, StatisticsRequestDTO.class);
+                StatisticsRequestDTO request = YAML.loadAs(inputStream, StatisticsRequestDTO.class);
 
-            sqlToStatistics(request);
+                sqlToStatistics(request);
+
+                statisticsControl.setCompletedAt(LocalDateTime.now());
+                statisticsControl.setStatus(StatisticsControlStatus.COMPLETED.name());
+                statisticsControlRepository.save(statisticsControl);
+            } catch (Exception e) {
+                log.error("Error while processing {}", resource.getFilename(), e);
+                statisticsControl.setMessage(StringUtils.abbreviate(e.getMessage(), 200));
+                statisticsControl.setStatus(StatisticsControlStatus.FAILED.name());
+                statisticsControlRepository.save(statisticsControl);
+            }
         }
     }
 
@@ -279,6 +302,7 @@ public StatisticsResponseDTO create(@Valid StatisticsDTO statisticsDTO) {
     public void deleteAll() {
         log.info("Delete all statistics");
         statisticsRepository.deleteAll();
+        statisticsControlRepository.deleteAll();
         log.info("Deleted");
     }
 

server/src/main/resources/schema.sql

@@ -12,6 +12,15 @@ create table if not exists statistics (
     primary key (path)
 );
 
+create table if not exists statistics_control (
+    path varchar(100),
+    status varchar(20) not null,
+    message varchar(200),
+    created_at datetime not null,
+    completed_at datetime,
+    primary key (path)
+);
+
 create table if not exists best_ever_rank (
     person_id varchar(10),
     best_ever_rank json not null,

server/src/main/resources/statistics-request-list/best-medal-collection.yml

@@ -69,6 +69,7 @@ queries:
           medals_rank
       where
           medal_rank <= 20
+      order by golds desc, silvers desc, bronzes desc
     sqlQueryCustom: |
       select
           personName,