feat: Analytics database [DHIS2-16705]

dhis-2/dhis-api/src/main/java/org/hisp/dhis/resourcetable/ResourceTableService.java

@@ -36,6 +36,9 @@ public interface ResourceTableService {
   /** Generates resource tables. */
   void generateResourceTables();
 
+  /** Replicates resource tables in the analytics database. */
+  void replicateAnalyticsResourceTables();
+
   /** Generates data approval resource tables. */
   void generateDataApprovalResourceTables();
 

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java

@@ -45,6 +45,7 @@
 import org.hisp.dhis.analytics.AnalyticsTableUpdateParams;
 import org.hisp.dhis.analytics.cache.AnalyticsCache;
 import org.hisp.dhis.analytics.cache.OutliersCache;
+import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings;
 import org.hisp.dhis.resourcetable.ResourceTableService;
 import org.hisp.dhis.scheduling.JobProgress;
 import org.hisp.dhis.setting.SystemSettingsService;
@@ -64,6 +65,8 @@ public class DefaultAnalyticsTableGenerator implements AnalyticsTableGenerator {
 
   private final SystemSettingsService settingsService;
 
+  private final AnalyticsTableSettings settings;
+
   private final AnalyticsCache analyticsCache;
 
   private final OutliersCache outliersCache;
@@ -93,6 +96,11 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr
 
     if (!params.isSkipResourceTables() && !params.isLatestUpdate()) {
       generateResourceTablesInternal(progress);
+
+      if (settings.isAnalyticsDatabaseConfigured()) {
+        log.info("Replicating resource tables in analytics database");
+        resourceTableService.replicateAnalyticsResourceTables();
+      }
     }
 
     Set<AnalyticsTableType> skipTypes = emptyIfNull(params.getSkipTableTypes());

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/init/AnalyticsDatabaseInit.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2004-2024, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.analytics.table.init;
+
+import javax.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings;
+import org.hisp.dhis.db.model.Database;
+import org.hisp.dhis.db.sql.SqlBuilder;
+import org.hisp.dhis.db.sql.SqlBuilderProvider;
+import org.hisp.dhis.external.conf.ConfigurationKey;
+import org.hisp.dhis.external.conf.DhisConfigurationProvider;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Component;
+
+/**
+ * Class responsible for performing work for initialization of an analytics database.
+ *
+ * <p>The following steps are required to introduce a new analytics database platform.
+ *
+ * <ul>
+ *   <li>Add value to enum {@link Database}
+ *   <li>Add implementation of interface {@link SqlBuilder}
+ *   <li>Add entry to switch statement in {@link SqlBuilderProvider}
+ *   <li>Add method to {@link AnalyticsDatabaseInit} if necessary
+ * </ul>
+ *
+ * @author Lars Helge Overland
+ */
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class AnalyticsDatabaseInit {
+  private final DhisConfigurationProvider config;
+
+  private final AnalyticsTableSettings settings;
+
+  @Qualifier("analyticsJdbcTemplate")
+  private final JdbcTemplate jdbcTemplate;
+
+  private final SqlBuilder sqlBuilder;
+
+  @PostConstruct
+  public void init() {
+    if (!config.isAnalyticsDatabaseConfigured()) {
+      return;
+    }
+
+    Database database = settings.getAnalyticsDatabase();
+
+    switch (database) {
+      case DORIS -> initDoris();
+      case POSTGRESQL -> initPostgreSql();
+    }
+
+    log.info("Initialized analytics database: '{}'", database);
+  }
+
+  /**
+   * Work for initializing a Doris analytics database. Creates a JDBC catalog which is used to
+   * connect to and read from the PostgreSQL transaction database as an external data source. Read
+   * more at {@link https://t.ly/igk10}.
+   */
+  private void initDoris() {
+    String connectionUrl = config.getProperty(ConfigurationKey.CONNECTION_URL);
+    String username = config.getProperty(ConfigurationKey.CONNECTION_USERNAME);
+    String password = config.getProperty(ConfigurationKey.CONNECTION_PASSWORD);
+
+    jdbcTemplate.execute(sqlBuilder.dropCatalogIfExists());
+    jdbcTemplate.execute(sqlBuilder.createCatalog(connectionUrl, username, password));
+  }
+
+  /** Work for initializing a PostgreSQL analytics database. */
+  private void initPostgreSql() {
+    // No work at this point
+  }
+}

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java

@@ -110,6 +110,14 @@ public void generateResourceTables() {
     }
   }
 
+  @Override
+  @Transactional
+  public void replicateAnalyticsResourceTables() {
+    for (ResourceTable table : getResourceTables()) {
+      resourceTableStore.replicateAnalyticsResourceTable(table);
+    }
+  }
+
   @Override
   @Transactional
   public void generateDataApprovalResourceTables() {

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/ResourceTableStore.java

@@ -37,4 +37,6 @@ public interface ResourceTableStore {
    * @param resourceTable the resource table.
    */
   void generateResourceTable(ResourceTable resourceTable);
+
+  void replicateAnalyticsResourceTable(ResourceTable resourceTable);
 }

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/jdbc/JdbcResourceTableStore.java

@@ -27,6 +27,7 @@
  */
 package org.hisp.dhis.resourcetable.jdbc;
 
+import static java.lang.String.format;
 import static org.apache.commons.collections4.CollectionUtils.isNotEmpty;
 import static org.hisp.dhis.commons.util.TextUtils.removeLastComma;
 
@@ -46,6 +47,7 @@
 import org.hisp.dhis.resourcetable.ResourceTableStore;
 import org.hisp.dhis.resourcetable.ResourceTableType;
 import org.hisp.dhis.system.util.Clock;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Service;
 
@@ -59,9 +61,14 @@ public class JdbcResourceTableStore implements ResourceTableStore {
 
   private final AnalyticsTableHookService analyticsTableHookService;
 
+  private final SqlBuilder sqlBuilder = new PostgreSqlBuilder();
+
   private final JdbcTemplate jdbcTemplate;
 
-  private final SqlBuilder sqlBuilder = new PostgreSqlBuilder();
+  private final SqlBuilder analyticsSqlBuilder;
+
+  @Qualifier("analyticsJdbcTemplate")
+  private final JdbcTemplate analyticsJdbcTemplate;
 
   @Override
   public void generateResourceTable(ResourceTable resourceTable) {
@@ -92,6 +99,58 @@ public void generateResourceTable(ResourceTable resourceTable) {
     log.info("Resource table update done: '{}' '{}'", tableName, clock.time());
   }
 
+  @Override
+  public void replicateAnalyticsResourceTable(ResourceTable resourceTable) {
+    final Clock clock = new Clock().startClock();
+    final Table table = resourceTable.getMainTable();
+    final String tableName = table.getName();
+
+    dropAnalyticsTable(table);
+
+    createAnalyticsTable(table);
+
+    replicateAnalyticsTable(table);
+
+    log.info("Analytics resource table replication done: '{}' '{}'", tableName, clock.time());
+  }
+
+  /**
+   * Drops the given analytics database table.
+   *
+   * @param table the {@link Table}.
+   */
+  private void dropAnalyticsTable(Table table) {
+    String sql = analyticsSqlBuilder.dropTableIfExists(table);
+    log.info("Drop table SQL: '{}'", sql);
+    analyticsJdbcTemplate.execute(sql);
+  }
+
+  /**
+   * Creates the given analytics database table.
+   *
+   * @param table the {@link Table}.
+   */
+  private void createAnalyticsTable(Table table) {
+    String sql = analyticsSqlBuilder.createTable(table);
+    log.info("Create table SQL: '{}'", sql);
+    analyticsJdbcTemplate.execute(sql);
+  }
+
+  /**
+   * Replicates the given table in the analytics database.
+   *
+   * @param table the {@link Table}.
+   */
+  private void replicateAnalyticsTable(Table table) {
+    String sql =
+        format(
+            "insert into %s select * from %s",
+            analyticsSqlBuilder.quote(table.getName()),
+            analyticsSqlBuilder.qualifyTable(table.getName()));
+    log.info("Replicate table SQL: '{}'", sql);
+    analyticsJdbcTemplate.execute(sql);
+  }
+
   /**
    * Drops the given table.
    *

dhis-2/dhis-support/dhis-support-hibernate/pom.xml

@@ -94,6 +94,10 @@
       <groupId>org.postgresql</groupId>
       <artifactId>postgresql</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.mysql</groupId>
+      <artifactId>mysql-connector-j</artifactId>
+    </dependency>
 
     <!-- Test -->
 

dhis-2/pom.xml

@@ -122,6 +122,7 @@
     <HikariCP.version>6.0.0</HikariCP.version>
     <postgresql-driver.version>42.7.4</postgresql-driver.version>
     <postgis-jdbc.version>2.5.1</postgis-jdbc.version>
+    <mysql-connector-j.version>8.3.0</mysql-connector-j.version>
     <datasource-proxy.version>1.10</datasource-proxy.version>
 
     <!-- Servlet -->
@@ -1084,6 +1085,11 @@
         <artifactId>postgis-jdbc</artifactId>
         <version>${postgis-jdbc.version}</version>
       </dependency>
+      <dependency>
+        <groupId>com.mysql</groupId>
+        <artifactId>mysql-connector-j</artifactId>
+        <version>${mysql-connector-j.version}</version>
+      </dependency>
 
       <!-- Flyway -->
       <dependency>
@@ -2093,6 +2099,7 @@ jasperreports.version=${jasperreports.version}
               <ignoredUnusedDeclaredDependency>org.hibernate:hibernate-ehcache</ignoredUnusedDeclaredDependency>
               <ignoredUnusedDeclaredDependency>jakarta.xml.bind:jakarta.xml.bind-api</ignoredUnusedDeclaredDependency>
               <!-- Required only at runtime -->
+              <ignoredUnusedDeclaredDependency>com.mysql:mysql-connector-j</ignoredUnusedDeclaredDependency>
               <ignoredUnusedDeclaredDependency>org.hisp.dhis:dhis-support-hibernate</ignoredUnusedDeclaredDependency>
               <ignoredUnusedDeclaredDependency>org.hisp.dhis:dhis-support-db-migration</ignoredUnusedDeclaredDependency>
               <ignoredUnusedDeclaredDependency>org.hisp.dhis:dhis-support-audit</ignoredUnusedDeclaredDependency>