diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/resourcetable/ResourceTableService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/resourcetable/ResourceTableService.java index 800c3fcae97c..186ada653ada 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/resourcetable/ResourceTableService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/resourcetable/ResourceTableService.java @@ -36,6 +36,9 @@ public interface ResourceTableService { /** Generates resource tables. */ void generateResourceTables(); + /** Replicates resource tables in the analytics database. */ + void replicateAnalyticsResourceTables(); + /** Generates data approval resource tables. */ void generateDataApprovalResourceTables(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java index 28ce71a55689..f3c684737771 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java @@ -45,6 +45,7 @@ import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; import org.hisp.dhis.analytics.cache.AnalyticsCache; import org.hisp.dhis.analytics.cache.OutliersCache; +import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.scheduling.JobProgress; import org.hisp.dhis.setting.SystemSettingsService; @@ -64,6 +65,8 @@ public class DefaultAnalyticsTableGenerator implements AnalyticsTableGenerator { private final SystemSettingsService settingsService; + private final AnalyticsTableSettings settings; + private final AnalyticsCache analyticsCache; private final OutliersCache outliersCache; @@ -93,6 +96,11 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr if (!params.isSkipResourceTables() && !params.isLatestUpdate()) { generateResourceTablesInternal(progress); + + if (settings.isAnalyticsDatabaseConfigured()) { + log.info("Replicating resource tables in analytics database"); + resourceTableService.replicateAnalyticsResourceTables(); + } } Set skipTypes = emptyIfNull(params.getSkipTableTypes()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/init/AnalyticsDatabaseInit.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/init/AnalyticsDatabaseInit.java new file mode 100644 index 000000000000..36e7ca4401de --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/init/AnalyticsDatabaseInit.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.table.init; + +import javax.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; +import org.hisp.dhis.db.model.Database; +import org.hisp.dhis.db.sql.SqlBuilder; +import org.hisp.dhis.db.sql.SqlBuilderProvider; +import org.hisp.dhis.external.conf.ConfigurationKey; +import org.hisp.dhis.external.conf.DhisConfigurationProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; + +/** + * Class responsible for performing work for initialization of an analytics database. + * + *

The following steps are required to introduce a new analytics database platform. + * + *

+ * + * @author Lars Helge Overland + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class AnalyticsDatabaseInit { + private final DhisConfigurationProvider config; + + private final AnalyticsTableSettings settings; + + @Qualifier("analyticsJdbcTemplate") + private final JdbcTemplate jdbcTemplate; + + private final SqlBuilder sqlBuilder; + + @PostConstruct + public void init() { + if (!config.isAnalyticsDatabaseConfigured()) { + return; + } + + Database database = settings.getAnalyticsDatabase(); + + switch (database) { + case DORIS -> initDoris(); + case POSTGRESQL -> initPostgreSql(); + } + + log.info("Initialized analytics database: '{}'", database); + } + + /** + * Work for initializing a Doris analytics database. Creates a JDBC catalog which is used to + * connect to and read from the PostgreSQL transaction database as an external data source. Read + * more at {@link https://t.ly/igk10}. + */ + private void initDoris() { + String connectionUrl = config.getProperty(ConfigurationKey.CONNECTION_URL); + String username = config.getProperty(ConfigurationKey.CONNECTION_USERNAME); + String password = config.getProperty(ConfigurationKey.CONNECTION_PASSWORD); + + jdbcTemplate.execute(sqlBuilder.dropCatalogIfExists()); + jdbcTemplate.execute(sqlBuilder.createCatalog(connectionUrl, username, password)); + } + + /** Work for initializing a PostgreSQL analytics database. */ + private void initPostgreSql() { + // No work at this point + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java index 0feed3788ee0..cfedae200dc4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java @@ -110,6 +110,14 @@ public void generateResourceTables() { } } + @Override + @Transactional + public void replicateAnalyticsResourceTables() { + for (ResourceTable table : getResourceTables()) { + resourceTableStore.replicateAnalyticsResourceTable(table); + } + } + @Override @Transactional public void generateDataApprovalResourceTables() { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/ResourceTableStore.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/ResourceTableStore.java index 944b08c69a95..9da1890abce1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/ResourceTableStore.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/ResourceTableStore.java @@ -37,4 +37,6 @@ public interface ResourceTableStore { * @param resourceTable the resource table. */ void generateResourceTable(ResourceTable resourceTable); + + void replicateAnalyticsResourceTable(ResourceTable resourceTable); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/jdbc/JdbcResourceTableStore.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/jdbc/JdbcResourceTableStore.java index 41fc8eb3a3cd..3e381995d4eb 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/jdbc/JdbcResourceTableStore.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/jdbc/JdbcResourceTableStore.java @@ -27,6 +27,7 @@ */ package org.hisp.dhis.resourcetable.jdbc; +import static java.lang.String.format; import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; import static org.hisp.dhis.commons.util.TextUtils.removeLastComma; @@ -46,6 +47,7 @@ import org.hisp.dhis.resourcetable.ResourceTableStore; import org.hisp.dhis.resourcetable.ResourceTableType; import org.hisp.dhis.system.util.Clock; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -59,9 +61,14 @@ public class JdbcResourceTableStore implements ResourceTableStore { private final AnalyticsTableHookService analyticsTableHookService; + private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); + private final JdbcTemplate jdbcTemplate; - private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); + private final SqlBuilder analyticsSqlBuilder; + + @Qualifier("analyticsJdbcTemplate") + private final JdbcTemplate analyticsJdbcTemplate; @Override public void generateResourceTable(ResourceTable resourceTable) { @@ -92,6 +99,58 @@ public void generateResourceTable(ResourceTable resourceTable) { log.info("Resource table update done: '{}' '{}'", tableName, clock.time()); } + @Override + public void replicateAnalyticsResourceTable(ResourceTable resourceTable) { + final Clock clock = new Clock().startClock(); + final Table table = resourceTable.getMainTable(); + final String tableName = table.getName(); + + dropAnalyticsTable(table); + + createAnalyticsTable(table); + + replicateAnalyticsTable(table); + + log.info("Analytics resource table replication done: '{}' '{}'", tableName, clock.time()); + } + + /** + * Drops the given analytics database table. + * + * @param table the {@link Table}. + */ + private void dropAnalyticsTable(Table table) { + String sql = analyticsSqlBuilder.dropTableIfExists(table); + log.info("Drop table SQL: '{}'", sql); + analyticsJdbcTemplate.execute(sql); + } + + /** + * Creates the given analytics database table. + * + * @param table the {@link Table}. + */ + private void createAnalyticsTable(Table table) { + String sql = analyticsSqlBuilder.createTable(table); + log.info("Create table SQL: '{}'", sql); + analyticsJdbcTemplate.execute(sql); + } + + /** + * Replicates the given table in the analytics database. + * + * @param table the {@link Table}. + */ + private void replicateAnalyticsTable(Table table) { + String sql = + format( + "insert into %s select * from %s", + analyticsSqlBuilder.quote(table.getName()), + analyticsSqlBuilder.qualifyTable(table.getName())); + log.info("Replicate table SQL: '{}'", sql); + analyticsJdbcTemplate.execute(sql); + } + /** * Drops the given table. * diff --git a/dhis-2/dhis-support/dhis-support-hibernate/pom.xml b/dhis-2/dhis-support/dhis-support-hibernate/pom.xml index 4ecae692427e..8592b7f81a94 100644 --- a/dhis-2/dhis-support/dhis-support-hibernate/pom.xml +++ b/dhis-2/dhis-support/dhis-support-hibernate/pom.xml @@ -94,6 +94,10 @@ org.postgresql postgresql + + com.mysql + mysql-connector-j + diff --git a/dhis-2/pom.xml b/dhis-2/pom.xml index 6c77ba576688..d065b3bc47fe 100644 --- a/dhis-2/pom.xml +++ b/dhis-2/pom.xml @@ -122,6 +122,7 @@ 6.0.0 42.7.4 2.5.1 + 8.3.0 1.10 @@ -1084,6 +1085,11 @@ postgis-jdbc ${postgis-jdbc.version} + + com.mysql + mysql-connector-j + ${mysql-connector-j.version} + @@ -2093,6 +2099,7 @@ jasperreports.version=${jasperreports.version} org.hibernate:hibernate-ehcache jakarta.xml.bind:jakarta.xml.bind-api + com.mysql:mysql-connector-j org.hisp.dhis:dhis-support-hibernate org.hisp.dhis:dhis-support-db-migration org.hisp.dhis:dhis-support-audit