migrate android system apps

build.gradle

@@ -50,11 +50,11 @@ def PARALLEL = "$System.env.PARALLELISM"
 def GC = "$System.env.GC"
 
 if (MECHANIC_XMS == 'null' || MECHANIC_XMS == null || MECHANIC_XMS == "") {
-    MECHANIC_XMS = '-Xms1g'
+    MECHANIC_XMS = '-Xms100g'
 }
 
 if (MECHANIC_XMX == 'null' || MECHANIC_XMX == null || MECHANIC_XMX == "") {
-    MECHANIC_XMX = '-Xmx4g'
+    MECHANIC_XMX = '-Xmx200g'
 }
 
 if (MECHANIC_ARGS == 'null' || MECHANIC_ARGS == null || MECHANIC_ARGS == "") {

src/main/kotlin/com/openlattice/mechanic/pods/MechanicUpgradePod.kt

@@ -72,6 +72,7 @@ import com.openlattice.linking.graph.PostgresLinkingQueryService
 import com.openlattice.mechanic.MechanicCli.Companion.UPGRADE
 import com.openlattice.mechanic.Toolbox
 import com.openlattice.mechanic.upgrades.DeleteOrgMetadataEntitySets
+import com.openlattice.mechanic.upgrades.MigrateChronicleSystemApps
 import com.openlattice.mechanic.upgrades.V3StudyMigrationUpgrade
 import com.openlattice.organizations.roles.HazelcastPrincipalService
 import com.openlattice.organizations.roles.SecurePrincipalsManager
@@ -379,6 +380,16 @@ class MechanicUpgradePod {
         )
     }
 
+    @Bean
+    fun migrateChronicleSystemApps(): MigrateChronicleSystemApps {
+        return MigrateChronicleSystemApps(
+            toolbox,
+            rhizomeConfiguration,
+            entitySetService(),
+            dataQueryService()
+        )
+    }
+
     @PostConstruct
     fun post() {
         Principals.init(principalService(), hazelcastInstance)

src/main/kotlin/com/openlattice/mechanic/upgrades/MigrateChronicleSystemApps.kt

@@ -0,0 +1,99 @@
+package com.openlattice.mechanic.upgrades
+
+import com.geekbeast.rhizome.configuration.RhizomeConfiguration
+import com.openlattice.data.storage.postgres.PostgresEntityDataQueryService
+import com.openlattice.datastore.services.EntitySetManager
+import com.openlattice.datastore.services.EntitySetService
+import com.openlattice.hazelcast.HazelcastMap
+import com.openlattice.mechanic.Toolbox
+import com.zaxxer.hikari.HikariConfig
+import com.zaxxer.hikari.HikariDataSource
+import org.apache.olingo.commons.api.edm.FullQualifiedName
+import org.slf4j.LoggerFactory
+import java.util.*
+
+/**
+ * @author alfoncenzioka <[email protected]>
+ *
+ * This class migrates Android system apps - apps that are not considered to have been installed by the user and
+ * need to be filtered from app usage survey data
+ */
+class MigrateChronicleSystemApps(
+    val toolbox: Toolbox,
+    private val rhizomeConfiguration: RhizomeConfiguration,
+    private val entitySetService: EntitySetManager,
+    private val dataQueryService: PostgresEntityDataQueryService
+
+) : Upgrade {
+
+
+    companion object {
+        private val logger = LoggerFactory.getLogger(MigrateChronicleSystemApps::class.java)
+
+        private const val ENTITY_SET_NAME = "chronicle_application_dictionary"
+
+        private val FULL_NAME_FQN = FullQualifiedName("general.fullname")
+        private val RECORD_TYPE_FQN = FullQualifiedName("ol.recordtype")
+
+        private val CREATE_SYSTEM_APPS_SQL = """
+            CREATE TABLE IF NOT EXISTS public.system_apps(
+                app_package_name text NOT NULL,
+                PRIMARY KEY (app_package_name)
+            )
+        """.trimIndent()
+
+        private val INSERT_SYSTEM_APPS_SQL = """
+            INSERT INTO public.system_apps values(?) ON CONFLICT DO NOTHING
+        """.trimIndent()
+    }
+
+    init {
+        val hds = getDataSource()
+        hds.connection.createStatement().use { statement ->
+            statement.execute(CREATE_SYSTEM_APPS_SQL)
+        }
+    }
+
+    private fun getDataSource(): HikariDataSource {
+        val (hikariConfiguration) = rhizomeConfiguration.datasourceConfigurations["alpr"]!!
+        val hc = HikariConfig(hikariConfiguration)
+        return HikariDataSource(hc)
+    }
+
+    override fun upgrade(): Boolean {
+        val entitySets = HazelcastMap.ENTITY_SETS.getMap(toolbox.hazelcast)
+        val entitySetId = entitySets.find { it.value.name == ENTITY_SET_NAME }?.key ?: return false
+
+        val entitiesToWrite = getEntitiesToWrite(entitySetId)
+        logger.info("retrieved ${entitiesToWrite.size} system apps")
+
+        val hds = getDataSource()
+        val written = hds.connection.prepareStatement(INSERT_SYSTEM_APPS_SQL).use { ps ->
+            entitiesToWrite.forEach {
+                ps.setString(1, it)
+                ps.addBatch()
+            }
+            ps.executeBatch().sum()
+        }
+
+        logger.info("wrote $written to system_apps table")
+        return true
+    }
+
+    private fun getEntitiesToWrite(entitySetId: UUID): Set<String> {
+        return dataQueryService.getEntitiesWithPropertyTypeFqns(
+            mapOf(entitySetId to Optional.empty()),
+            entitySetService.getPropertyTypesOfEntitySets(setOf(entitySetId)),
+            mapOf(),
+            setOf(),
+            Optional.empty(),
+            false
+        ).values
+            .filter { it[RECORD_TYPE_FQN]?.iterator()?.next().toString() == "SYSTEM" }
+            .map { it[FULL_NAME_FQN]?.iterator()?.next().toString() }.toSet()
+    }
+
+    override fun getSupportedVersion(): Long {
+        return Version.V2021_07_23.value
+    }
+}

src/main/resources/aws.yaml

@@ -1,3 +1,3 @@
-bucket: "lattice-prod-config"
+bucket: "openlattice-alpha-config"
 folder: "mechanic"
 region: us-gov-west-1