diff --git a/analytics-core/pom.xml b/analytics-core/pom.xml index fdcca52..fc6a228 100644 --- a/analytics-core/pom.xml +++ b/analytics-core/pom.xml @@ -16,6 +16,9 @@ 11 UTF-8 1.1.1 + org.sunbird + cloud-store-sdk_2.12 + 1.4.6 diff --git a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CSPUtils.scala b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CSPUtils.scala index a0d1736..db02ac5 100644 --- a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CSPUtils.scala +++ b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CSPUtils.scala @@ -65,5 +65,9 @@ object OCIProvider extends ICloudStorageProvider { JobLogger.log("Configuring OCI Access Key & Secret Key to SparkContext") sc.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", key); sc.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", secret); + val storageEndpoint = AppConf.getConfig("cloud_storage_endpoint_with_protocol") + if (storageEndpoint.nonEmpty) { + sc.hadoopConfiguration.set("fs.s3n.endpoint", storageEndpoint) + } } } \ No newline at end of file diff --git a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala index 2e02a07..c692742 100644 --- a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala +++ b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala @@ -16,11 +16,9 @@ import org.ekstep.analytics.framework.Level._ import org.ekstep.analytics.framework.Period._ import org.ekstep.analytics.framework.util.CloudStorageProviders.setSparkCSPConfigurations import org.ekstep.analytics.framework.{DtRange, Event, JobConfig, _} - import scala.collection.mutable.ListBuffer //import org.ekstep.analytics.framework.conf.AppConf import java.util.zip.{ZipEntry, ZipOutputStream} - import org.apache.commons.lang3.StringUtils import org.joda.time.format.{DateTimeFormat, DateTimeFormatter} import org.joda.time.{DateTime, DateTimeZone, Days, LocalDate, Weeks, Years} @@ -91,7 +89,9 @@ object CommonUtil { } val sc = new SparkContext(conf) - setSparkCSPConfigurations(sc, AppConf.getConfig("cloud_storage_type"), None, None) + val key = AppConf.getConfig("storage.key.config") + val secret = AppConf.getConfig("storage.secret.config") + setSparkCSPConfigurations(sc, AppConf.getConfig("cloud_storage_type"), Option(key), Option(secret)) JobLogger.log("Spark Context initialized") sc }