[COST-5368] POC - Glue

.env.example

@@ -32,12 +32,6 @@ TAG_ENABLED_LIMIT=200 # Set the max amount of tags per account
 DELAYED_TASK_TIME=30 # Set the seconds before a delayed summary task should expire
 DELAYED_TASK_POLLING_MINUTES=5 # Set the time before the delayed task kick off.
 
-ENABLE_S3_ARCHIVING=True
-S3_BUCKET_NAME=koku-bucket
-S3_BUCKET_PATH=data
-S3_ENDPOINT=http://koku-minio:9000
-S3_ACCESS_KEY=kokuminioaccess
-S3_SECRET=kokuminiosecret
 SKIP_MINIO_DATA_DELETION=False
 
 # AWS
@@ -66,7 +60,6 @@ INSIGHTS_KAFKA_PORT=29092
 KEEPDB=True
 PROMETHEUS_PUSHGATEWAY="pushgateway:9091"
 AUTO_DATA_INGEST=True
-DEVELOPMENT_IDENTITY='{"identity": {"account_number": "10001", "org_id": "1234567", "type": "User", "user": {"username": "user_dev", "email": "[email protected]", "is_org_admin": "True", "access": {}}},"entitlements": {"cost_management": {"is_entitled": "True"}}}'
 CACHED_VIEWS_DISABLED=False
 ACCOUNT_ENHANCED_METRICS=False
 ENHANCED_ORG_ADMIN=True
@@ -76,3 +69,22 @@ QE_SCHEMA=""
 
 # USER_ID=   # Set to the value returned by id -u
 # GROUP_ID=  # Set to the value returned by id -g
+
+# Minio
+# S3_BUCKET_NAME=koku-bucket
+# S3_ENDPOINT=http://koku-minio:9000
+# S3_ACCESS_KEY=kokuminioaccess
+# S3_SECRET=kokuminiosecret
+
+DEVELOPMENT_IDENTITY='{"identity": {"account_number": "10001", "org_id": "1234567", "type": "User", "user": {"username": "user_dev", "email": "[email protected]", "is_org_admin": "True", "access": {}}},"entitlements": {"cost_management": {"is_entitled": "True"}}}'
+
+# Glue
+SCHEMA_SUFFIX=""
+
+AWS_CATALOG_ID=589173575009
+S3_ENDPOINT=https://s3.us-east-1.amazonaws.com
+
+S3_BUCKET_NAME=CHANGEME
+S3_ACCESS_KEY=CHANGEME
+S3_SECRET=CHANGEME
+S3_REGION=us-east-1

Jenkinsfile

@@ -42,7 +42,7 @@ pipeline {
 
         IQE_PLUGINS="cost_management"
         BUILD_URL="https://ci.ext.devshift.net/job/koku-pipeline-pr-check-main/${BUILD_NUMBER}/"
-        IQE_ENV_VARS="JOB_NAME=koku-ci-jenkins,BUILD_NUMBER=${BUILD_NUMBER},BUILD_URL=${BUILD_URL}"
+        IQE_ENV_VARS="JOB_NAME=koku-ci-jenkins,BUILD_NUMBER=${BUILD_NUMBER},BUILD_URL=${BUILD_URL},SCHEMA_SUFFIX=_${IMAGE_TAG}"
         IQE_PARALLEL_ENABLED="false"
 
         GITHUB_API_ROOT='https://api.github.com/repos/project-koku/koku'

Makefile

@@ -98,6 +98,7 @@ help:
 	@echo "  requirements                          generate Pipfile.lock"
 	@echo "  clowdapp                              generates a new clowdapp.yaml"
 	@echo "  delete-db                             delete local directory $(TOPDIR)/dev/containers/postgresql/data"
+	@echo "  delete-glue-data                      delete s3 files + database created in AWS/glue"
 	@echo "  delete-test-db                        delete the django test db"
 	@echo "  reset-db-statistics                   clear the pg_stat_statements statistics"
 	@echo "  run-migrations                        run migrations against database"
@@ -216,6 +217,9 @@ make-migrations:
 delete-db:
 	@$(PREFIX) rm -rf $(TOPDIR)/dev/containers/postgresql/data/
 
+delete-glue-data:
+	@$(PYTHON) $(SCRIPTDIR)/clean_glue.py $(schema)
+
 delete-test-db:
 	@PGPASSWORD=$$DATABASE_PASSWORD psql -h $$POSTGRES_SQL_SERVICE_HOST \
                                          -p $$POSTGRES_SQL_SERVICE_PORT \
@@ -338,7 +342,10 @@ docker-up-no-build: docker-up-db
 # basic dev environment targets
 docker-up-min: docker-build docker-up-min-no-build
 
-docker-up-min-no-build: docker-host-dir-setup docker-up-db
+# docker-up-min-no-build: docker-host-dir-setup docker-up-db
+# 	$(DOCKER_COMPOSE) up -d --scale koku-worker=$(scale) redis koku-server masu-server koku-worker trino hive-metastore
+
+docker-up-min-no-build: docker-host-dir-setup
 	$(DOCKER_COMPOSE) up -d --scale koku-worker=$(scale) redis koku-server masu-server koku-worker trino hive-metastore
 
 # basic dev environment targets

ci/functions.sh

@@ -94,37 +94,44 @@ function run_smoke_tests_stage() {
     source ${CICD_ROOT}/_common_deploy_logic.sh
     export NAMESPACE=$(bonfire namespace reserve --duration ${RESERVATION_TIMEOUT})
 
-    oc get secret/koku-aws -o json -n ephemeral-base | jq -r '.data' > aws-creds.json
-    oc get secret/koku-gcp -o json -n ephemeral-base | jq -r '.data' > gcp-creds.json
-    oc get secret/koku-oci -o json -n ephemeral-base | jq -r '.data' > oci-creds.json
+    oc get secret koku-aws -o yaml -n ephemeral-base | grep -v '^\s*namespace:\s' | oc apply --namespace=${NAMESPACE} -f -
+    oc get secret koku-gcp -o yaml -n ephemeral-base | grep -v '^\s*namespace:\s' | oc apply --namespace=${NAMESPACE} -f -
+    oc get secret koku-oci -o yaml -n ephemeral-base | grep -v '^\s*namespace:\s' | oc apply --namespace=${NAMESPACE} -f -
 
-    AWS_CREDENTIALS_EPH=$(jq -r '."aws-credentials"' < aws-creds.json)
-    GCP_CREDENTIALS_EPH=$(jq -r '."gcp-credentials"' < gcp-creds.json)
-    OCI_CREDENTIALS_EPH=$(jq -r '."oci-credentials"' < oci-creds.json)
-    OCI_CONFIG_EPH=$(jq -r '."oci-config"' < oci-creds.json)
+    # oc get secret/koku-aws -o json -n ephemeral-base | jq -r '.data' > aws-creds.json
+    # oc get secret/koku-gcp -o json -n ephemeral-base | jq -r '.data' > gcp-creds.json
+    # oc get secret/koku-oci -o json -n ephemeral-base | jq -r '.data' > oci-creds.json
+
+    # AWS_CREDENTIALS_EPH=$(jq -r '."aws-credentials"' < aws-creds.json)
+    # GCP_CREDENTIALS_EPH=$(jq -r '."gcp-credentials"' < gcp-creds.json)
+    # OCI_CREDENTIALS_EPH=$(jq -r '."oci-credentials"' < oci-creds.json)
+    # OCI_CONFIG_EPH=$(jq -r '."oci-config"' < oci-creds.json)
 
     bonfire deploy \
         ${APP_NAME} \
         --ref-env insights-production \
         --set-template-ref ${APP_NAME}/${COMPONENT_NAME}=${ghprbActualCommit} \
+        --set-template-ref trino=9d20acdd35ec5049745b8575eff4eb262ba40424 \
         --set-image-tag ${IMAGE}=${IMAGE_TAG} \
         --namespace ${NAMESPACE} \
         ${COMPONENTS_ARG} \
         ${COMPONENTS_RESOURCES_ARG} \
         --optional-deps-method hybrid \
         --set-parameter rbac/MIN_REPLICAS=1 \
-        --set-parameter koku/AWS_CREDENTIALS_EPH=${AWS_CREDENTIALS_EPH} \
-        --set-parameter koku/GCP_CREDENTIALS_EPH=${GCP_CREDENTIALS_EPH} \
-        --set-parameter koku/OCI_CREDENTIALS_EPH=${OCI_CREDENTIALS_EPH} \
-        --set-parameter koku/OCI_CONFIG_EPH=${OCI_CONFIG_EPH} \
         --set-parameter koku/DBM_IMAGE=${IMAGE} \
         --set-parameter koku/DBM_IMAGE_TAG=${IMAGE_TAG} \
         --set-parameter koku/DBM_INVOCATION=${DBM_INVOCATION} \
         --set-parameter koku/IMAGE=${IMAGE} \
+        --set-parameter koku/SCHEMA_SUFFIX=_${IMAGE_TAG} \
+        --set-parameter trino/IMAGE=quay.io/redhat-user-workloads/cost-mgmt-dev-tenant/ubi-trino \
+        --set-parameter trino/IMAGE_TAG=pr-153-9d20acd \
+        --set-parameter trino/HIVE_PROPERTIES_FILE=glue.properties \
+        --set-parameter trino/GLUE_PROPERTIES_FILE=hive.properties \
         --no-single-replicas \
         --source=appsre \
         --timeout 600
 
+
     echo "Running E2E tests with IQE:"
     echo "IQE_MARKER_EXPRESSION: '$IQE_MARKER_EXPRESSION'"
     echo "IQE_FILTER_EXPRESSION: '$IQE_FILTER_EXPRESSION'"
@@ -183,7 +190,7 @@ function wait_for_image() {
 
     local count=0
     local max=60  # Try for up to 30 minutes
-    until podman image search --limit 500 --list-tags "${IMAGE}" | grep -q "${IMAGE_TAG}"; do
+    until [[ $(curl -k -XGET "https://quay.io/api/v1/repository/redhat-user-workloads/cost-mgmt-dev-tenant/koku/tag?specificTag=${IMAGE_TAG}" -Ls | jq '.tags | length') -gt 0  ]]; do
         echo "${count}: Checking for image ${IMAGE}:${IMAGE_TAG}..."
         sleep 30
         ((count+=1))