Merge pull request #260 from NASA-IMPACT/fix/reveal-transfer-exception

fix(transfer-dag): add needed assume role and stop discarding exceptions
NASA-IMPACT · Nov 22, 2024 · ef7be91 · ef7be91
2 parents 32b2acb + 4f14c96
commit ef7be91
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 10 deletions.
diff --git a/dags/veda_data_pipeline/utils/transfer.py b/dags/veda_data_pipeline/utils/transfer.py
@@ -44,6 +44,7 @@ def get_matching_files(s3_client, bucket, prefix, regex_pattern):
 def transfer_files_within_s3(
     s3_client, origin_bucket, matching_files, destination_bucket, collection
 ):
+    transfer_exceptions = False
     for file_key in matching_files:
         filename = file_key.split("/")[-1]
         # print(f"Transferring file: {filename}")
@@ -66,12 +67,22 @@ def transfer_files_within_s3(
             )
         except s3_client.exceptions.ClientError as err:
             if err.response["Error"]["Code"] == "404":
-                # print(f"Copying file: {filename}")
+                # File not found OK to copy
                 s3_client.copy_object(
                     CopySource=copy_source,
                     Bucket=destination_bucket,
                     Key=target_key
                 )
+            else:
+                msg = f"ClientError copying {filename=} {err=}"
+                print(msg)
+                transfer_exceptions = True
+        except Exception as e:
+            msg = f"Exception copying {filename=} {e=}"
+            print(msg)
+            transfer_exceptions = True
+    if transfer_exceptions:
+        raise Exception(f"{transfer_exceptions=}")
 
 
 def data_transfer_handler(event, role_arn=None):

diff --git a/sm2a/infrastructure/main.tf b/sm2a/infrastructure/main.tf
@@ -89,6 +89,8 @@ module "sma-base" {
     STAC_INGESTOR_API_URL = var.stac_ingestor_api_url
     STAC_URL              = var.stac_url
     VECTOR_SECRET_NAME    = var.vector_secret_name
+    ASSUME_ROLE_READ_ARN = var.assume_role_read_arn
+    ASSUME_ROLE_WRITE_ARN = var.assume_role_write_arn
   }
 }
 
diff --git a/sm2a/infrastructure/s3_event_bridge_lambda.tf b/sm2a/infrastructure/s3_event_bridge_lambda.tf
@@ -113,7 +113,7 @@ data "archive_file" "python_lambda_package" {
 
 
 resource "aws_lambda_function" "lambda" {
-  count = var.eis_storage_bucket_name != null ? 1 : 0
+  count = var.eis_storage_bucket_name != "null" ? 1 : 0
 
   provider         = aws.aws_current
   filename         = "/tmp/s3_event_bridge_to_sfn_execute.zip"
@@ -136,7 +136,7 @@ resource "aws_lambda_function" "lambda" {
 
 resource "aws_cloudwatch_log_group" "group" {
 
-  count = var.eis_storage_bucket_name != null ? 1 : 0
+  count = var.eis_storage_bucket_name != "null" ? 1 : 0
 
 
   provider          = aws.aws_current
@@ -149,7 +149,7 @@ resource "aws_cloudwatch_log_group" "group" {
 #####################################################
 
 resource "aws_lambda_permission" "s3_invoke" {
-  count = var.eis_storage_bucket_name != null ? 1 : 0
+  count = var.eis_storage_bucket_name != "null" ? 1 : 0
 
   provider      = aws.aws_current
   action        = "lambda:InvokeFunction"
@@ -163,7 +163,7 @@ resource "aws_lambda_permission" "s3_invoke" {
 
 
 resource "aws_s3_bucket_notification" "bucket_notification" {
-  count = var.eis_storage_bucket_name != null ? 1 : 0
+  count = var.eis_storage_bucket_name != "null" ? 1 : 0
   bucket = var.eis_storage_bucket_name
 
   lambda_function {

diff --git a/sm2a/infrastructure/variables.tf b/sm2a/infrastructure/variables.tf
@@ -123,26 +123,26 @@ variable "stac_url" {
 
 variable "vector_secret_name" {
   type = string
-  default = null
+  default = "null"
 }
 
 variable "eis_storage_bucket_name" {
   type = string
-  default = null
+  default = "null"
 }
 
 variable "eis_s3_invoke_filter_prefix" {
   type = string
-  default = null
+  default = "null"
 }
 variable "sm2a_secret_manager_name" {
   type = string
-  default = null
+  default = "null"
 }
 
 variable "target_dag_id" {
   type = string
-  default = null
+  default = "null"
 }
 
 
@@ -172,3 +172,13 @@ variable "workers_logs_retention_days" {
 variable "workers_task_retries" {
   default = "1"
 }
+
+variable "assume_role_read_arn" {
+  type = string
+  default = ""
+}
+
+variable "assume_role_write_arn" {
+  type = string
+  default = ""
+}