From dbcd631f3f85da9e60250292c27157de820f46e2 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 21 Nov 2024 13:51:16 -0700 Subject: [PATCH 1/4] print all transfer exceptions --- dags/veda_data_pipeline/utils/transfer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dags/veda_data_pipeline/utils/transfer.py b/dags/veda_data_pipeline/utils/transfer.py index 20823f37..0274e944 100644 --- a/dags/veda_data_pipeline/utils/transfer.py +++ b/dags/veda_data_pipeline/utils/transfer.py @@ -66,12 +66,14 @@ def transfer_files_within_s3( ) except s3_client.exceptions.ClientError as err: if err.response["Error"]["Code"] == "404": - # print(f"Copying file: {filename}") + print(f"404 Error copying file: {filename}") s3_client.copy_object( CopySource=copy_source, Bucket=destination_bucket, Key=target_key ) + except Exception as e: + print(f"Exception copying file: {filename} exception: {e}") def data_transfer_handler(event, role_arn=None): From 4374f34ac31fa1a6bccb494b8cc43beea1deb1a1 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Fri, 22 Nov 2024 10:54:34 -0700 Subject: [PATCH 2/4] print client err on failed transfer --- dags/veda_data_pipeline/utils/transfer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dags/veda_data_pipeline/utils/transfer.py b/dags/veda_data_pipeline/utils/transfer.py index 0274e944..239bc06f 100644 --- a/dags/veda_data_pipeline/utils/transfer.py +++ b/dags/veda_data_pipeline/utils/transfer.py @@ -66,14 +66,16 @@ def transfer_files_within_s3( ) except s3_client.exceptions.ClientError as err: if err.response["Error"]["Code"] == "404": - print(f"404 Error copying file: {filename}") + # File not found OK to copy s3_client.copy_object( CopySource=copy_source, Bucket=destination_bucket, Key=target_key ) + else: + print(f"ClientError copying {filename=} {err=}") except Exception as e: - print(f"Exception copying file: {filename} exception: {e}") + print(f"Exception copying {filename=} {e=}") def data_transfer_handler(event, role_arn=None): From 091bc80b00fa2b34f3132f63fb14463c1040a770 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Fri, 22 Nov 2024 11:42:55 -0700 Subject: [PATCH 3/4] raise transfer exception --- dags/veda_data_pipeline/utils/transfer.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/dags/veda_data_pipeline/utils/transfer.py b/dags/veda_data_pipeline/utils/transfer.py index 239bc06f..a56e2e23 100644 --- a/dags/veda_data_pipeline/utils/transfer.py +++ b/dags/veda_data_pipeline/utils/transfer.py @@ -44,6 +44,7 @@ def get_matching_files(s3_client, bucket, prefix, regex_pattern): def transfer_files_within_s3( s3_client, origin_bucket, matching_files, destination_bucket, collection ): + transfer_exceptions = False for file_key in matching_files: filename = file_key.split("/")[-1] # print(f"Transferring file: {filename}") @@ -73,9 +74,15 @@ def transfer_files_within_s3( Key=target_key ) else: - print(f"ClientError copying {filename=} {err=}") + msg = f"ClientError copying {filename=} {err=}" + print(msg) + transfer_exceptions = True except Exception as e: - print(f"Exception copying {filename=} {e=}") + msg = f"Exception copying {filename=} {e=}" + print(msg) + transfer_exceptions = True + if transfer_exceptions: + raise Exception(f"{transfer_exceptions=}") def data_transfer_handler(event, role_arn=None): From 4f14c96f55ceaaaf2cff1d93cb95357f2342302d Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Fri, 22 Nov 2024 12:00:48 -0700 Subject: [PATCH 4/4] update secret variable type to string and add assume role arns to airflow_custom_variables" --- sm2a/infrastructure/main.tf | 2 ++ sm2a/infrastructure/s3_event_bridge_lambda.tf | 8 ++++---- sm2a/infrastructure/variables.tf | 20 ++++++++++++++----- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/sm2a/infrastructure/main.tf b/sm2a/infrastructure/main.tf index 3e735010..41d610e8 100644 --- a/sm2a/infrastructure/main.tf +++ b/sm2a/infrastructure/main.tf @@ -89,6 +89,8 @@ module "sma-base" { STAC_INGESTOR_API_URL = var.stac_ingestor_api_url STAC_URL = var.stac_url VECTOR_SECRET_NAME = var.vector_secret_name + ASSUME_ROLE_READ_ARN = var.assume_role_read_arn + ASSUME_ROLE_WRITE_ARN = var.assume_role_write_arn } } diff --git a/sm2a/infrastructure/s3_event_bridge_lambda.tf b/sm2a/infrastructure/s3_event_bridge_lambda.tf index 53882637..f0aca022 100644 --- a/sm2a/infrastructure/s3_event_bridge_lambda.tf +++ b/sm2a/infrastructure/s3_event_bridge_lambda.tf @@ -113,7 +113,7 @@ data "archive_file" "python_lambda_package" { resource "aws_lambda_function" "lambda" { - count = var.eis_storage_bucket_name != null ? 1 : 0 + count = var.eis_storage_bucket_name != "null" ? 1 : 0 provider = aws.aws_current filename = "/tmp/s3_event_bridge_to_sfn_execute.zip" @@ -136,7 +136,7 @@ resource "aws_lambda_function" "lambda" { resource "aws_cloudwatch_log_group" "group" { - count = var.eis_storage_bucket_name != null ? 1 : 0 + count = var.eis_storage_bucket_name != "null" ? 1 : 0 provider = aws.aws_current @@ -149,7 +149,7 @@ resource "aws_cloudwatch_log_group" "group" { ##################################################### resource "aws_lambda_permission" "s3_invoke" { - count = var.eis_storage_bucket_name != null ? 1 : 0 + count = var.eis_storage_bucket_name != "null" ? 1 : 0 provider = aws.aws_current action = "lambda:InvokeFunction" @@ -163,7 +163,7 @@ resource "aws_lambda_permission" "s3_invoke" { resource "aws_s3_bucket_notification" "bucket_notification" { - count = var.eis_storage_bucket_name != null ? 1 : 0 + count = var.eis_storage_bucket_name != "null" ? 1 : 0 bucket = var.eis_storage_bucket_name lambda_function { diff --git a/sm2a/infrastructure/variables.tf b/sm2a/infrastructure/variables.tf index a6fe186a..b2ec0b5d 100644 --- a/sm2a/infrastructure/variables.tf +++ b/sm2a/infrastructure/variables.tf @@ -123,26 +123,26 @@ variable "stac_url" { variable "vector_secret_name" { type = string - default = null + default = "null" } variable "eis_storage_bucket_name" { type = string - default = null + default = "null" } variable "eis_s3_invoke_filter_prefix" { type = string - default = null + default = "null" } variable "sm2a_secret_manager_name" { type = string - default = null + default = "null" } variable "target_dag_id" { type = string - default = null + default = "null" } @@ -172,3 +172,13 @@ variable "workers_logs_retention_days" { variable "workers_task_retries" { default = "1" } + +variable "assume_role_read_arn" { + type = string + default = "" +} + +variable "assume_role_write_arn" { + type = string + default = "" +}