From eb75646aaeca1b1274e41059fdf6d87943da114e Mon Sep 17 00:00:00 2001
From: Cyril Fait <cfait@misdre.info>
Date: Mon, 9 Oct 2023 15:46:57 +0200
Subject: [PATCH] provide vpc connection to glue jobs in glue-job-deployer, add
 disable-proxy-v2 to default arguments

---
 sdlf-cicd/template-glue-job.yaml              | 26 ++++++++++++++-----
 .../scripts/legislators-glue-job.yaml         | 21 ++++++++++++---
 2 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/sdlf-cicd/template-glue-job.yaml b/sdlf-cicd/template-glue-job.yaml
index 4590cb64..13f56f29 100644
--- a/sdlf-cicd/template-glue-job.yaml
+++ b/sdlf-cicd/template-glue-job.yaml
@@ -106,12 +106,21 @@ Resources:
           Name: glueetl
           PythonVersion: "3"
           ScriptLocation: !Sub s3://${pArtifactsBucket}/transforms/${pDomain}/${pEnvironment}/${pTeamName}/${GlueJobName}-${pGitRef}.py
-        DefaultArguments:
-          "--job-bookmark-option": job-bookmark-disable
-          "--enable-glue-datacatalog": "true"
-          "--enable-continuous-cloudwatch-log": "true"
-          "--enable-continuous-log-filter": "true"
-          "--enable-metrics": "true"
+        DefaultArguments: !If
+          - RunInVpc
+          -
+            "--job-bookmark-option": job-bookmark-disable
+            "--enable-glue-datacatalog": "true"
+            "--enable-continuous-cloudwatch-log": "true"
+            "--enable-continuous-log-filter": "true"
+            "--enable-metrics": "true"
+            "--disable-proxy-v2": "true"
+          -
+            "--job-bookmark-option": job-bookmark-disable
+            "--enable-glue-datacatalog": "true"
+            "--enable-continuous-cloudwatch-log": "true"
+            "--enable-continuous-log-filter": "true"
+            "--enable-metrics": "true"
         ExecutionProperty:
           MaxConcurrentRuns: 10
         MaxRetries: 0
@@ -120,3 +129,8 @@ Resources:
         Name: !Sub sdlf-${pTeamName}-${GlueJobName}
         SecurityConfiguration: !Sub "{{resolve:ssm:/SDLF/Glue/${pTeamName}/SecurityConfigurationId}}"
         Role: !Ref rGlueRole
+        Connections: !If
+          - RunInVpc
+          - Connections:
+              - !Ref rGlueConnection
+          - !Ref "AWS::NoValue"
\ No newline at end of file
diff --git a/sdlf-utils/pipeline-examples/legislators/scripts/legislators-glue-job.yaml b/sdlf-utils/pipeline-examples/legislators/scripts/legislators-glue-job.yaml
index 283956c7..46d124c0 100755
--- a/sdlf-utils/pipeline-examples/legislators/scripts/legislators-glue-job.yaml
+++ b/sdlf-utils/pipeline-examples/legislators/scripts/legislators-glue-job.yaml
@@ -14,6 +14,13 @@ Parameters:
     Description: The team name
     Type: String
     Default: engineering
+  pEnableVpc:
+    Description: Deploy SDLF resources in a VPC
+    Type: AWS::SSM::Parameter::Value<String>
+    Default: /SDLF/VPC/Enabled
+
+Conditions:
+  RunInVpc: !Equals [!Ref pEnableVpc, true]
 
 Resources:
   rGlueRole:
@@ -50,7 +57,7 @@ Resources:
                 Resource:
                   - !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/InfraKeyId:1}}"
                   - !Sub "{{resolve:ssm:/SDLF/KMS/${pTeamName}/DataKeyId:1}}"
-                  - !Sub "{{resolve:ssm:/SDLF/KMS/KeyArn:1}}"
+                  - "{{resolve:ssm:/SDLF/KMS/KeyArn:1}}"
 
   rGlueJob:
     Type: AWS::Glue::Job
@@ -59,9 +66,15 @@ Resources:
         Name: glueetl
         PythonVersion: "3"
         ScriptLocation: !Sub s3://${pArtifactsBucket}/artifacts/${pDatasetName}-glue-job.py
-      DefaultArguments:
-        "--job-bookmark-option": job-bookmark-enable
-        "--enable-metrics": ""
+      DefaultArguments: !If
+        - RunInVpc
+        -
+          "--job-bookmark-option": job-bookmark-enable
+          "--enable-metrics": ""
+          "--disable-proxy-v2": "true"
+        -
+          "--job-bookmark-option": job-bookmark-enable
+          "--enable-metrics": ""
       ExecutionProperty:
         MaxConcurrentRuns: 3
       MaxRetries: 0