From 572db64b5200fa10a53a35c310cec48451bbbf0b Mon Sep 17 00:00:00 2001 From: kukushking Date: Thu, 9 May 2024 10:18:35 +0100 Subject: [PATCH 1/3] fix: Model deploy x-acc pipeline permissions & add multi-acc manifest (#83) * fix xacc pipeline bucket permissions Signed-off-by: Anton Kukushkin * add multi-acc manifest example Signed-off-by: Anton Kukushkin * use account principal construct Signed-off-by: Anton Kukushkin --------- Signed-off-by: Anton Kukushkin --- .../mlops-sagemaker-multiacc/deployment.yaml | 37 +++++++++++ .../kernels-modules.yaml | 40 ++++++++++++ .../networking-modules.yaml | 20 ++++++ .../sagemaker-studio-modules.yaml | 30 +++++++++ .../sagemaker-templates-modules.yaml | 64 +++++++++++++++++++ .../storage-modules.yaml | 15 +++++ .../deploy_pipeline_construct.py | 15 +++++ .../templates/model_deploy/product_stack.py | 52 ++++++++++++++- 8 files changed, 271 insertions(+), 2 deletions(-) create mode 100644 manifests/mlops-sagemaker-multiacc/deployment.yaml create mode 100644 manifests/mlops-sagemaker-multiacc/kernels-modules.yaml create mode 100644 manifests/mlops-sagemaker-multiacc/networking-modules.yaml create mode 100644 manifests/mlops-sagemaker-multiacc/sagemaker-studio-modules.yaml create mode 100644 manifests/mlops-sagemaker-multiacc/sagemaker-templates-modules.yaml create mode 100644 manifests/mlops-sagemaker-multiacc/storage-modules.yaml diff --git a/manifests/mlops-sagemaker-multiacc/deployment.yaml b/manifests/mlops-sagemaker-multiacc/deployment.yaml new file mode 100644 index 00000000..adb36b5e --- /dev/null +++ b/manifests/mlops-sagemaker-multiacc/deployment.yaml @@ -0,0 +1,37 @@ +name: mlops-sagemaker +toolchainRegion: us-east-1 +forceDependencyRedeploy: true +groups: + - name: networking + path: manifests/mlops-sagemaker/networking-modules.yaml + - name: storage + path: manifests/mlops-sagemaker/storage-modules.yaml + - name: sagemaker-studio + path: manifests/mlops-sagemaker/sagemaker-studio-modules.yaml + - name: sagemaker-templates + path: manifests/mlops-sagemaker/sagemaker-templates-modules.yaml + - name: sagemaker-kernels + path: manifests/mlops-sagemaker/kernels-modules.yaml +targetAccountMappings: + - alias: dev + accountId: + valueFrom: + envVariable: DEV_ACCOUNT + default: true + regionMappings: + - region: us-east-1 + default: true + - alias: pre-prod + accountId: + valueFrom: + envVariable: PRE_PROD_ACCOUNT + regionMappings: + - region: us-east-1 + default: true + - alias: prod + accountId: + valueFrom: + envVariable: PROD_ACCOUNT + regionMappings: + - region: us-east-1 + default: true \ No newline at end of file diff --git a/manifests/mlops-sagemaker-multiacc/kernels-modules.yaml b/manifests/mlops-sagemaker-multiacc/kernels-modules.yaml new file mode 100644 index 00000000..ba1f6dd6 --- /dev/null +++ b/manifests/mlops-sagemaker-multiacc/kernels-modules.yaml @@ -0,0 +1,40 @@ +name: sagemaker-custom-kernel +path: modules/sagemaker/sagemaker-custom-kernel/ +targetAccount: dev +parameters: + - name: ecr-repo-name + valueFrom: + moduleMetadata: + group: storage + name: ecr-sagemaker-kernel + key: EcrRepositoryName + - name: studio-domain-id + valueFrom: + moduleMetadata: + group: sagemaker-studio + name: studio + key: StudioDomainId + - name: studio-domain-name + valueFrom: + moduleMetadata: + group: sagemaker-studio + name: studio + key: StudioDomainName + - name: studio-execution-role-arn + valueFrom: + moduleMetadata: + group: sagemaker-studio + name: studio + key: SageMakerExecutionRoleArn + - name: sagemaker-image-name + value: echo-kernel + - name: app-image-config-name + value: echo-kernel-app-config + - name: custom-kernel-name + value: echo + - name: kernel-user-uid + value: '0' + - name: kernel-user-gid + value: '0' + - name: kernel-user-home-mount-path + value: /root \ No newline at end of file diff --git a/manifests/mlops-sagemaker-multiacc/networking-modules.yaml b/manifests/mlops-sagemaker-multiacc/networking-modules.yaml new file mode 100644 index 00000000..2775e636 --- /dev/null +++ b/manifests/mlops-sagemaker-multiacc/networking-modules.yaml @@ -0,0 +1,20 @@ +name: networking-dev +path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk?ref=release/1.3.0&depth=1 +targetAccount: dev +parameters: + - name: internet-accessible + value: True +--- +name: networking-pre-prod +path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk?ref=release/1.3.0&depth=1 +targetAccount: pre-prod +parameters: + - name: internet-accessible + value: True +--- +name: networking-prod +path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk?ref=release/1.3.0&depth=1 +targetAccount: prod +parameters: + - name: internet-accessible + value: True \ No newline at end of file diff --git a/manifests/mlops-sagemaker-multiacc/sagemaker-studio-modules.yaml b/manifests/mlops-sagemaker-multiacc/sagemaker-studio-modules.yaml new file mode 100644 index 00000000..86c05fa6 --- /dev/null +++ b/manifests/mlops-sagemaker-multiacc/sagemaker-studio-modules.yaml @@ -0,0 +1,30 @@ +name: studio +path: modules/sagemaker/sagemaker-studio +targetAccount: dev +parameters: + - name: vpc_id + valueFrom: + moduleMetadata: + group: networking + name: networking-dev + key: VpcId + - name: subnet_ids + valueFrom: + moduleMetadata: + group: networking + name: networking-dev + key: PrivateSubnetIds + - name: data_science_users + value: + - ds-user-1 + - name: lead_data_science_users + value: + - lead-ds-user-1 + - name: server_lifecycle_name + value: studio-auto-shutdown + - name: studio_bucket_name + value: mlops-* + - name: retain_efs + value: 'False' + - name: enable_custom_sagemaker_projects + value: 'True' \ No newline at end of file diff --git a/manifests/mlops-sagemaker-multiacc/sagemaker-templates-modules.yaml b/manifests/mlops-sagemaker-multiacc/sagemaker-templates-modules.yaml new file mode 100644 index 00000000..cd00f6bf --- /dev/null +++ b/manifests/mlops-sagemaker-multiacc/sagemaker-templates-modules.yaml @@ -0,0 +1,64 @@ +name: service-catalog +path: modules/sagemaker/sagemaker-templates-service-catalog +targetAccount: dev +parameters: + - name: portfolio-access-role-arn + valueFrom: + moduleMetadata: + group: sagemaker-studio + name: studio + key: LeadDataScientistRoleArn + - name: dev-account-id + valueFrom: + envVariable: DEV_ACCOUNT + - name: dev-region + valueFrom: + envVariable: DEV_REGION + - name: dev-vpc-id + valueFrom: + moduleMetadata: + group: networking + name: networking-dev + key: VpcId + - name: dev-subnet-ids + valueFrom: + moduleMetadata: + group: networking + name: networking-dev + key: PrivateSubnetIds + - name: pre-prod-account-id + valueFrom: + envVariable: PRE_PROD_ACCOUNT + - name: pre-prod-region + valueFrom: + envVariable: PRE_PROD_REGION + - name: pre-prod-vpc-id + valueFrom: + moduleMetadata: + group: networking + name: networking-pre-prod + key: VpcId + - name: pre-prod-subnet-ids + valueFrom: + moduleMetadata: + group: networking + name: networking-pre-prod + key: PrivateSubnetIds + - name: prod-account-id + valueFrom: + envVariable: PROD_ACCOUNT + - name: prod-region + valueFrom: + envVariable: PROD_REGION + - name: prod-vpc-id + valueFrom: + moduleMetadata: + group: networking + name: networking-prod + key: VpcId + - name: prod-subnet-ids + valueFrom: + moduleMetadata: + group: networking + name: networking-prod + key: PrivateSubnetIds \ No newline at end of file diff --git a/manifests/mlops-sagemaker-multiacc/storage-modules.yaml b/manifests/mlops-sagemaker-multiacc/storage-modules.yaml new file mode 100644 index 00000000..5a9b8fd5 --- /dev/null +++ b/manifests/mlops-sagemaker-multiacc/storage-modules.yaml @@ -0,0 +1,15 @@ +name: ecr-sagemaker-kernel +path: git::https://github.com/awslabs/idf-modules.git//modules/storage/ecr?ref=release/1.7.0&depth=1 +targetAccount: dev +parameters: + - name: image-tag-mutability + value: MUTABLE +--- +name: buckets +path: git::https://github.com/awslabs/idf-modules.git//modules/storage/buckets?ref=release/1.7.0&depth=1 +targetAccount: dev +parameters: + - name: encryption-type + value: SSE + - name: retention-type + value: RETAIN \ No newline at end of file diff --git a/modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/pipeline_constructs/deploy_pipeline_construct.py b/modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/pipeline_constructs/deploy_pipeline_construct.py index 9b28f0b0..27eeec12 100644 --- a/modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/pipeline_constructs/deploy_pipeline_construct.py +++ b/modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/pipeline_constructs/deploy_pipeline_construct.py @@ -119,6 +119,21 @@ def __init__( ) ] ), + "STS": iam.PolicyDocument( + statements=[ + iam.PolicyStatement( + actions=[ + "sts:AssumeRole", + ], + effect=iam.Effect.ALLOW, + resources=[ + f"arn:{Aws.PARTITION}:iam::{dev_account_id}:role/cdk-*", + f"arn:{Aws.PARTITION}:iam::{pre_prod_account_id}:role/cdk-*", + f"arn:{Aws.PARTITION}:iam::{prod_account_id}:role/cdk-*", + ], + ) + ] + ), }, ) diff --git a/modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/product_stack.py b/modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/product_stack.py index c8ff084b..3dbde69b 100644 --- a/modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/product_stack.py +++ b/modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/product_stack.py @@ -42,6 +42,7 @@ def __init__( ) -> None: super().__init__(scope, id) + dev_account_id = Aws.ACCOUNT_ID pre_prod_account_id = Aws.ACCOUNT_ID if not pre_prod_account_id else pre_prod_account_id prod_account_id = Aws.ACCOUNT_ID if not prod_account_id else prod_account_id pre_prod_region = Aws.REGION if not pre_prod_region else pre_prod_region @@ -124,8 +125,26 @@ def __init__( actions=["kms:*"], effect=iam.Effect.ALLOW, resources=["*"], - principals=[iam.AccountRootPrincipal()], - ) + principals=[ + iam.AccountRootPrincipal(), + ], + ), + iam.PolicyStatement( + actions=[ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey", + ], + resources=[ + "*", + ], + principals=[ + iam.AccountPrincipal(pre_prod_account_id), + iam.AccountPrincipal(prod_account_id), + ], + ), ] ), ) @@ -138,6 +157,35 @@ def __init__( versioned=True, removal_policy=RemovalPolicy.DESTROY, ) + # Block non-SSL + pipeline_artifact_bucket.add_to_resource_policy( + iam.PolicyStatement( + sid="AllowSSLOnly", + actions=["s3:*"], + effect=iam.Effect.DENY, + resources=[ + pipeline_artifact_bucket.bucket_arn, + pipeline_artifact_bucket.arn_for_objects(key_pattern="*"), + ], + conditions={"Bool": {"aws:SecureTransport": "false"}}, + principals=[iam.AnyPrincipal()], + ) + ) + # Add cross-account access + pipeline_artifact_bucket.add_to_resource_policy( + iam.PolicyStatement( + sid="CrossAccountPermissions", + actions=["s3:List*", "s3:Get*", "s3:Put*"], + resources=[ + pipeline_artifact_bucket.arn_for_objects(key_pattern="*"), + pipeline_artifact_bucket.bucket_arn, + ], + principals=[ + iam.AccountPrincipal(pre_prod_account_id), + iam.AccountPrincipal(prod_account_id), + ], + ) + ) DeployPipelineConstruct( self, From 6d9a0f729d6caa93fd79fb2aab1603579bf12ae4 Mon Sep 17 00:00:00 2001 From: kukushking Date: Thu, 9 May 2024 14:56:14 +0100 Subject: [PATCH 2/3] fix: multiacc manifest paths (#84) * fix multiacc manifest paths Signed-off-by: Anton Kukushkin * CHANGELOG Signed-off-by: Anton Kukushkin * linting Signed-off-by: Anton Kukushkin --------- Signed-off-by: Anton Kukushkin --- CHANGELOG.md | 6 ++++++ manifests/mlops-sagemaker-multiacc/deployment.yaml | 10 +++++----- .../templates/model_deploy/product_stack.py | 1 - 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36ab0b17..dbe1bece 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## UNRELEASED +### **Added** +- added multi-acc sagemaker-mlops manifest example + +### **Changed** +- fixed model deploy cross-account permissions + ## v1.1.0 ### **Added** diff --git a/manifests/mlops-sagemaker-multiacc/deployment.yaml b/manifests/mlops-sagemaker-multiacc/deployment.yaml index adb36b5e..67b9de43 100644 --- a/manifests/mlops-sagemaker-multiacc/deployment.yaml +++ b/manifests/mlops-sagemaker-multiacc/deployment.yaml @@ -3,15 +3,15 @@ toolchainRegion: us-east-1 forceDependencyRedeploy: true groups: - name: networking - path: manifests/mlops-sagemaker/networking-modules.yaml + path: manifests/mlops-sagemaker-multiacc/networking-modules.yaml - name: storage - path: manifests/mlops-sagemaker/storage-modules.yaml + path: manifests/mlops-sagemaker-multiacc/storage-modules.yaml - name: sagemaker-studio - path: manifests/mlops-sagemaker/sagemaker-studio-modules.yaml + path: manifests/mlops-sagemaker-multiacc/sagemaker-studio-modules.yaml - name: sagemaker-templates - path: manifests/mlops-sagemaker/sagemaker-templates-modules.yaml + path: manifests/mlops-sagemaker-multiacc/sagemaker-templates-modules.yaml - name: sagemaker-kernels - path: manifests/mlops-sagemaker/kernels-modules.yaml + path: manifests/mlops-sagemaker-multiacc/kernels-modules.yaml targetAccountMappings: - alias: dev accountId: diff --git a/modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/product_stack.py b/modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/product_stack.py index 3dbde69b..db332087 100644 --- a/modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/product_stack.py +++ b/modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/product_stack.py @@ -42,7 +42,6 @@ def __init__( ) -> None: super().__init__(scope, id) - dev_account_id = Aws.ACCOUNT_ID pre_prod_account_id = Aws.ACCOUNT_ID if not pre_prod_account_id else pre_prod_account_id prod_account_id = Aws.ACCOUNT_ID if not prod_account_id else prod_account_id pre_prod_region = Aws.REGION if not pre_prod_region else pre_prod_region From 912fdac249b73f28c523eb4db093ea1f77a0c85a Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Fri, 10 May 2024 09:28:01 -0500 Subject: [PATCH 3/3] (sagemaker templates) add bucket and model package group names as stack outputs (#85) --- CHANGELOG.md | 1 + .../templates/hf_import_models/product_stack.py | 14 +++++++++++++- .../templates/xgboost_abalone/product_stack.py | 14 +++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbe1bece..d61bd805 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### **Changed** - fixed model deploy cross-account permissions +- added bucket and model package group names as stack outputs in the `sagemaker-templates` module ## v1.1.0 diff --git a/modules/sagemaker/sagemaker-templates-service-catalog/templates/hf_import_models/product_stack.py b/modules/sagemaker/sagemaker-templates-service-catalog/templates/hf_import_models/product_stack.py index 3c148579..93649c32 100644 --- a/modules/sagemaker/sagemaker-templates-service-catalog/templates/hf_import_models/product_stack.py +++ b/modules/sagemaker/sagemaker-templates-service-catalog/templates/hf_import_models/product_stack.py @@ -19,7 +19,7 @@ import aws_cdk import aws_cdk.aws_servicecatalog as servicecatalog -from aws_cdk import Aws, Tags +from aws_cdk import Aws, CfnOutput, Tags from aws_cdk import aws_iam as iam from aws_cdk import aws_kms as kms from aws_cdk import aws_s3 as s3 @@ -214,3 +214,15 @@ def __init__( hf_access_token_secret=hf_access_token_secret, hf_model_id=hf_model_id, ) + + CfnOutput( + self, + "Model Bucket Name", + value=s3_artifact.bucket_name, + ) + + CfnOutput( + self, + "Model Package Group Name", + value=model_package_group_name, + ) diff --git a/modules/sagemaker/sagemaker-templates-service-catalog/templates/xgboost_abalone/product_stack.py b/modules/sagemaker/sagemaker-templates-service-catalog/templates/xgboost_abalone/product_stack.py index d234d14b..0a79c9f0 100644 --- a/modules/sagemaker/sagemaker-templates-service-catalog/templates/xgboost_abalone/product_stack.py +++ b/modules/sagemaker/sagemaker-templates-service-catalog/templates/xgboost_abalone/product_stack.py @@ -9,7 +9,7 @@ import aws_cdk.aws_s3_assets as s3_assets import aws_cdk.aws_sagemaker as sagemaker import aws_cdk.aws_servicecatalog as servicecatalog -from aws_cdk import Aws, CfnParameter, CfnTag, RemovalPolicy, Tags +from aws_cdk import Aws, CfnOutput, CfnParameter, CfnTag, RemovalPolicy, Tags from constructs import Construct from templates.xgboost_abalone.pipeline_constructs.build_pipeline_construct import ( @@ -237,3 +237,15 @@ def __init__( pipeline_artifact_bucket=pipeline_artifact_bucket, repo_asset=build_app_asset, ) + + CfnOutput( + self, + "Model Bucket Name", + value=model_bucket.bucket_name, + ) + + CfnOutput( + self, + "Model Package Group Name", + value=model_package_group_name, + )