Merge branch 'main' into main

CHANGELOG.md

@@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+### **Added**
+- added multi-acc sagemaker-mlops manifest example
+
+### **Changed**
+- fixed model deploy cross-account permissions
+- added bucket and model package group names as stack outputs in the `sagemaker-templates` module
+
 ## v1.1.0
 
 ### **Added**

manifests/mlops-sagemaker-multiacc/deployment.yaml

@@ -0,0 +1,37 @@
+name: mlops-sagemaker
+toolchainRegion: us-east-1
+forceDependencyRedeploy: true
+groups:
+  - name: networking
+    path: manifests/mlops-sagemaker-multiacc/networking-modules.yaml
+  - name: storage
+    path: manifests/mlops-sagemaker-multiacc/storage-modules.yaml
+  - name: sagemaker-studio
+    path: manifests/mlops-sagemaker-multiacc/sagemaker-studio-modules.yaml
+  - name: sagemaker-templates
+    path: manifests/mlops-sagemaker-multiacc/sagemaker-templates-modules.yaml
+  - name: sagemaker-kernels
+    path: manifests/mlops-sagemaker-multiacc/kernels-modules.yaml
+targetAccountMappings:
+  - alias: dev
+    accountId:
+      valueFrom:
+        envVariable: DEV_ACCOUNT
+    default: true
+    regionMappings:
+      - region: us-east-1
+        default: true
+  - alias: pre-prod
+    accountId:
+      valueFrom:
+        envVariable: PRE_PROD_ACCOUNT
+    regionMappings:
+      - region: us-east-1
+        default: true
+  - alias: prod
+    accountId:
+      valueFrom:
+        envVariable: PROD_ACCOUNT
+    regionMappings:
+      - region: us-east-1
+        default: true

manifests/mlops-sagemaker-multiacc/kernels-modules.yaml

@@ -0,0 +1,40 @@
+name: sagemaker-custom-kernel
+path: modules/sagemaker/sagemaker-custom-kernel/
+targetAccount: dev
+parameters:
+  - name: ecr-repo-name
+    valueFrom:
+      moduleMetadata:
+        group: storage
+        name: ecr-sagemaker-kernel
+        key: EcrRepositoryName
+  - name: studio-domain-id
+    valueFrom:
+      moduleMetadata:
+        group: sagemaker-studio
+        name: studio
+        key: StudioDomainId
+  - name: studio-domain-name
+    valueFrom:
+      moduleMetadata:
+        group: sagemaker-studio
+        name: studio
+        key: StudioDomainName
+  - name: studio-execution-role-arn
+    valueFrom:
+      moduleMetadata:
+        group: sagemaker-studio
+        name: studio
+        key: SageMakerExecutionRoleArn
+  - name: sagemaker-image-name
+    value: echo-kernel
+  - name: app-image-config-name
+    value: echo-kernel-app-config
+  - name: custom-kernel-name
+    value: echo
+  - name: kernel-user-uid
+    value: '0'
+  - name: kernel-user-gid
+    value: '0'
+  - name: kernel-user-home-mount-path
+    value: /root

manifests/mlops-sagemaker-multiacc/networking-modules.yaml

@@ -0,0 +1,20 @@
+name: networking-dev
+path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk?ref=release/1.3.0&depth=1
+targetAccount: dev
+parameters:
+  - name: internet-accessible
+    value: True
+---
+name: networking-pre-prod
+path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk?ref=release/1.3.0&depth=1
+targetAccount: pre-prod
+parameters:
+  - name: internet-accessible
+    value: True
+---
+name: networking-prod
+path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk?ref=release/1.3.0&depth=1
+targetAccount: prod
+parameters:
+  - name: internet-accessible
+    value: True

manifests/mlops-sagemaker-multiacc/sagemaker-studio-modules.yaml

@@ -0,0 +1,30 @@
+name: studio
+path: modules/sagemaker/sagemaker-studio
+targetAccount: dev
+parameters:
+  - name: vpc_id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking-dev
+        key: VpcId
+  - name: subnet_ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking-dev
+        key: PrivateSubnetIds
+  - name: data_science_users
+    value:
+    - ds-user-1
+  - name: lead_data_science_users
+    value:
+    - lead-ds-user-1
+  - name: server_lifecycle_name
+    value: studio-auto-shutdown
+  - name: studio_bucket_name
+    value: mlops-*
+  - name: retain_efs
+    value: 'False'
+  - name: enable_custom_sagemaker_projects
+    value: 'True'

manifests/mlops-sagemaker-multiacc/sagemaker-templates-modules.yaml

@@ -0,0 +1,64 @@
+name: service-catalog
+path: modules/sagemaker/sagemaker-templates-service-catalog
+targetAccount: dev
+parameters:
+  - name: portfolio-access-role-arn
+    valueFrom:
+      moduleMetadata:
+        group: sagemaker-studio
+        name: studio
+        key: LeadDataScientistRoleArn
+  - name: dev-account-id
+    valueFrom:
+      envVariable: DEV_ACCOUNT
+  - name: dev-region
+    valueFrom:
+      envVariable: DEV_REGION
+  - name: dev-vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking-dev
+        key: VpcId
+  - name: dev-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking-dev
+        key: PrivateSubnetIds
+  - name: pre-prod-account-id
+    valueFrom:
+      envVariable: PRE_PROD_ACCOUNT
+  - name: pre-prod-region
+    valueFrom:
+      envVariable: PRE_PROD_REGION
+  - name: pre-prod-vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking-pre-prod
+        key: VpcId
+  - name: pre-prod-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking-pre-prod
+        key: PrivateSubnetIds
+  - name: prod-account-id
+    valueFrom:
+      envVariable: PROD_ACCOUNT
+  - name: prod-region
+    valueFrom:
+      envVariable: PROD_REGION
+  - name: prod-vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking-prod
+        key: VpcId
+  - name: prod-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking-prod
+        key: PrivateSubnetIds

manifests/mlops-sagemaker-multiacc/storage-modules.yaml

@@ -0,0 +1,15 @@
+name: ecr-sagemaker-kernel
+path: git::https://github.com/awslabs/idf-modules.git//modules/storage/ecr?ref=release/1.7.0&depth=1
+targetAccount: dev
+parameters:
+  - name: image-tag-mutability
+    value: MUTABLE
+---
+name: buckets
+path: git::https://github.com/awslabs/idf-modules.git//modules/storage/buckets?ref=release/1.7.0&depth=1
+targetAccount: dev
+parameters:
+  - name: encryption-type
+    value: SSE
+  - name: retention-type
+    value: RETAIN

modules/sagemaker/sagemaker-templates-service-catalog/templates/hf_import_models/product_stack.py

@@ -19,7 +19,7 @@
 
 import aws_cdk
 import aws_cdk.aws_servicecatalog as servicecatalog
-from aws_cdk import Aws, Tags
+from aws_cdk import Aws, CfnOutput, Tags
 from aws_cdk import aws_iam as iam
 from aws_cdk import aws_kms as kms
 from aws_cdk import aws_s3 as s3
@@ -214,3 +214,15 @@ def __init__(
             hf_access_token_secret=hf_access_token_secret,
             hf_model_id=hf_model_id,
         )
+
+        CfnOutput(
+            self,
+            "Model Bucket Name",
+            value=s3_artifact.bucket_name,
+        )
+
+        CfnOutput(
+            self,
+            "Model Package Group Name",
+            value=model_package_group_name,
+        )

modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/pipeline_constructs/deploy_pipeline_construct.py

@@ -119,6 +119,21 @@ def __init__(
                         )
                     ]
                 ),
+                "STS": iam.PolicyDocument(
+                    statements=[
+                        iam.PolicyStatement(
+                            actions=[
+                                "sts:AssumeRole",
+                            ],
+                            effect=iam.Effect.ALLOW,
+                            resources=[
+                                f"arn:{Aws.PARTITION}:iam::{dev_account_id}:role/cdk-*",
+                                f"arn:{Aws.PARTITION}:iam::{pre_prod_account_id}:role/cdk-*",
+                                f"arn:{Aws.PARTITION}:iam::{prod_account_id}:role/cdk-*",
+                            ],
+                        )
+                    ]
+                ),
             },
         )
 

modules/sagemaker/sagemaker-templates-service-catalog/templates/model_deploy/product_stack.py

@@ -124,8 +124,26 @@ def __init__(
                         actions=["kms:*"],
                         effect=iam.Effect.ALLOW,
                         resources=["*"],
-                        principals=[iam.AccountRootPrincipal()],
-                    )
+                        principals=[
+                            iam.AccountRootPrincipal(),
+                        ],
+                    ),
+                    iam.PolicyStatement(
+                        actions=[
+                            "kms:Encrypt",
+                            "kms:Decrypt",
+                            "kms:ReEncrypt*",
+                            "kms:GenerateDataKey*",
+                            "kms:DescribeKey",
+                        ],
+                        resources=[
+                            "*",
+                        ],
+                        principals=[
+                            iam.AccountPrincipal(pre_prod_account_id),
+                            iam.AccountPrincipal(prod_account_id),
+                        ],
+                    ),
                 ]
             ),
         )
@@ -138,6 +156,35 @@ def __init__(
             versioned=True,
             removal_policy=RemovalPolicy.DESTROY,
         )
+        # Block non-SSL
+        pipeline_artifact_bucket.add_to_resource_policy(
+            iam.PolicyStatement(
+                sid="AllowSSLOnly",
+                actions=["s3:*"],
+                effect=iam.Effect.DENY,
+                resources=[
+                    pipeline_artifact_bucket.bucket_arn,
+                    pipeline_artifact_bucket.arn_for_objects(key_pattern="*"),
+                ],
+                conditions={"Bool": {"aws:SecureTransport": "false"}},
+                principals=[iam.AnyPrincipal()],
+            )
+        )
+        # Add cross-account access
+        pipeline_artifact_bucket.add_to_resource_policy(
+            iam.PolicyStatement(
+                sid="CrossAccountPermissions",
+                actions=["s3:List*", "s3:Get*", "s3:Put*"],
+                resources=[
+                    pipeline_artifact_bucket.arn_for_objects(key_pattern="*"),
+                    pipeline_artifact_bucket.bucket_arn,
+                ],
+                principals=[
+                    iam.AccountPrincipal(pre_prod_account_id),
+                    iam.AccountPrincipal(prod_account_id),
+                ],
+            )
+        )
 
         DeployPipelineConstruct(
             self,

modules/sagemaker/sagemaker-templates-service-catalog/templates/xgboost_abalone/product_stack.py

@@ -9,7 +9,7 @@
 import aws_cdk.aws_s3_assets as s3_assets
 import aws_cdk.aws_sagemaker as sagemaker
 import aws_cdk.aws_servicecatalog as servicecatalog
-from aws_cdk import Aws, CfnParameter, CfnTag, RemovalPolicy, Tags
+from aws_cdk import Aws, CfnOutput, CfnParameter, CfnTag, RemovalPolicy, Tags
 from constructs import Construct
 
 from templates.xgboost_abalone.pipeline_constructs.build_pipeline_construct import (
@@ -237,3 +237,15 @@ def __init__(
             pipeline_artifact_bucket=pipeline_artifact_bucket,
             repo_asset=build_app_asset,
         )
+
+        CfnOutput(
+            self,
+            "Model Bucket Name",
+            value=model_bucket.bucket_name,
+        )
+
+        CfnOutput(
+            self,
+            "Model Package Group Name",
+            value=model_package_group_name,
+        )