Updated Q&A Rag ingestion module to include ingestion modules as well

awslabs · May 21, 2024 · 53737d7 · 53737d7
1 parent 3b8f21b
commit 53737d7
Show file tree

Hide file tree

Showing 14 changed files with 125 additions and 24 deletions.
diff --git a/examples/manifests/deployment.yaml b/examples/manifests/deployment.yaml
@@ -16,8 +16,6 @@ groups:
     path: manifests/sagemaker-model-package-promote-pipeline-modules.yaml
   - name: storage
     path: examples/manifests/storage-modules.yaml
-  - name: qna-rag
-    path: examples/manifests/qna-rag-modules-local.yaml
 
 targetAccountMappings:
   - alias: primary

diff --git a/manifests/fmops-qna-rag/deployment.yaml b/manifests/fmops-qna-rag/deployment.yaml
@@ -0,0 +1,22 @@
+# This is an example manifest.
+# Replace the parameters in referenced manifest groups with your values prior the deployment.
+name: mlops-new
+toolchainRegion: us-east-1
+forceDependencyRedeploy: true
+groups:
+  - name: networking
+    path: manifests/fmops-qna-rag/networking-modules.yaml
+  - name: storage
+    path: manifests/fmops-qna-rag/storage-modules.yaml
+  - name: qna-rag
+    path: manifests/fmops-qna-rag/qna-rag-modules.yaml
+
+targetAccountMappings:
+  - alias: primary
+    accountId:
+      valueFrom:
+        envVariable: PRIMARY_ACCOUNT
+    default: true
+    regionMappings:
+      - region: us-east-1
+        default: true
diff --git a/manifests/fmops-qna-rag/networking-modules.yaml b/manifests/fmops-qna-rag/networking-modules.yaml
@@ -0,0 +1,6 @@
+name: networking
+path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk?ref=release/1.3.0&depth=1
+targetAccount: primary
+parameters:
+  - name: internet-accessible
+    value: True
diff --git a/examples/manifests/qna-rag-modules.yaml → manifests/fmops-qna-rag/qna-rag-modules.yaml b/examples/manifests/qna-rag-modules.yaml → manifests/fmops-qna-rag/qna-rag-modules.yaml
diff --git a/manifests/fmops-qna-rag/storage-modules.yaml b/manifests/fmops-qna-rag/storage-modules.yaml
@@ -0,0 +1,22 @@
+---
+name: opensearch
+path: git::https://github.com/awslabs/idf-modules.git//modules/storage/opensearch/
+targetAccount: primary
+targetRegion: us-east-1
+parameters:
+  - name: encryption-type
+    value: SSE
+  - name: retention-type
+    value: RETAIN
+  - name: vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: VpcId
+  - name: private-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: PrivateSubnetIds
diff --git a/manifests/qna-rag-modules.yaml b/manifests/qna-rag-modules.yaml
@@ -0,0 +1,24 @@
+name: qna-rag
+path: modules/fmops/qna-rag
+parameters:
+  - name: cognito-pool-id
+    #Replace below value with valid congnito pool id
+    value: us-east-1_XXXXX
+  - name: os-domain-endpoint
+    valueFrom:
+      moduleMetadata:
+        group: storage
+        name: opensearch
+        key: OpenSearchDomainEndpoint
+  - name: os-security-group-id
+    valueFrom:
+      moduleMetadata:
+        group: storage
+        name: opensearch
+        key: OpenSearchSecurityGroupId
+  - name: vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: VpcId
diff --git a/manifests/uber-deployment.yaml b/manifests/uber-deployment.yaml
@@ -22,6 +22,8 @@ groups:
     path: manifests/mwaa-modules.yaml
   - name: mwaa-dags
     path: manifests/mwaa-dag-modules.yaml
+  - name: qna-rag
+    path: manifests/qna-rag-modules.yaml
 targetAccountMappings:
   - alias: primary
     accountId:

diff --git a/modules/fmops/qna-rag/README.md b/modules/fmops/qna-rag/README.md
@@ -2,11 +2,12 @@
 
 ## Description
 
-Deploys an AWS AppSync endpoint for a Question and Answering model using RAG 
+Deploys an AWS AppSync endpoint for ingestion of data and use it as knowledge base for a Question and Answering model using RAG 
 
 The module uses [AWS Generative AI CDK Constructs](https://github.com/awslabs/generative-ai-cdk-constructs/tree/main).
 
 ### Architecture
+![AWS Appsync Ingestion Endpoint Module Architecture](docs/_static/ingestion_architecture.png "AWS Appsync Ingestion Endpoint Module Architecture")
 
 ![AWS Appsync Question and Answering Endpoint Module Architecture](docs/_static/architecture.png "AWS Appsync Question and Answering RAG module Endpoint Module Architecture")
 
@@ -23,8 +24,12 @@ The module uses [AWS Generative AI CDK Constructs](https://github.com/awslabs/ge
 
 ### Module Metadata Outputs
 
-- `GraphqlApiId` - Graphql API ID.
-- `GraphqlArn` - Graphql API ARN.
+- `IngestionGraphqlApiId` - Ingestion Graphql API ID.
+- `IngestionGraphqlArn` - Ingestion Graphql API ARN.
+- `QnAGraphqlApiId` - Graphql API ID.
+- `QnAGraphqlArn` - Graphql API ARN.
+- `InputAssetBucket` - Input S3 bucket.
+- `ProcessedInputBucket` - S3 bucket for storing processed output.
 
 ## Examples
 
@@ -35,6 +40,7 @@ name: qna-rag
 path: modules/fmops/qna-rag
 parameters:
   - name: cognito-pool-id
+    #Replace below value with valid congnito pool id
     value: us-east-1_XXXXX
   - name: os-domain-endpoint
     valueFrom:

diff --git a/modules/fmops/qna-rag/app.py b/modules/fmops/qna-rag/app.py
@@ -43,6 +43,7 @@ def _param(name: str) -> str:
     cognito_pool_id=cognito_pool_id,
     os_domain_endpoint=os_domain_endpoint,
     os_security_group_id=os_security_group_id,
+    os_index_name="rag-index",
     env=aws_cdk.Environment(
         account=os.environ["CDK_DEFAULT_ACCOUNT"],
         region=os.environ["CDK_DEFAULT_REGION"],
@@ -54,8 +55,12 @@ def _param(name: str) -> str:
     id="metadata",
     value=stack.to_json_string(
         {
-            "GraphqlApiId": stack.rag_resource.graphql_api.api_id,
-            "GraphqlArn": stack.rag_resource.graphql_api.arn,
+            "IngestionGraphqlApiId": stack.rag_ingest_resource.graphql_api.api_id,
+            "IngestionGraphqlArn": stack.rag_ingest_resource.graphql_api.arn,
+            "QnAGraphqlApiId": stack.rag_resource.graphql_api.api_id,
+            "QnAGraphqlArn": stack.rag_resource.graphql_api.arn,
+            "InputAssetBucket": stack.rag_ingest_resource.s3_input_assets_bucket.bucket_name,
+            "ProcessedInputBucket": stack.rag_ingest_resource.s3_processed_assets_bucket.bucket_name,
         }
     ),
 )

diff --git a/modules/fmops/qna-rag/docs/_static/ingestion_architecture.png b/modules/fmops/qna-rag/docs/_static/ingestion_architecture.png
diff --git a/modules/fmops/qna-rag/requirements.in b/modules/fmops/qna-rag/requirements.in
@@ -1,5 +1,4 @@
 aws-cdk-lib==2.137.0
-cdk-nag==2.28.91
 boto3~=1.34.84
 attrs==23.2.0
-cdklabs-generative-ai-cdk-constructs==0.1.119
+cdklabs-generative-ai-cdk-constructs==0.1.146
diff --git a/modules/fmops/qna-rag/requirements.txt b/modules/fmops/qna-rag/requirements.txt
@@ -20,19 +20,17 @@ aws-cdk-lib==2.137.0
     #   -r requirements.in
     #   cdk-nag
     #   cdklabs-generative-ai-cdk-constructs
-boto3==1.34.86
+boto3==1.34.108
     # via -r requirements.in
-botocore==1.34.86
+botocore==1.34.108
     # via
     #   boto3
     #   s3transfer
 cattrs==23.2.3
     # via jsii
-cdk-nag==2.28.91
-    # via
-    #   -r requirements.in
-    #   cdklabs-generative-ai-cdk-constructs
-cdklabs-generative-ai-cdk-constructs==0.1.119
+cdk-nag==2.28.118
+    # via cdklabs-generative-ai-cdk-constructs
+cdklabs-generative-ai-cdk-constructs==0.1.145
     # via -r requirements.in
 constructs==10.3.0
     # via
@@ -45,7 +43,7 @@ jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-jsii==1.97.0
+jsii==1.98.0
     # via
     #   aws-cdk-asset-awscli-v1
     #   aws-cdk-asset-kubectl-v20
@@ -84,5 +82,5 @@ typeguard==2.13.3
     #   jsii
 typing-extensions==4.11.0
     # via jsii
-urllib3==1.25.4
+urllib3==1.26.17
     # via botocore
diff --git a/modules/fmops/qna-rag/stack.py b/modules/fmops/qna-rag/stack.py
@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from constructs import Construct
+from cdklabs.generative_ai_cdk_constructs import RagAppsyncStepfnOpensearch
 from cdklabs.generative_ai_cdk_constructs import QaAppsyncOpensearch
 from aws_cdk import Stack
 from aws_cdk import aws_ec2 as ec2
@@ -20,6 +21,7 @@ def __init__(
         cognito_pool_id: str,
         os_domain_endpoint: str,
         os_security_group_id: str,
+        os_index_name: str,
         **kwargs,
     ) -> None:
         super().__init__(
@@ -28,6 +30,7 @@ def __init__(
             description=" This stack creates resources for the LLM - QA RAG ",
             **kwargs,
         )
+
         # get an existing OpenSearch provisioned cluster
         os_domain = os.Domain.from_domain_endpoint(
             self,
@@ -49,25 +52,40 @@ def __init__(
             "myuserpool",
             user_pool_id=cognito_pool_id,
         )
+        # 1. Create Ingestion pipeline
+        rag_ingest_resource = RagAppsyncStepfnOpensearch(
+            self,
+            "RagAppsyncStepfnOpensearch",
+            existing_vpc=vpc,
+            existing_opensearch_domain=os_domain,
+            open_search_index_name=os_index_name,
+            cognito_user_pool=user_pool_loaded,
+        )
+
+        self.security_group_id = rag_ingest_resource.security_group.security_group_id
 
-        rag_source = QaAppsyncOpensearch(
+        self.rag_ingest_resource = rag_ingest_resource
+        # 2. create question and answer pipeline
+        rag_qa_source = QaAppsyncOpensearch(
             self,
             "QaAppsyncOpensearch",
             existing_vpc=vpc,
             existing_opensearch_domain=os_domain,
-            open_search_index_name="qa-appsync-index",
+            open_search_index_name=os_index_name,
             cognito_user_pool=user_pool_loaded,
+            existing_input_assets_bucket_obj=rag_ingest_resource.s3_processed_assets_bucket,
+            existing_security_group=rag_ingest_resource.security_group,
         )
 
-        security_group = rag_source.security_group
+        security_group = rag_qa_source.security_group
 
         os_security_group = ec2.SecurityGroup.from_security_group_id(
             self, "OSSecurityGroup", os_security_group_id
         )
         os_security_group.add_ingress_rule(
             peer=security_group,
             connection=ec2.Port.tcp(443),
-            description="Allow inbound HTTPS to open search from question answering lambda",
+            description="Allow inbound HTTPS to open search from embeddings lambda and question answering lambda",
         )
 
-        self.rag_resource = rag_source
+        self.rag_resource = rag_qa_source
diff --git a/modules/fmops/qna-rag/tests/test_stack.py b/modules/fmops/qna-rag/tests/test_stack.py
@@ -39,6 +39,7 @@ def stack_model_package_input() -> cdk.Stack:
         cognito_pool_id=cognito_pool_id,
         os_domain_endpoint=os_domain_endpoint,
         os_security_group_id=os_security_group_id,
+        os_index_name="sample",
         env=cdk.Environment(
             account="111111111111",
             region="us-east-1",
@@ -53,4 +54,4 @@ def stack(request, stack_model_package_input) -> cdk.Stack:  # type: ignore[no-u
 
 def test_synthesize_stack(stack: cdk.Stack) -> None:
     template = Template.from_stack(stack)
-    template.resource_count_is("AWS::AppSync::Resolver", 2)
+    template.resource_count_is("AWS::AppSync::Resolver", 4)