Merge pull request #2 from act-lilly/qa

Qa to Prd

Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.11-slim-buster
+WORKDIR /app
+COPY bedrock-chat/requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+COPY bedrock-chat/ .
+
+ENV AWS_BEDROCK_REGION="us-east-1"
+ENV LOGGING_LEVEL="INFO"
+ENV BOTOCORE_LOGGING_LEVEL="INFO"
+
+EXPOSE 8080
+CMD ["python", "-m", "streamlit", "run", "--server.enableCORS", "true", "--server.enableXsrfProtection", "true", "--server.address", "0.0.0.0", "--server.port", "8080", "main.py"]

README.md

@@ -1,2 +1,76 @@
-# aws-bedrock-chatbot-with-anthropic-claude-models-llm-inference
-Useful but simple chatbot utilizing anthropic claude models on aws bedrock
+# About
+A useful but simple chatbot utilizing Anthropic Claude models on AWS Bedrock.
+
+This app was developed entirely using Anthropic Claude 3.5 Sonnet. It began with a simple prompt: 
+> let's develop a python streamlit app that uses boto3 to utilize aws bedrock models.
+
+Currently, three of the leading Anthropic models are included as options the user can choose:
+- Claude 3.5 Sonnet
+  - Claude 3.5 Sonnet is comparable to ChatGPT 4o.
+- Claude 3 Haiku
+- Claude Instant
+
+The other Anthropic models are easy to include. Simply add them as options in main.py. 
+
+Select [this link](https://aws.amazon.com/bedrock/claude) to learn more about the Anthropic models on Bedrock.
+
+# Settings - options
+While running the app, you can do the following:
+- Change the selected model
+- Choose streaming (real time) or non-streaming (wait for it, get it all at once) mode for the model responses
+- Set custom model paramters:
+    - Temperature
+    - Top K
+- Select a system prompt from simple pre-supplied prompts.
+    - System Prompts are separate from the user prompt. They are used to instruct the model how to behave and they are included in each request a user makes.
+    - Example system prompt:
+    - > Respond to my requests and questions like a kind school teacher.
+- Create and use a **Custom system prompt** to test out various prompt techniques to tailor the bot for your use.
+- Clear chat history.
+
+# User experience
+A few keyboard control options to know while using the app:
+- You can add new lines to the input prompt by using **Shift + Enter** (aka a soft return). When you press Enter alone, it will submit the input field to the model for evaluation.
+- You can add a Custom System Prompt by selecting Custom from the options. In the Custom System Prompt text field, you will use **Control + Enter** to submit, to make the new prompt live.
+
+# Running the app
+A Dockerfile is included for reference on how to build the image to run in Docker or to deploy as a container.
+
+Otherwise, a simple start and stop script are also included to run the app, assuming python and the required python libaries have been installed. Reference the Dockerfile for guidance here too.
+
+# Authentication
+The app is intended to run with AWS credentials with an AWS account that has access to bedrock. You will need an IAM user or role with access to the becrock models to run the app.
+
+For example, you can set the environment variables where the app is running:
+```
+export AWS_ACCESS_KEY_ID=your_access_key
+export AWS_SECRET_ACCESS_KEY=your_secret_key
+```
+
+Or add these values directly to the boto3 client in main.py
+
+```
+AWS_ACCESS_KEY_ID = "your_access_key_id"
+AWS_SECRET_ACCESS_KEY = "your_secret_access_key"
+AWS_REGION = "us-east-1"  # or your preferred region
+
+bedrock = boto3.client(
+    service_name='bedrock-runtime',
+    region_name=AWS_REGION,
+    aws_access_key_id=AWS_ACCESS_KEY_ID,
+    aws_secret_access_key=AWS_SECRET_ACCESS_KEY
+)
+```
+
+# How Bedrock models are called (libraries)
+The app simply uses boto3 bedrock-runtime and the Converse API. 
+```
+import boto3
+bedrock = boto3.client( service_name='bedrock-runtime' )
+response = bedrock.converse()
+
+```
+You could instead use LangChain, for example.
+```
+from langchain_aws import ChatBedrockConverse
+```

simple_chatbot/main.py

@@ -0,0 +1,203 @@
+# file: main.py
+import streamlit as st
+import boto3
+from botocore.exceptions import ClientError
+import logging
+import textwrap
+
+def setup_logger():
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.DEBUG)
+
+    # Remove any existing handlers
+    for handler in logger.handlers[:]:
+        logger.removeHandler(handler)
+
+    # Create a new handler
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+    handler.setFormatter(formatter)
+
+    logger.addHandler(handler)
+
+    return logger
+
+# Setup the logger
+logger = setup_logger()
+
+# Initialize the AWS Bedrock client
+bedrock = boto3.client(
+    service_name='bedrock-runtime',
+    region_name='us-east-1'
+)
+
+# Streamlit title for app and page and subtitle
+st.set_page_config(page_title="Neo", layout="wide")
+st.title("neo")
+st.caption("a chatbot for claude models on aws bedrock")
+
+# Log the start of the session
+logger.info("Starting new session")
+
+# System prompt options
+SYSTEM_PROMPTS = {
+    "Default Assistant": "You are Claude, an AI assistant created by Anthropic to be helpful, harmless, and honest.",
+    "Coding Assistant": "You are a coding assistant, expert in multiple programming languages. Provide clear, concise, and efficient code solutions.",
+    "Creative Writer": "You are a creative writing assistant, skilled in various genres and styles. Help users craft engaging stories and narratives.",
+    "Custom": ""  # Empty string for custom input
+}
+
+# Model options
+MODEL_OPTIONS = {
+    "Claude 3.5 Sonnet": "anthropic.claude-3-5-sonnet-20240620-v1:0",
+    "Claude 3 Haiku": "anthropic.claude-3-haiku-20240307-v1:0",
+    "Claude Instant": "anthropic.claude-instant-v1"
+}
+
+# Initialize system_prompt in session state if it doesn't exist
+if "system_prompt" not in st.session_state:
+    st.session_state.system_prompt = SYSTEM_PROMPTS["Default Assistant"]
+    logger.info(f"Initial system prompt: {st.session_state.system_prompt}")
+
+# Function to clear chat history
+def clear_chat_history():
+    st.session_state.messages = []
+    logger.info("Chat history cleared")
+
+# Sidebar for settings
+with st.sidebar:
+    # Add a "Clear Chat History" button
+    if st.button("Clear Chat History", key="refresh_chat"):
+        clear_chat_history()
+
+    st.header("Settings")
+    use_streaming = st.checkbox("Use streaming responses", value=True)
+
+    st.subheader("Model Selection")
+    selected_model_name = st.selectbox("Select Model", list(MODEL_OPTIONS.keys()))
+    selected_model_id = MODEL_OPTIONS[selected_model_name]
+
+    st.subheader("Model Parameters")
+    temperature = st.slider("Temperature", min_value=0.0, max_value=1.0, value=0.5, step=0.01)
+    top_k = st.slider("Top K", min_value=0, max_value=500, value=100, step=10)
+
+    st.subheader("System Prompt")
+    selected_prompt_name = st.selectbox("Select System Prompt", list(SYSTEM_PROMPTS.keys()))
+
+    if selected_prompt_name == "Custom":
+        system_prompt = st.text_area("Custom System Prompt", value="", height=100)
+    else:
+        system_prompt = st.text_area("System Prompt", value=SYSTEM_PROMPTS[selected_prompt_name], height=100, disabled=True)
+
+    # Check if system prompt has changed
+    if system_prompt != st.session_state.system_prompt:
+        st.session_state.system_prompt = system_prompt
+        logger.info(f"System prompt changed to: {system_prompt}")
+
+# Initialize chat history
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+# Display chat messages from history
+for message in st.session_state.messages:
+    with st.chat_message(message["role"]):
+        st.markdown(message["content"][0]["text"])
+
+# Function to generate response from Claude using Converse API (non-streaming)
+def generate_claude_response(messages):
+    try:
+        # Ensure messages alternate between user and assistant
+        formatted_messages = []
+        for i, msg in enumerate(messages):
+            if i == 0 or msg["role"] != formatted_messages[-1]["role"]:
+                formatted_messages.append(msg)
+
+        logger.debug(f"Sending request with messages: {formatted_messages}")
+        response = bedrock.converse(
+            modelId=selected_model_id,
+            messages=formatted_messages,
+            system=[{"text": st.session_state.system_prompt}],
+            inferenceConfig={"temperature": temperature},
+            additionalModelRequestFields={"top_k": top_k}
+        )
+        logger.debug(f"Received response: {response}")
+
+        # Extract the response content
+        if 'output' in response and 'message' in response['output']:
+            message = response['output']['message']
+            if 'content' in message and len(message['content']) > 0:
+                return message['content'][0]['text']
+
+        # If the expected structure is not found, return an error message
+        st.error("Unexpected response structure from the model.")
+        logger.error(f"Unexpected response structure: {response}")
+        return None
+
+    except ClientError as e:
+        st.error(f"An error occurred: {e}")
+        logger.error(f"ClientError: {e}")
+        return None
+
+# Function to generate streaming response from Claude
+def generate_claude_streaming_response(messages):
+    try:
+        response = bedrock.converse_stream(
+            modelId=selected_model_id,
+            messages=messages,
+            system=[{"text": st.session_state.system_prompt}],
+            inferenceConfig={"temperature": temperature},
+            additionalModelRequestFields={"top_k": top_k}
+        )
+
+        for event in response.get('stream'):
+            if 'contentBlockDelta' in event:
+                yield event['contentBlockDelta']['delta']['text']
+    except ClientError as e:
+        st.error(f"An error occurred: {e}")
+        logger.error(f"ClientError in streaming: {e}")
+        yield None
+
+# Accept user input
+if prompt := st.chat_input("How may I assist?"):
+    # Log the user input
+    logger.info(f"Received user input: {prompt}")
+
+    # Add user message to chat history
+    st.session_state.messages.append({"role": "user", "content": [{"text": prompt}]})
+    # Display user message in chat message container
+    with st.chat_message("user"):
+        st.markdown(prompt)
+
+    # Prepare messages for Claude
+    claude_messages = [{"role": msg["role"], "content": msg["content"]} for msg in st.session_state.messages]
+
+    # Generate Claude's response
+    with st.chat_message("assistant"):
+        if use_streaming:
+            # Streaming response
+            logger.info("Generating streaming response")
+            response_container = st.empty()
+            full_response = ""
+            for response_chunk in generate_claude_streaming_response(claude_messages):
+                if response_chunk:
+                    full_response += response_chunk
+                    response_container.markdown(full_response + "▌")
+            response_container.markdown(full_response)
+        else:
+            # Non-streaming response
+            logger.info("Generating non-streaming response")
+            response = generate_claude_response(claude_messages)
+            if response:
+                st.markdown(response)
+                full_response = response
+            else:
+                st.error("Failed to get a response from the model.")
+                full_response = ""
+
+    # Add assistant response to chat history
+    if full_response:
+        st.session_state.messages.append({"role": "assistant", "content": [{"text": full_response}]})
+
+    # Log the assistant's response
+    response_summary = textwrap.shorten(full_response, width=1000, placeholder="...")
+    logger.info(f"Generated assistant response: {response_summary}")

simple_chatbot/requirements.txt

@@ -0,0 +1,2 @@
+streamlit
+boto3

simple_chatbot/start.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/local/bin/python -m streamlit run --server.enableCORS true --server.enableXsrfProtection true --server.address 0.0.0.0 --server.port 8080 main.py & echo "Service started with process ID: $!"

simple_chatbot/stop.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+pgrep -f "streamlit run --server.enableCORS true --server.enableXsrfProtection true --server.address 0.0.0.0 --server.port 8080 main.py" | xargs kill